From 53b50bfbc80fbe5cd72eba93d5baac7ef47a7e31 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Fri, 19 Jan 2024 16:46:12 -0600 Subject: [PATCH 01/29] Collect function. --- .../Entities/External/IExternalDataSource.cs | 2 + .../Localization/Strings.cs | 4 + .../Texl/Builtins/Collect.cs | 374 ++++++++++++++++++ .../Microsoft.PowerFx.Core/Types/DType.cs | 2 + .../Utils/MutationUtils.cs | 67 ++++ .../Environment/PowerFxConfigExtensions.cs | 4 +- .../Functions/LibraryMutation.cs | 134 +++++++ .../Functions/Mutation/MutationUtils.cs | 35 -- src/strings/PowerFxResources.en-US.resx | 10 + .../TestDVEntity.cs | 2 + .../ExpressionTestCases/Collect.txt | 18 +- .../ExpressionTestCases/Collect_V1Compat.txt | 2 +- .../Collect_V1CompatDisabled.txt | 2 +- .../Helpers/TestTabularDataSource.cs | 2 + .../CollectFunctionTests.cs | 2 +- .../MutationFunctionsTests.cs | 20 + .../MutationScripts/Collect.txt | 30 +- 17 files changed, 664 insertions(+), 46 deletions(-) create mode 100644 src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs create mode 100644 src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs create mode 100644 src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs delete mode 100644 src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs diff --git a/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalDataSource.cs b/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalDataSource.cs index ed7791bb8a..e5e184e182 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalDataSource.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalDataSource.cs @@ -19,6 +19,8 @@ internal interface IExternalDataSource : IExternalEntity, IExternalPageableSymbo bool RequiresAsync { get; } + bool IsWritable { get; } + IExternalDataEntityMetadataProvider DataEntityMetadataProvider { get; } DataSourceKind Kind { get; } diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs index 025ad7b716..2bedf2c33e 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs @@ -485,6 +485,10 @@ internal static class TexlStrings public static StringGetter CollectDataSourceArg = (b) => StringResources.Get("CollectDataSourceArg", b); public static StringGetter CollectRecordArg = (b) => StringResources.Get("CollectRecordArg", b); + public static StringGetter CollectItemArg = (b) => StringResources.Get("CollectItemArg", b); + public static StringGetter AboutCollect_data_source = (b) => StringResources.Get("AboutCollect_data_source", b); + public static StringGetter AboutCollect_item = (b) => StringResources.Get("AboutCollect_item", b); + public static StringGetter AboutClearCollect = (b) => StringResources.Get("AboutClearCollect", b); public static StringGetter ClearCollectDataSourceArg = (b) => StringResources.Get("ClearCollectDataSourceArg", b); public static StringGetter ClearCollectRecordArg = (b) => StringResources.Get("ClearCollectRecordArg", b); diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs new file mode 100644 index 0000000000..ce24b5fa11 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -0,0 +1,374 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.PowerFx.Core.App.ErrorContainers; +using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Entities; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Functions.FunctionArgValidators; +using Microsoft.PowerFx.Core.Localization; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Core.Utils; +using Microsoft.PowerFx.Syntax; +using Microsoft.PowerFx.Types; + +namespace Microsoft.PowerFx.Core.Texl.Builtins +{ + // Collect(collection:*[...], item1:![...]|*[...], ...) + internal class CollectFunction : BuiltinFunction + { + public override bool AffectsCollectionSchemas => true; + + public bool CanSuggestThisItem => true; + + public override bool SupportsParamCoercion => false; + + public override bool ManipulatesCollections => true; + + public override bool ModifiesValues => true; + + public override bool IsSelfContained => false; + + public override bool RequiresDataSourceScope => true; + + protected virtual bool IsScalar => false; + + public override bool CanSuggestInputColumns => true; + + public override bool MutatesArg0 => true; + + /// + /// Since Arg1 and Arg2 depends on type of Arg1 return false for them. + /// + public override bool TryGetTypeForArgSuggestionAt(int argIndex, out DType type) + { + if (argIndex == 1 || argIndex == 2) + { + type = default; + return false; + } + + return base.TryGetTypeForArgSuggestionAt(argIndex, out type); + } + + public override bool ArgMatchesDatasourceType(int argNum) + { + return argNum >= 1; + } + + public override bool IsLazyEvalParam(int index, Features features) + { + // First argument to mutation functions is Lazy for datasources that are copy-on-write. + // If there are any side effects in the arguments, we want those to have taken place before we make the copy. + return index == 0; + } + + public CollectFunction() + : this("Collect", TexlStrings.AboutCollect) + { + } + + protected CollectFunction(string name, TexlStrings.StringGetter description) + : base(name, description, FunctionCategories.Behavior, DType.EmptyRecord, 0, 2, int.MaxValue, DType.EmptyTable) + { + } + + public override IEnumerable GetSignatures() + { + yield return new[] { TexlStrings.CollectDataSourceArg, TexlStrings.CollectItemArg }; + } + + public override IEnumerable GetSignatures(int arity) + { + if (arity > 2) + { + return GetGenericSignatures(arity, TexlStrings.CollectDataSourceArg, TexlStrings.CollectItemArg); + } + + return base.GetSignatures(arity); + } + + public virtual DType GetCollectedType(PowerFx.Features features, DType argType) + { + Contracts.Assert(argType.IsValid); + + return argType; + } + + // Attempt to get the unified schema of the items being collected by an invocation. + private bool TryGetUnifiedCollectedType(CheckTypesContext context, TexlNode[] args, DType[] argTypes, IErrorContainer errors, out DType collectedType) + { + Contracts.AssertValue(args); + Contracts.AssertAllValues(args); + Contracts.AssertValue(argTypes); + Contracts.Assert(args.Length == argTypes.Length); + Contracts.AssertValue(errors); + Contracts.Assert(MinArity <= args.Length && args.Length <= MaxArity); + + var fValid = true; + + DType itemType = DType.Invalid; + DType datasourceType = argTypes[0]; + + var argc = args.Length; + + for (var i = 1; i < argc; i++) + { + DType argType = GetCollectedType(context.Features, argTypes[i]); + + // The subsequent args should all be aggregates. + if (!argType.IsAggregate) + { + errors.EnsureError(args[i], TexlStrings.ErrBadType_Type, argType.GetKindString()); + fValid = false; + continue; + } + + // Promote the arg type to a table to facilitate unioning. + if (!argType.IsRecord) + { + argType = argType.ToRecord(); + } + + // Checks if all record names exist against table type and if its possible to coerce. + bool checkAggregateNames = argType.CheckAggregateNames(datasourceType, args[i], errors, context.Features, SupportsParamCoercion); + fValid = fValid && checkAggregateNames; + + if (!itemType.IsValid) + { + itemType = argType; + } + else + { + var fUnionError = false; + itemType = DType.Union(ref fUnionError, itemType, argType, useLegacyDateTimeAccepts: true, context.Features); + if (fUnionError) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrIncompatibleTypes); + fValid = false; + } + } + + // We only support accessing entities in collections if the collection has only 1 argument that contributes to it's type + if (argc != 2 && itemType.ContainsDataEntityType(DPath.Root)) + { + fValid &= DropAllOfKindNested(ref itemType, errors, args[i], DKind.DataEntity); + } + } + + Contracts.Assert(!itemType.IsValid || itemType.IsRecord); + collectedType = itemType.IsValid ? itemType : DType.EmptyRecord; + return fValid; + } + + // Typecheck an invocation of Collect. + public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DType[] argTypes, IErrorContainer errors, out DType returnType, out Dictionary nodeToCoercedTypeMap) + { + Contracts.AssertValue(args); + Contracts.AssertAllValues(args); + Contracts.AssertValue(argTypes); + Contracts.Assert(args.Length == argTypes.Length); + Contracts.AssertValue(errors); + Contracts.Assert(MinArity <= args.Length && args.Length <= MaxArity); + + var fValid = base.CheckTypes(context, args, argTypes, errors, out returnType, out nodeToCoercedTypeMap); + + // Need a collection for the 1st arg + DType collectionType = argTypes[0]; + if (!collectionType.IsTable) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrInvalidArgs_Func, Name); + fValid = false; + } + + // Get the unified collected type on the RHS. This will generate appropriate + // document errors for invalid arguments such as unsupported aggregate types. + fValid &= TryGetUnifiedCollectedType(context, args, argTypes, errors, out DType collectedType); + Contracts.Assert(collectedType.IsRecord); + + if (fValid) + { + if (!collectedType.TryGetCoercionSubType(collectionType, out DType coercionType, out var coercionNeeded, context.Features)) + { + fValid = false; + } + else + { + if (coercionNeeded) + { + CollectionUtils.Add(ref nodeToCoercedTypeMap, args[1], coercionType); + } + + var fError = false; + + returnType = DType.Union(ref fError, collectionType.ToRecord(), collectedType, useLegacyDateTimeAccepts: false, context.Features, allowCoerce: true); + + if (argTypes.Length == 2) + { + if (argTypes[1].IsTable && argTypes[1].Kind != DKind.ObjNull) + { + returnType = returnType.ToTable(); + } + } + else + { + returnType = returnType.ToTable(); + } + + if (fError) + { + fValid = false; + if (!SetErrorForMismatchedColumns(collectionType, collectedType, args[1], errors, context.Features)) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrTableDoesNotAcceptThisType); + } + } + } + } + + return fValid; + } + + public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors) + { + DType dataSourceType = argTypes[0]; + bool isConnected = binding.EntityScope != null + && binding.EntityScope.TryGetDataSource(args[0], out IExternalDataSource dataSourceInfo) + && (dataSourceInfo.Kind == DataSourceKind.Connected || dataSourceInfo.Kind == DataSourceKind.CdsNative); + + if (isConnected) + { + for (int i = 1; i < args.Length; i++) + { + DType curType = argTypes[i]; + foreach (var typedName in curType.GetNames(DPath.Root)) + { + DName name = typedName.Name; + if (!dataSourceType.TryGetType(name, out DType dsNameType)) + { + dataSourceType.ReportNonExistingName(FieldNameKind.Display, errors, name, args[i], DocumentErrorSeverity.Warning); + } + } + } + } + + base.CheckSemantics(binding, args, argTypes, errors); + base.ValidateArgumentIsMutable(binding, args[0], errors); + + int skip = 1; + + MutationUtils.CheckSemantics(binding, this, args, argTypes, errors); + MutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(skip).ToArray(), argTypes.Skip(skip).ToArray(), errors); + } + + // This method returns true if there are special suggestions for a particular parameter of the function. + public override bool HasSuggestionsForParam(int argumentIndex) + { + Contracts.Assert(argumentIndex >= 0); + + return argumentIndex == 0; + } + + public override bool TryGetDataSourceNodes(CallNode callNode, TexlBinding binding, out IList dsNodes) + { + Contracts.AssertValue(callNode); + Contracts.AssertValue(binding); + + dsNodes = new List(); + if (callNode.Args.Count != 2) + { + return false; + } + + var args = Contracts.VerifyValue(callNode.Args.Children); + var arg1 = Contracts.VerifyValue(args[1]); + + // Only the second arg can contribute to the output for the purpose of delegation + return ArgValidators.DataSourceArgNodeValidator.TryGetValidValue(arg1, binding, out dsNodes); + } + + public override IEnumerable GetIdentifierOfModifiedValue(TexlNode[] args, out TexlNode identifierNode) + { + Contracts.AssertValue(args); + + identifierNode = null; + if (args.Length == 0) + { + return null; + } + + var firstNameNode = args[0]?.AsFirstName(); + identifierNode = firstNameNode; + + if (firstNameNode == null) + { + return null; + } + + var identifiers = new List + { + firstNameNode.Ident + }; + return identifiers; + } + + public override bool IsAsyncInvocation(CallNode callNode, TexlBinding binding) + { + Contracts.AssertValue(callNode); + Contracts.AssertValue(binding); + + return Arg0RequiresAsync(callNode, binding); + } + + public static DType GetCollectedTypeForGivenArgType(Features features, DType argType) + { + Contracts.Assert(argType.IsValid); + + if (!argType.IsPrimitive) + { + return argType; + } + + // Passed a scalar; make a record out of it, using a name that depends on the type. + string fieldName = Contracts.VerifyValue(CreateInvariantFieldName(features, argType.Kind)); + return DType.CreateRecord(new TypedName[] { new TypedName(argType, new DName(fieldName)) }); + } + + protected static string CreateInvariantFieldName(PowerFx.Features features, DKind dKind) + { + Contracts.Assert(dKind >= DKind._Min && dKind < DKind._Lim); + + return GetScalarSingleColumnNameForType(features, dKind); + } + + private static string GetScalarSingleColumnNameForType(Features features, DKind kind) + { + return kind switch + { + DKind.Image or + DKind.Hyperlink or + DKind.Media or + DKind.Blob or + DKind.PenImage => features.ConsistentOneColumnTableResult ? TableValue.ValueName : "Url", + + _ => TableValue.ValueName + }; + } + } + + // Collect(collection:*[...], item1, ...) + internal class CollectScalarFunction : CollectFunction + { + protected override bool IsScalar => true; + + public override bool SupportsParamCoercion => false; + + public override DType GetCollectedType(PowerFx.Features features, DType argType) + { + return GetCollectedTypeForGivenArgType(features, argType); + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs index a4ef4f77ff..d37679f0f6 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs @@ -637,6 +637,8 @@ public bool IsActivityPointer public bool HasPolymorphicInfo => PolymorphicInfo != null; + public bool IsSingleColumnTable => IsTable && GetNames(DPath.Root).Count() == 1; + /// /// Whether this type is a subtype of all possible types, meaning that it can be placed in /// any location without coercion. diff --git a/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs new file mode 100644 index 0000000000..cf5d4fe3c4 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Linq; +using Microsoft.PowerFx.Core.App.ErrorContainers; +using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Entities; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Localization; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Syntax; + +namespace Microsoft.PowerFx.Core.Utils +{ + internal class MutationUtils + { + public static void CheckForReadOnlyFields(DType dataSourceType, TexlNode[] args, DType[] argTypes, IErrorContainer errors) + { + if (dataSourceType.AssociatedDataSources.Any()) + { + var tableDsInfo = dataSourceType.AssociatedDataSources.Single(); + + if (tableDsInfo is IExternalCdsDataSource cdsTableInfo) + { + for (int i = 0; i < argTypes.Length; i++) + { + if (!cdsTableInfo.IsArgTypeValidForMutation(argTypes[i], out var invalidFieldNames)) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrRecordContainsInvalidFields_Arg, string.Join(", ", invalidFieldNames)); + } + } + } + } + } + + /// + /// Adds specific errors for mutation functions. + /// + public static void CheckSemantics(TexlBinding binding, TexlFunction function, TexlNode[] args, DType[] argTypes, IErrorContainer errors) + { + var targetArg = args[0]; + + // control!property are not valid targets for Collect, Remove, etc. + DottedNameNode dotted; + if ((dotted = targetArg.AsDottedName()) != null + && binding.TryCastToFirstName(dotted.Left, out var firstNameInfo) + && firstNameInfo.Kind == BindKind.Control) + { + errors.EnsureError(targetArg, TexlStrings.ErrInvalidArgs_Func, function.Name); + return; + } + + // Checks for something similar to Collect(x.a, 4). + if (!binding.TryCastToFirstName(targetArg, out firstNameInfo)) + { + return; + } + + if (firstNameInfo.Data is IExternalDataSource info && !info.IsWritable) + { + errors.EnsureError(targetArg, TexlStrings.ErrInvalidArgs_Func, function.Name); + return; + } + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs index 8ea79c6a0d..c024274451 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Texl.Builtins; using Microsoft.PowerFx.Functions; using Microsoft.PowerFx.Interpreter; @@ -37,11 +38,12 @@ public static void EnableSetFunction(this PowerFxConfig powerFxConfig) public static void EnableMutationFunctions(this SymbolTable symbolTable) { symbolTable.AddFunction(new RecalcEngineSetFunction()); - symbolTable.AddFunction(new CollectFunction()); symbolTable.AddFunction(new PatchFunction()); symbolTable.AddFunction(new RemoveFunction()); symbolTable.AddFunction(new ClearFunction()); symbolTable.AddFunction(new ClearCollectFunction()); + symbolTable.AddFunction(new CollectImpl()); + symbolTable.AddFunction(new CollectScalarImpl()); } [Obsolete("RegEx is still in preview. Grammar may change.")] diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs new file mode 100644 index 0000000000..cc8c3a392e --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Functions; +using Microsoft.PowerFx.Interpreter; +using Microsoft.PowerFx.Types; + +namespace Microsoft.PowerFx.Core.Texl.Builtins +{ + internal class CollectImpl : CollectFunction, IAsyncTexlFunction + { + public async Task InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken) + { + return await new CollectProcess().Process(args, cancellationToken).ConfigureAwait(false); + } + } + + internal class CollectScalarImpl : CollectScalarFunction, IAsyncTexlFunction + { + public async Task InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken) + { + return await new CollectProcess().Process(args, cancellationToken).ConfigureAwait(false); + } + } + + internal class CollectProcess + { + internal async Task Process(FormulaValue[] args, CancellationToken cancellationToken) + { + FormulaValue arg0; + var argc = args.Length; + var returnIsTable = args.Length > 2; + + // Need to check if the Lazy first argument has been evaluated since it may have already been + // evaluated in the ClearCollect case. + if (args[0] is LambdaFormulaValue arg0lazy) + { + arg0 = await arg0lazy.EvalAsync().ConfigureAwait(false); + } + else + { + arg0 = args[0]; + } + + if (arg0 is BlankValue) + { + return arg0; + } + + if (arg0 is ErrorValue) + { + return arg0; + } + + if (arg0 is not TableValue) + { + return CommonErrors.RuntimeTypeMismatch(IRContext.NotInSource(arg0.Type)); + } + + var tableValue = arg0 as TableValue; + + List> resultRows = new List>(); + + for (int i = 1; i < argc; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var arg = args[i]; + + if (arg is TableValue argTableValue) + { + returnIsTable = true; + + foreach (DValue row in argTableValue.Rows) + { + if (row.IsBlank) + { + continue; + } + else if (row.IsError) + { + return row.Error; + } + else + { + var recordValueCopy = (RecordValue)row.ToFormulaValue().MaybeShallowCopy(); + resultRows.Add(await tableValue.AppendAsync(recordValueCopy, cancellationToken).ConfigureAwait(false)); + } + } + } + else if (arg is RecordValue) + { + var recordValueCopy = (RecordValue)arg.MaybeShallowCopy(); + resultRows.Add(await tableValue.AppendAsync(recordValueCopy, cancellationToken).ConfigureAwait(false)); + } + else if (arg is ErrorValue) + { + return arg; + } + else if (arg is BlankValue && !tableValue.Type._type.IsSingleColumnTable) + { + continue; + } + else + { + // If arg is a scalar value, then we need to create a single column record. + NamedValue namedValue = new NamedValue(tableValue.Type.SingleColumnFieldName, arg); + var singleColumnRecord = FormulaValue.NewRecordFromFields(namedValue); + + resultRows.Add(await tableValue.AppendAsync(singleColumnRecord, cancellationToken).ConfigureAwait(false)); + } + } + + if (resultRows.Count == 0) + { + return FormulaValue.NewBlank(arg0.Type); + } + else if (returnIsTable) + { + return CompileTimeTypeWrapperTableValue.AdjustType(tableValue.Type, new InMemoryTableValue(IRContext.NotInSource(arg0.Type), resultRows)); + } + else + { + return CompileTimeTypeWrapperRecordValue.AdjustType(tableValue.Type.ToRecord(), (RecordValue)resultRows.First().ToFormulaValue()); + } + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs deleted file mode 100644 index 224c9d1cd7..0000000000 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Linq; -using Microsoft.PowerFx.Core.App.ErrorContainers; -using Microsoft.PowerFx.Core.Entities; -using Microsoft.PowerFx.Core.Errors; -using Microsoft.PowerFx.Core.Localization; -using Microsoft.PowerFx.Core.Types; -using Microsoft.PowerFx.Syntax; - -namespace Microsoft.PowerFx.Interpreter -{ - internal class MutationUtils - { - public static void CheckForReadOnlyFields(DType dataSourceType, TexlNode[] args, DType[] argTypes, IErrorContainer errors) - { - if (dataSourceType.AssociatedDataSources.Any()) - { - var tableDsInfo = dataSourceType.AssociatedDataSources.Single(); - - if (tableDsInfo is IExternalCdsDataSource cdsTableInfo) - { - for (int i = 0; i < argTypes.Length; i++) - { - if (!cdsTableInfo.IsArgTypeValidForMutation(argTypes[i], out var invalidFieldNames)) - { - errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrRecordContainsInvalidFields_Arg, string.Join(", ", invalidFieldNames)); - } - } - } - } - } - } -} diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index 56dd88c664..6386ed63a1 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -4430,4 +4430,14 @@ **Disclaimer:** AI-generated content can have mistakes. Make sure it's accurate and appropriate before using it. [See terms](https://go.microsoft.com/fwlink/?linkid=2225491) The disclaimer we show on AI functions. This is github flavored markdown. So ** marks text in bold. + + The data source that you want to add data to. + + + A record or table to collect. A record will be appended to the datasource. A table will have its rows appended to the datasource. + + + item + function_parameter - Second parameter for the Collect function. The item to be added. + \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/AssociatedDataSourcesTests/TestDVEntity.cs b/src/tests/Microsoft.PowerFx.Core.Tests/AssociatedDataSourcesTests/TestDVEntity.cs index e93b903077..f0bda8e028 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/AssociatedDataSourcesTests/TestDVEntity.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests/AssociatedDataSourcesTests/TestDVEntity.cs @@ -30,6 +30,8 @@ public class AccountsEntity : IExternalEntity, IExternalDataSource public bool IsPageable => true; + public bool IsWritable => true; + DType IExternalEntity.Type => AccountsTypeHelper.GetDType(); IExternalDataEntityMetadataProvider IExternalDataSource.DataEntityMetadataProvider => throw new NotImplementedException(); diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt index 80bcf18900..8e9b0c6959 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt @@ -27,19 +27,19 @@ true {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false} >> Collect(t1) -Errors: Error 0-11: Invalid number of arguments: received 1, expected 2. +Errors: Error 0-11: Invalid number of arguments: received 1, expected 2 or more. >> Collect(t1, r2, r2) -Errors: Error 0-19: Invalid number of arguments: received 3, expected 2. +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) >> Collect(t1, r2, 1; 2; r2) -Errors: Error 0-25: Invalid number of arguments: received 3, expected 2. +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) >> Collect(t1, "x") -Errors: Error 12-15: Invalid argument type (Text). Expecting a Record value instead.|Error 12-15: Invalid argument type. Cannot use Text values in this context.|Error 0-16: The function 'Collect' has some invalid arguments. +Errors: Error 0-16: The function 'Collect' has some invalid arguments.|Error 12-15: Invalid argument type. Cannot use Text values in this context. >> Collect(t1, 1) -Errors: Error 12-13: Invalid argument type (Decimal). Expecting a Record value instead.|Error 12-13: Invalid argument type. Cannot use Decimal values in this context.|Error 0-14: The function 'Collect' has some invalid arguments. +Errors: Error 0-14: The function 'Collect' has some invalid arguments.|Error 12-13: Invalid argument type. Cannot use Decimal values in this context. >> Collect(Foo,r2) Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-14: The specified column 'Field1' does not exist.|Error 0-15: The function 'Collect' has some invalid arguments. @@ -54,4 +54,10 @@ Errors: Error 12-15: Name isn't valid. 'Foo' isn't recognized. Errors: Error 11-22: The specified column 'Price' does not exist.|Error 0-23: The function 'Collect' has some invalid arguments.|Error 23-29: Name isn't valid. 'Price' isn't recognized. >> Collect(t_empty,{Value:200}).Value -200 \ No newline at end of file +200 + +>> Collect(t1, Table(r2,r2)) +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Collect(t1, Table(r2,r2), {Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt index db0ffc0398..dbc64f52b6 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt @@ -19,4 +19,4 @@ Errors: Error 17-19: The specified column 'Field1' does not exist.|Error 8-15: T Errors: Error 8-15: The value passed to the 'Collect' function cannot be changed. >> Collect("", "") -Errors: Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type (Text). Expecting a Record value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed.|Error 0-15: The function 'Collect' has some invalid arguments. +Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt index 228b45371b..39237bfccb 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt @@ -19,4 +19,4 @@ Errors: Error 17-19: The specified column 'Field1' does not exist.|Error 0-20: T Blank() >> Collect("", "") -Errors: Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type (Text). Expecting a Record value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 0-15: The function 'Collect' has some invalid arguments. +Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context. \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/Helpers/TestTabularDataSource.cs b/src/tests/Microsoft.PowerFx.Core.Tests/Helpers/TestTabularDataSource.cs index cb708b3a17..5b2e98b43a 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/Helpers/TestTabularDataSource.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests/Helpers/TestTabularDataSource.cs @@ -219,6 +219,8 @@ internal TestDataSource(string name, DType schema, string[] keyColumns = null, I IDelegationMetadata IExternalDataSource.DelegationMetadata => DelegationMetadata; + public bool IsWritable => throw new NotImplementedException(); + public bool CanIncludeExpand(IExpandInfo expandToAdd) { throw new NotImplementedException(); diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/CollectFunctionTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests/CollectFunctionTests.cs index c1673211d7..4f0fafd52b 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/CollectFunctionTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/CollectFunctionTests.cs @@ -61,7 +61,7 @@ public async Task AppendCountTest(string script, int expected) [InlineData("Collect(lazyTable, lazyCoercibleRecord)", true)] [InlineData("Collect(lazyTable, lazyNotCoercibleRecord)", false)] [InlineData("Collect(lazyTable, {Value:1})", false)] - [InlineData("Collect(lazyTable, lazyTable)", false)] + [InlineData("Collect(lazyTable, lazyTable)", true)] [InlineData("Collect(lazyRecord, lazyRecord)", false)] [InlineData("Collect(lazyRecord, lazyTable)", false)] diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs index ce59e88dbb..70a16670ee 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs @@ -364,6 +364,26 @@ public void DontCopyDerivedRecordValuesTest(string expression) Assert.Equal("x", fileObjectRecordValue.SomeProperty); } + [Fact] + public void SymbolTableEnableMutationFuntionsTest() + { + var expr = "Collect()"; + var engine = new RecalcEngine(); + + var symbolTable = new SymbolTable(); + var symbolTableEnabled = new SymbolTable(); + + symbolTableEnabled.EnableMutationFunctions(); + + // Mutation functions not listed. + var check = engine.Check(expr, symbolTable: symbolTable); + Assert.DoesNotContain(check.Symbols.Functions.FunctionNames, f => f == "Collect"); + + // Mutation functions is listed. + var checkEnabled = engine.Check(expr, symbolTable: symbolTableEnabled); + Assert.Contains(checkEnabled.Symbols.Functions.FunctionNames, f => f == "Collect"); + } + internal class FileObjectRecordValue : InMemoryRecordValue { public string SomeProperty { get; set; } diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt index c76b69f83a..98c150f492 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt @@ -49,7 +49,7 @@ Blank() Table({Value:1},{Value:2},{Value:3}) >> Collect(temp1,{Value:"200"}).Value -Errors: Error 14-27: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 0-28: The function 'Collect' has some invalid arguments.|Error 28-34: Name isn't valid. 'Value' isn't recognized. +Errors: Error 14-27: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 0-28: The function 'Collect' has some invalid arguments. >> Collect( temp1, { Value:"11"+0 } ) {Value:11} @@ -86,3 +86,31 @@ Table({Value:1},{Value:2},{Value:3},{Value:4}) >> With({r:{Value:5}}, Collect(t, r); Patch(t, Last(t), {Value:-1}); r) {Value:5} + +>> Set(t1, [1,2,3]) +Table({Value:1},{Value:2},{Value:3}) + +>> Collect(t1, 4) +{Value:4} + +>> Collect(t1, [5,6,7],8,9,10) +Table({Value:5},{Value:6},{Value:7},{Value:8},{Value:9},{Value:10}) + +>> 0;t1 +Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7},{Value:8},{Value:9},{Value:10}) + +>> Collect(t1, Table({Value:11},Blank())) +Table({Value:11}) + +>> Collect(t1, Table({Value:12},Error({Kind:ErrorKind.Custom}))) +Error({Kind:ErrorKind.Custom}) + +>> Collect(t1, [13,Blank(),14]) +Table({Value:13},{Value:Blank()},{Value:14}) + +// {Value:16} wont be collected because it appears after the error. +>> Collect(t1, Table({Value:15}, Error({Kind:ErrorKind.Custom}), {Value:16})) +Error({Kind:ErrorKind.Custom}) + +>> 1;t1 +Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7},{Value:8},{Value:9},{Value:10},{Value:11},{Value:12},{Value:13},{Value:Blank()},{Value:14},{Value:15}) \ No newline at end of file From 0750a3440aa4001ff83c971a4e26f7d04e5f1c05 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Tue, 30 Jan 2024 21:01:40 -0600 Subject: [PATCH 02/29] PA fixes --- .../Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index ce24b5fa11..9549019ce2 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -34,7 +34,7 @@ internal class CollectFunction : BuiltinFunction public override bool RequiresDataSourceScope => true; - protected virtual bool IsScalar => false; + public virtual bool IsScalar => false; public override bool CanSuggestInputColumns => true; @@ -362,10 +362,17 @@ DKind.Blob or // Collect(collection:*[...], item1, ...) internal class CollectScalarFunction : CollectFunction { - protected override bool IsScalar => true; + public override bool IsScalar => true; public override bool SupportsParamCoercion => false; + // This method returns the name of the record to be used when a scalar is passed to Collect and converted. + // It is critical that these are *always* the invariant names. + public static string GetInvariantNameForRecord(PowerFx.Features features, DKind dKind) + { + return CreateInvariantFieldName(features, dKind); + } + public override DType GetCollectedType(PowerFx.Features features, DType argType) { return GetCollectedTypeForGivenArgType(features, argType); From b4d96ec043deff26654444f569d62befe4606f60 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Thu, 8 Feb 2024 14:25:26 -0600 Subject: [PATCH 03/29] Working on V1 compat. --- .../Texl/Builtins/Collect.cs | 24 +- .../Functions/LibraryMutation.cs | 18 +- .../ExpressionTestCases/Collect.txt | 9 - .../ExpressionTestCases/Collect_V1Compat.txt | 9 + .../Collect_V1CompatDisabled.txt | 12 +- .../AndOr_V1CompatDisabled.txt | 12 +- .../Coalesce_CoalesceShortCircuitDisabled.txt | 12 +- .../MutationScripts/DeepMutation.txt | 2 +- .../DeepMutation_V1CompatDisabled.txt | 412 ++++++++++++++++++ .../MutationScripts/ForAllMutate.txt | 2 +- .../ForAllMutate_V1CompatDisabled.txt | 107 +++++ 11 files changed, 571 insertions(+), 48 deletions(-) create mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt create mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 9549019ce2..8c9ffc786c 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -206,18 +206,6 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp returnType = DType.Union(ref fError, collectionType.ToRecord(), collectedType, useLegacyDateTimeAccepts: false, context.Features, allowCoerce: true); - if (argTypes.Length == 2) - { - if (argTypes[1].IsTable && argTypes[1].Kind != DKind.ObjNull) - { - returnType = returnType.ToTable(); - } - } - else - { - returnType = returnType.ToTable(); - } - if (fError) { fValid = false; @@ -229,6 +217,18 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp } } + if (context.Features.PowerFxV1CompatibilityRules && argTypes.Length == 2) + { + if (argTypes[1].IsTable && argTypes[1].Kind != DKind.ObjNull) + { + returnType = returnType.ToTable(); + } + } + else + { + returnType = returnType.ToTable(); + } + return fValid; } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index cc8c3a392e..2f04e566e0 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -13,29 +13,29 @@ namespace Microsoft.PowerFx.Core.Texl.Builtins { - internal class CollectImpl : CollectFunction, IAsyncTexlFunction + internal class CollectImpl : CollectFunction, IAsyncTexlFunction3 { - public async Task InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken) + public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { - return await new CollectProcess().Process(args, cancellationToken).ConfigureAwait(false); + return await new CollectProcess().Process(irContext, args, cancellationToken).ConfigureAwait(false); } } - internal class CollectScalarImpl : CollectScalarFunction, IAsyncTexlFunction + internal class CollectScalarImpl : CollectScalarFunction, IAsyncTexlFunction3 { - public async Task InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken) + public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { - return await new CollectProcess().Process(args, cancellationToken).ConfigureAwait(false); + return await new CollectProcess().Process(irContext, args, cancellationToken).ConfigureAwait(false); } } internal class CollectProcess { - internal async Task Process(FormulaValue[] args, CancellationToken cancellationToken) + internal async Task Process(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { FormulaValue arg0; var argc = args.Length; - var returnIsTable = args.Length > 2; + var returnIsTable = irContext._type.IsTable; // Need to check if the Lazy first argument has been evaluated since it may have already been // evaluated in the ClearCollect case. @@ -75,8 +75,6 @@ internal async Task Process(FormulaValue[] args, CancellationToken if (arg is TableValue argTableValue) { - returnIsTable = true; - foreach (DValue row in argTableValue.Rows) { if (row.IsBlank) diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt index 8e9b0c6959..8fa72a0608 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt @@ -14,18 +14,12 @@ Blank() >> Collect(t1, If(1>0, r2));CountRows(t1) 2 ->> Collect(t_name, {name: "textInput1"}) -{name:"textInput1"} - >> IsBlank(Collect(t1, Blank())) true >> IsError(Collect(t1, If(1/0, r2))) true ->> Collect(t1, {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}) -{Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false} - >> Collect(t1) Errors: Error 0-11: Invalid number of arguments: received 1, expected 2 or more. @@ -53,9 +47,6 @@ Errors: Error 12-15: Name isn't valid. 'Foo' isn't recognized. >> Collect(t1,{Price:200}).Price Errors: Error 11-22: The specified column 'Price' does not exist.|Error 0-23: The function 'Collect' has some invalid arguments.|Error 23-29: Name isn't valid. 'Price' isn't recognized. ->> Collect(t_empty,{Value:200}).Value -200 - >> Collect(t1, Table(r2,r2)) Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt index dbc64f52b6..22aa02418f 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt @@ -1,5 +1,14 @@ #SETUP: EnableExpressionChaining,MutationFunctionsTestSetup,PowerFxV1CompatibilityRules +>> Collect(t_name, {name: "textInput1"}) +{name:"textInput1"} + +>> Collect(t1, {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}) +{Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false} + +>> Collect(t_empty,{Value:200}).Value +200 + >> Collect(Table({name: "VC"}), {surname: "textInput1"}) Errors: Error 29-52: The specified column 'surname' does not exist. The column with the most similar name is 'name'.|Error 8-27: The value passed to the 'Collect' function cannot be changed.|Error 0-53: The function 'Collect' has some invalid arguments. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt index 39237bfccb..098011a9db 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt @@ -1,11 +1,17 @@ #SETUP: EnableExpressionChaining,MutationFunctionsTestSetup,disable:PowerFxV1CompatibilityRules +>> Collect(FirstN(t_name, 0), {name: "textInput1"}) +Table({name:"textInput1"}) + +>> Collect(t1, {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}) +Table({Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}) + +>> Last(Collect(t_empty,{Value:200})).Value +200 + >> Collect(Table({name: "VC"}), {surname: "textInput1"}) Errors: Error 29-52: The specified column 'surname' does not exist. The column with the most similar name is 'name'.|Error 0-53: The function 'Collect' has some invalid arguments. ->> Collect(FirstN(t_name, 0), {name: "textInput1"}) -{name:"textInput1"} - >> Collect(Error({Kind:ErrorKind.Custom}), r2) Errors: Error 40-42: The specified column 'Field1' does not exist.|Error 0-43: The function 'Collect' has some invalid arguments. diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/AndOr_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/AndOr_V1CompatDisabled.txt index 28cf875118..eb98a43650 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/AndOr_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/AndOr_V1CompatDisabled.txt @@ -6,7 +6,7 @@ Table({Value:false},{Value:true}) // AND ->> false && Collect(t1, {Value:true}).Value;t1 +>> false && Last(Collect(t1, {Value:true})).Value;t1 Table({Value:false},{Value:true}) >> 0.00;t1 @@ -30,7 +30,7 @@ Table({Value:false},{Value:true}) >> 0.03;t1 Table({Value:false},{Value:true}) ->> false And Collect(t1, {Value:true}).Value;t1 +>> false And Last(Collect(t1, {Value:true})).Value;t1 Table({Value:false},{Value:true}) >> 0.10;t1 @@ -54,7 +54,7 @@ Table({Value:false},{Value:true}) >> 0.13;t1 Table({Value:false},{Value:true}) ->> And(false, Collect(t1, {Value:true}).Value);t1 +>> And(false, Last(Collect(t1, {Value:true})).Value);t1 Table({Value:false},{Value:true}) >> 0.14;t1 @@ -79,7 +79,7 @@ Table({Value:false},{Value:true}) Table({Value:false},{Value:true}) // OR with true, none of these should execute ->> true || Collect(t1, {Value:true}).Value +>> true || Last(Collect(t1, {Value:true})).Value true >> -1;t1 @@ -105,7 +105,7 @@ Table({Value:false},{Value:true}) Table({Value:false},{Value:true}) // OR with false, these should all execute ->> false || Collect(t1, {Value:true}).Value +>> false || Last(Collect(t1, {Value:true})).Value true >> 1;t1 @@ -129,7 +129,7 @@ Table({Value:false},{Value:true}) >> 3.1;t1 Table({Value:false},{Value:true}) ->> Or(false, Collect(t1, {Value:true}).Value) +>> Or(false, Last(Collect(t1, {Value:true})).Value) true >> 4;t1 diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuitDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuitDisabled.txt index f5e816ab0a..878415c0a3 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuitDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuitDisabled.txt @@ -5,13 +5,13 @@ >> Set( t1, [1,2] ) Table({Value:1},{Value:2}) ->> Coalesce(1,Collect(t1,{Value:3}).Value) +>> Coalesce(1,Last(Collect(t1,{Value:3})).Value) 1 >> 1;t1 Table({Value:1},{Value:2},{Value:3}) ->> Coalesce(1,Blank(),Collect(t1,{Value:3.1}).Value) +>> Coalesce(1,Blank(),Last(Collect(t1,{Value:3.1})).Value) 1 >> 1.1;t1 @@ -20,25 +20,25 @@ Table({Value:1},{Value:2},{Value:3},{Value:3.1}) >> Set( bn, If(1<0,1)) Blank() ->> Coalesce( bn, Collect(t1,{Value:3.2}).Value ) +>> Coalesce( bn, Last(Collect(t1,{Value:3.2})).Value ) 3.2 >> 2;t1 Table({Value:1},{Value:2},{Value:3},{Value:3.1},{Value:3.2}) ->> Coalesce( bn, Collect(t1,{Value:4}).Value, Collect(t1,{Value:5}).Value ) +>> Coalesce( bn, Last(Collect(t1,{Value:4})).Value, Last(Collect(t1,{Value:5})).Value ) 4 >> 3;t1 Table({Value:1},{Value:2},{Value:3},{Value:3.1},{Value:3.2},{Value:4},{Value:5}) ->> Coalesce( bn, Blank(), Collect(t1,{Value:5.1}).Value ) +>> Coalesce( bn, Blank(), Last(Collect(t1,{Value:5.1})).Value ) 5.1 >> 4;t1 Table({Value:1},{Value:2},{Value:3},{Value:3.1},{Value:3.2},{Value:4},{Value:5},{Value:5.1}) ->> Coalesce( bn, Blank(), Collect(t1,{Value:6}).Value, Collect(t1,{Value:7}).Value ) +>> Coalesce( bn, Blank(), Last(Collect(t1,{Value:6})).Value, Last(Collect(t1,{Value:7})).Value ) 6 >> 5;t1 diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation.txt index a418ffd8cc..3553409788 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation.txt @@ -1,4 +1,4 @@ - +#SETUP: PowerFxV1CompatibilityRules // NESTED TABLES - PATCH >> Set( aa, [[1,2,3], [4,5,6]] ) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt new file mode 100644 index 0000000000..030e73fc32 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt @@ -0,0 +1,412 @@ +#SETUP: disable:PowerFxV1CompatibilityRules +// NESTED TABLES - PATCH + +>> Set( aa, [[1,2,3], [4,5,6]] ) +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +>> Set( ab, First(aa) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +>> Patch( ab.Value, {Value:2}, {Value:9}) +{Value:9} + +>> ab +{Value:Table({Value:1},{Value:9},{Value:3})} + +>> Set( ac, Last(aa) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +>> Patch( ac.Value, {Value:5}, {Value:8} ) +{Value:8} + +>> ac +{Value:Table({Value:4},{Value:8},{Value:6})} + +>> Set( ad, Index(aa,1) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +>> Set( ae, Index(aa,2) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +>> Patch( ad.Value, {Value:1}, {Value:10}) +{Value:10} + +>> Patch( ae.Value, {Value:6}, {Value:16}) +{Value:16} + +>> ad +{Value:Table({Value:10},{Value:2},{Value:3})} + +>> ae +{Value:Table({Value:4},{Value:5},{Value:16})} + +>> ab // confirm no changes +{Value:Table({Value:1},{Value:9},{Value:3})} + +>> ac // confirm no changes +{Value:Table({Value:4},{Value:8},{Value:6})} + +>> aa // confirm no changes +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +// NESTED TABLES - COLLECT + +>> Set( ca, [[1,2,3], [4,5,6]] ) +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +>> Set( cb, First(ca) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +>> Collect( cb.Value, {Value:9}) +Table({Value:9}) + +>> cb +{Value:Table({Value:1},{Value:2},{Value:3},{Value:9})} + +>> Set( cc, Last(ca) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +>> Collect( cc.Value, {Value:8} ) +Table({Value:8}) + +>> cc +{Value:Table({Value:4},{Value:5},{Value:6},{Value:8})} + +>> Set( cd, Index(ca,1) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +>> Set( ce, Index(ca,2) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +>> Collect( cd.Value, {Value:10}) +Table({Value:10}) + +>> Collect( ce.Value, {Value:16}) +Table({Value:16}) + +>> cd +{Value:Table({Value:1},{Value:2},{Value:3},{Value:10})} + +>> ce +{Value:Table({Value:4},{Value:5},{Value:6},{Value:16})} + +>> cb // confirm no changes +{Value:Table({Value:1},{Value:2},{Value:3},{Value:9})} + +>> cc // confirm no changes +{Value:Table({Value:4},{Value:5},{Value:6},{Value:8})} + +>> ca // confirm no changes +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +// NESTED TABLES - CLEARCOLLECT + +>> Set( cca, [[1,2,3], [4,5,6]] ) +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +>> Set( ccb, First(cca) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +>> ClearCollect( ccb.Value, {Value:9}) +{Value:9} + +>> ccb +{Value:Table({Value:9})} + +>> Set( ccc, Last(ca) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +>> ClearCollect( ccc.Value, {Value:8} ) +{Value:8} + +>> ccc +{Value:Table({Value:8})} + +>> Set( ccd, Index(ca,1) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +>> Set( cce, Index(ca,2) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +>> ClearCollect( ccd.Value, {Value:10}) +{Value:10} + +>> ClearCollect( cce.Value, {Value:16}) +{Value:16} + +>> ccd +{Value:Table({Value:10})} + +>> cce +{Value:Table({Value:16})} + +>> ccb // confirm no changes +{Value:Table({Value:9})} + +>> ccc // confirm no changes +{Value:Table({Value:8})} + +>> cca // confirm no changes +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +// NESTED TABLES - CLEAR + +>> Set( cla, [[1,2,3], [4,5,6]] ) +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +>> Set( clb, First(cla) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +// ; true on the end so that we get the same result with and without V1Compat +>> Clear( clb.Value ); true +true + +>> clb +{Value:Table()} + +>> Set( clc, Last(cla) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +// ; true on the end so that we get the same result with and without V1Compat +>> Clear( clc.Value ); true +true + +>> clc +{Value:Table()} + +>> Collect( clb.Value, {Value:91}) +Table({Value:91}) + +>> Collect( clc.Value, {Value:81}) +Table({Value:81}) + +>> Set( cld, Index(ca,1) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +>> Set( cle, Index(ca,2) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +// ; true on the end so that we get the same result with and without V1Compat +>> Clear( cld.Value); true +true + +// ; true on the end so that we get the same result with and without V1Compat +>> Clear( cle.Value); true +true + +>> cld +{Value:Table()} + +>> cle +{Value:Table()} + +>> clb // confirm no changes +{Value:Table({Value:91})} + +>> clc // confirm no changes +{Value:Table({Value:81})} + +>> cla // confirm no changes +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +// NESTED TABLES - REMOVE + +>> Set( ra, [[1,2,3], [4,5,6]] ) +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +>> Set( rb, First(ra) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +// ; true on the end so that we get the same result with and without V1Compat +>> Remove( rb.Value, {Value:2} ); true +true + +>> rb +{Value:Table({Value:1},{Value:3})} + +>> Set( rc, Last(ra) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +// ; true on the end so that we get the same result with and without V1Compat +>> Remove( rc.Value, {Value:4} ); true +true + +>> rc +{Value:Table({Value:5},{Value:6})} + +>> Set( rd, Index(ra,1) ) +{Value:Table({Value:1},{Value:2},{Value:3})} + +>> Set( re, Index(ra,2) ) +{Value:Table({Value:4},{Value:5},{Value:6})} + +// ; true on the end so that we get the same result with and without V1Compat +>> Remove( rd.Value, {Value:3}); true +true + +// ; true on the end so that we get the same result with and without V1Compat +>> Remove( re.Value, {Value:4}); true +true + +>> rd +{Value:Table({Value:1},{Value:2})} + +>> re +{Value:Table({Value:5},{Value:6})} + +>> rb // confirm no changes +{Value:Table({Value:1},{Value:3})} + +>> rc // confirm no changes +{Value:Table({Value:5},{Value:6})} + +>> ra // confirm no changes +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +// DEEP NESTING IN RECORDS - PATCH + +>> Set( ba, {a: {b: {c: {d: [1,2,3]} } } } ) +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3})}}}} + +>> Set( bb, ba.a.b ) +{c:{d:Table({Value:1},{Value:2},{Value:3})}} + +>> Set( bc, ba.a.b.c ) +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Set( bd, bb.c ) +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Patch( ba.a.b.c.d, {Value:3}, {Value:9} ) +{Value:9} + +>> ba +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:9})}}}} + +>> bb +{c:{d:Table({Value:1},{Value:2},{Value:3})}} + +>> bc +{d:Table({Value:1},{Value:2},{Value:3})} + +>> bd +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Patch( bb.c.d, {Value:2}, {Value:8} ) +{Value:8} + +>> ba // after bb +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:9})}}}} + +>> bb // after bb +{c:{d:Table({Value:1},{Value:8},{Value:3})}} + +>> bc // after bb +{d:Table({Value:1},{Value:2},{Value:3})} + +>> bd // after bb +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Patch( bc.d, {Value:1}, {Value:7}) +{Value:7} + +>> ba // after bc +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:9})}}}} + +>> bb // after bc +{c:{d:Table({Value:1},{Value:8},{Value:3})}} + +>> bc // after bc +{d:Table({Value:7},{Value:2},{Value:3})} + +>> bd // after bc +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Patch( bd.d, { Value: 2}, {Value: 11 }) +{Value:11} + +>> ba // after bd +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:9})}}}} + +>> bb // after bd +{c:{d:Table({Value:1},{Value:8},{Value:3})}} + +>> bc // after bd +{d:Table({Value:7},{Value:2},{Value:3})} + +>> bd // after bd +{d:Table({Value:1},{Value:11},{Value:3})} + +// DEEP NESTING IN RECORDS - COLLECT + +>> Set( dba, {a: {b: {c: {d: [1,2,3]} } } } ) +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3})}}}} + +>> Set( dbb, dba.a.b ) +{c:{d:Table({Value:1},{Value:2},{Value:3})}} + +>> Set( dbc, dba.a.b.c ) +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Set( dbd, dbb.c ) +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Collect( dba.a.b.c.d, {Value:9} ) +Table({Value:9}) + +>> dba +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} + +>> dbb +{c:{d:Table({Value:1},{Value:2},{Value:3})}} + +>> dbc +{d:Table({Value:1},{Value:2},{Value:3})} + +>> dbd +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Collect( dbb.c.d, {Value:8} ) +Table({Value:8}) + +>> dba // after dbb +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} + +>> dbb // after dbb +{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:8})}} + +>> dbc // after dbb +{d:Table({Value:1},{Value:2},{Value:3})} + +>> dbd // after dbb +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Collect( dbc.d, {Value:7}) +Table({Value:7}) + +>> dba // after dbc +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} + +>> dbb // after dbc +{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:8})}} + +>> dbc // after dbc +{d:Table({Value:1},{Value:2},{Value:3},{Value:7})} + +>> dbd // after dbc +{d:Table({Value:1},{Value:2},{Value:3})} + +>> Collect( dbd.d, {Value: 11}) +Table({Value:11}) + +>> dba // after dbd +{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} + +>> dbb // after dbd +{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:8})}} + +>> dbc // after dbd +{d:Table({Value:1},{Value:2},{Value:3},{Value:7})} + +>> dbd // after dbd +{d:Table({Value:1},{Value:2},{Value:3},{Value:11})} diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate.txt index cf16cf1a31..e50f43b564 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate.txt @@ -1,4 +1,4 @@ - +#SETUP: PowerFxV1CompatibilityRules // Test mutations on a collection from ForAll() // These are all verified in Power Apps diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt new file mode 100644 index 0000000000..4ece1d60d2 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt @@ -0,0 +1,107 @@ +#SETUP: disable:PowerFxV1CompatibilityRules +// Test mutations on a collection from ForAll() + +// These are all verified in Power Apps +// + +>> Set(t, Sequence(4)) +Table({Value:1},{Value:2},{Value:3},{Value:4}) + +>> Collect(t, { Value : 99}); Concat(t, Value) +"123499" + +>> Set(t, ForAll(Sequence(4), Value*10)) +Table({Value:10},{Value:20},{Value:30},{Value:40}) + +>> Collect(t, { Value : 99}); Concat(t, Value) // After t was changed +"1020304099" + +// Initializing t2 +>> Set(t2, [1]) +Table({Value:1}) + +>> With({t: Sequence(5)}, Set(t2, ForAll(t, Value * 2))); t2 +Table({Value:2},{Value:4},{Value:6},{Value:8},{Value:10}) + +// Inner sequence is unchaged +>> With( { t : Sequence(5)}, + Collect(t2, { Value : 99 }); + Concat(t, Value) & "," & Concat(t2, Value) + ) +"12345,24681099" + +// short circuiting shouldn't execute any loops for a blank or empty table + +>> Set(t3,If(1<0,[1,2,3])) +Blank() + +>> ForAll(t3,Patch(t3,ThisRecord,{Value:4})) +Blank() + +>> 1;t3 +Blank() + +// interpretation of an empty table + +>> Set(t4,[1]) +Table({Value:1}) + +// ; true on the end so that we get the same result with and without V1Compat +>> Remove(t4,First(t4)); true +true + +>> 1;t4 +Table() + +// Should do nothing as t4 is empty +>> ForAll(t4,Patch(t4,ThisRecord,{Value:4})) +Table() + +>> 2;t4 +Table() + +>> Set(t5,[1]) +Table({Value:1}) + +// ForAll shouldn't execute the collect, even once +>> ForAll(t4,Collect(t5,{Value:9.1})) +Table() + +// With shouldn't execute the collect, even once +>> With(First(t4),Collect(t5,{Value:9.2})) +Blank() + +// interpretation of one record + +>> 1;t5 +Table({Value:1}) + +>> Collect(t4,{Value:1}) +Table({Value:1}) + +// ForAll should execute the collect, exactly once +>> ForAll(t4,Collect(t5,{Value:8.1})) +Table({Value:8.1}) + +// With shouldn execute the collect, exactly once +>> With(First(t4),Collect(t5,{Value:8.2})) +{Value:8.2} + +>> 2;t5 +Table({Value:1},{Value:8.1},{Value:8.2}) + +// interpretation of a blank table + +>> Set( t4, Blank() ) +Blank() + +// ForAll shouldn't execute the collect, even once +>> ForAll(t4,Collect(t5,{Value:7.1})) +Blank() + +// With shouldn't execute the collect, even once +>> With(First(t4),Collect(t5,{Value:7.2})) +Blank() + +>> 3;t5 +Table({Value:1},{Value:8.1},{Value:8.2}) From b639a804dab625a68848f021626eca3a5d4d00e2 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Thu, 22 Feb 2024 21:43:59 -0600 Subject: [PATCH 04/29] Fixing return value when PFx1 is disabled. --- .../Functions/LibraryMutation.cs | 21 +- .../ParsedExpression.cs | 12 +- .../ExpressionTestCases/Collect.txt | 17 +- .../ExpressionTestCases/Collect_V1Compat.txt | 15 + .../Collect_V1CompatDisabled.txt | 19 +- .../DeepMutation_V1CompatDisabled.txt | 16 +- .../ForAllMutate_V1CompatDisabled.txt | 4 +- .../MutationScripts/If.txt | 1 + .../MutationScripts/IfError.txt | 1 + .../IfError_V1CompatDisabled.txt | 47 +++ .../MutationScripts/If_V1CompatDisabled.txt | 47 +++ .../{Set.txt => Set_V1Compat.txt} | 1 + .../MutationScripts/Set_V1CompatDisabled.txt | 273 ++++++++++++++++++ .../MutationScripts/Simple1.txt | 11 - .../MutationScripts/Switch.txt | 1 + .../Switch_V1CompatDisabled.txt | 54 ++++ 16 files changed, 492 insertions(+), 48 deletions(-) create mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError_V1CompatDisabled.txt create mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/If_V1CompatDisabled.txt rename src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/{Set.txt => Set_V1Compat.txt} (99%) create mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1CompatDisabled.txt delete mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Simple1.txt create mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index 2f04e566e0..dcbf58846a 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -13,29 +14,30 @@ namespace Microsoft.PowerFx.Core.Texl.Builtins { - internal class CollectImpl : CollectFunction, IAsyncTexlFunction3 + internal class CollectImpl : CollectFunction, IAsyncTexlFunction5 { - public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) + public async Task InvokeAsync(IServiceProvider runtimeServiceProvider, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { - return await new CollectProcess().Process(irContext, args, cancellationToken).ConfigureAwait(false); + return await new CollectProcess().Process(runtimeServiceProvider, irContext, args, cancellationToken).ConfigureAwait(false); } } - internal class CollectScalarImpl : CollectScalarFunction, IAsyncTexlFunction3 + internal class CollectScalarImpl : CollectScalarFunction, IAsyncTexlFunction5 { - public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) + public async Task InvokeAsync(IServiceProvider runtimeServiceProvider, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { - return await new CollectProcess().Process(irContext, args, cancellationToken).ConfigureAwait(false); + return await new CollectProcess().Process(runtimeServiceProvider, irContext, args, cancellationToken).ConfigureAwait(false); } } internal class CollectProcess { - internal async Task Process(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) + internal async Task Process(IServiceProvider runtimeServiceProvider, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { FormulaValue arg0; var argc = args.Length; var returnIsTable = irContext._type.IsTable; + var features = runtimeServiceProvider.GetService(); // Need to check if the Lazy first argument has been evaluated since it may have already been // evaluated in the ClearCollect case. @@ -115,6 +117,11 @@ internal async Task Process(FormulaType irContext, FormulaValue[] } } + if (!features.PowerFxV1CompatibilityRules) + { + return tableValue; + } + if (resultRows.Count == 0) { return FormulaValue.NewBlank(arg0.Type); diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs b/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs index 8236979405..5409eb8379 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs @@ -86,7 +86,8 @@ internal static IExpressionEvaluator GetEvaluator(this CheckResult result, Stack _globals = globals, _allSymbols = result.Symbols, _parameterSymbolTable = result.Parameters, - _additionalFunctions = result.Engine.Config.AdditionalFunctions + _additionalFunctions = result.Engine.Config.AdditionalFunctions, + _features = result.Engine.Config.Features }; return expr; @@ -98,12 +99,13 @@ internal class ParsedExpression : IExpressionEvaluator internal IntermediateNode _irnode; private readonly ScopeSymbol _topScopeSymbol; private readonly CultureInfo _cultureInfo; - private readonly StackDepthCounter _stackMarker; + private readonly StackDepthCounter _stackMarker; internal ReadOnlySymbolValues _globals; internal ReadOnlySymbolTable _allSymbols; internal ReadOnlySymbolTable _parameterSymbolTable; internal IReadOnlyDictionary _additionalFunctions; + internal Features _features; internal ParsedExpression(IntermediateNode irnode, ScopeSymbol topScope, StackDepthCounter stackMarker, CultureInfo cultureInfo = null) { @@ -133,6 +135,12 @@ public async Task EvalAsync(CancellationToken cancellationToken, I hasInnerServices = true; } + if (_features != null) + { + innerServices.AddService(_features); + hasInnerServices = true; + } + RuntimeConfig runtimeConfig2 = new RuntimeConfig { Values = symbolValues, diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt index 8fa72a0608..1292dc2f0a 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt @@ -13,9 +13,6 @@ Blank() >> Collect(t1, If(1>0, r2));CountRows(t1) 2 - ->> IsBlank(Collect(t1, Blank())) -true >> IsError(Collect(t1, If(1/0, r2))) true @@ -23,12 +20,6 @@ true >> Collect(t1) Errors: Error 0-11: Invalid number of arguments: received 1, expected 2 or more. ->> Collect(t1, r2, r2) -Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) - ->> Collect(t1, r2, 1; 2; r2) -Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) - >> Collect(t1, "x") Errors: Error 0-16: The function 'Collect' has some invalid arguments.|Error 12-15: Invalid argument type. Cannot use Text values in this context. @@ -45,10 +36,4 @@ Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-15: Name Errors: Error 12-15: Name isn't valid. 'Foo' isn't recognized. >> Collect(t1,{Price:200}).Price -Errors: Error 11-22: The specified column 'Price' does not exist.|Error 0-23: The function 'Collect' has some invalid arguments.|Error 23-29: Name isn't valid. 'Price' isn't recognized. - ->> Collect(t1, Table(r2,r2)) -Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) - ->> Collect(t1, Table(r2,r2), {Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) -Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) \ No newline at end of file +Errors: Error 11-22: The specified column 'Price' does not exist.|Error 0-23: The function 'Collect' has some invalid arguments.|Error 23-29: Name isn't valid. 'Price' isn't recognized. \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt index 22aa02418f..0bfe631858 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt @@ -29,3 +29,18 @@ Errors: Error 8-15: The value passed to the 'Collect' function cannot be changed >> Collect("", "") Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed. + +>> IsBlank(Collect(t1, Blank())) +true + +>> Collect(t1, r2, r2) +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Collect(t1, r2, 1; 2; r2) +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Collect(t1, Table(r2,r2)) +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Collect(t1, Table(r2,r2), {Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt index 098011a9db..8ffb9284ff 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt @@ -4,7 +4,7 @@ Table({name:"textInput1"}) >> Collect(t1, {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}) -Table({Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}) +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}) >> Last(Collect(t_empty,{Value:200})).Value 200 @@ -25,4 +25,19 @@ Errors: Error 17-19: The specified column 'Field1' does not exist.|Error 0-20: T Blank() >> Collect("", "") -Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context. \ No newline at end of file +Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context. + +>> IsBlank(Collect(t1, Blank())) +false + +>> Collect(t1, r2, r2) +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Collect(t1, r2, 1; 2; r2) +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Collect(t1, Table(r2,r2)) +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Collect(t1, Table(r2,r2), {Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt index 030e73fc32..b0b96e5c84 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt @@ -58,7 +58,7 @@ Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value {Value:Table({Value:1},{Value:2},{Value:3})} >> Collect( cb.Value, {Value:9}) -Table({Value:9}) +Table({Value:1},{Value:2},{Value:3},{Value:9}) >> cb {Value:Table({Value:1},{Value:2},{Value:3},{Value:9})} @@ -67,7 +67,7 @@ Table({Value:9}) {Value:Table({Value:4},{Value:5},{Value:6})} >> Collect( cc.Value, {Value:8} ) -Table({Value:8}) +Table({Value:4},{Value:5},{Value:6},{Value:8}) >> cc {Value:Table({Value:4},{Value:5},{Value:6},{Value:8})} @@ -79,10 +79,10 @@ Table({Value:8}) {Value:Table({Value:4},{Value:5},{Value:6})} >> Collect( cd.Value, {Value:10}) -Table({Value:10}) +Table({Value:1},{Value:2},{Value:3},{Value:10}) >> Collect( ce.Value, {Value:16}) -Table({Value:16}) +Table({Value:4},{Value:5},{Value:6},{Value:16}) >> cd {Value:Table({Value:1},{Value:2},{Value:3},{Value:10})} @@ -352,7 +352,7 @@ Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value {d:Table({Value:1},{Value:2},{Value:3})} >> Collect( dba.a.b.c.d, {Value:9} ) -Table({Value:9}) +Table({Value:1},{Value:2},{Value:3},{Value:9}) >> dba {a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} @@ -367,7 +367,7 @@ Table({Value:9}) {d:Table({Value:1},{Value:2},{Value:3})} >> Collect( dbb.c.d, {Value:8} ) -Table({Value:8}) +Table({Value:1},{Value:2},{Value:3},{Value:8}) >> dba // after dbb {a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} @@ -382,7 +382,7 @@ Table({Value:8}) {d:Table({Value:1},{Value:2},{Value:3})} >> Collect( dbc.d, {Value:7}) -Table({Value:7}) +Table({Value:1},{Value:2},{Value:3},{Value:7}) >> dba // after dbc {a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} @@ -397,7 +397,7 @@ Table({Value:7}) {d:Table({Value:1},{Value:2},{Value:3})} >> Collect( dbd.d, {Value: 11}) -Table({Value:11}) +Table({Value:1},{Value:2},{Value:3},{Value:11}) >> dba // after dbd {a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt index 4ece1d60d2..ddcc241ba7 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt @@ -81,11 +81,11 @@ Table({Value:1}) // ForAll should execute the collect, exactly once >> ForAll(t4,Collect(t5,{Value:8.1})) -Table({Value:8.1}) +Table({Value:Table({Value:1},{Value:8.1})}) // With shouldn execute the collect, exactly once >> With(First(t4),Collect(t5,{Value:8.2})) -{Value:8.2} +Table({Value:1},{Value:8.1},{Value:8.2}) >> 2;t5 Table({Value:1},{Value:8.1},{Value:8.2}) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/If.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/If.txt index 43a2233393..32da99cce1 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/If.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/If.txt @@ -1,3 +1,4 @@ +#SETUP: PowerFxV1CompatibilityRules // Cases to test how shortcut verification work along with behavior functions >> Set( t1, [1,2] ) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError.txt index 5e35612e06..98bbd8ee80 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError.txt @@ -1,3 +1,4 @@ +#SETUP: PowerFxV1CompatibilityRules // Cases to test how shortcut verification work along with behavior functions >> Set( t1, [1,2] ) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError_V1CompatDisabled.txt new file mode 100644 index 0000000000..d36193ffd2 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError_V1CompatDisabled.txt @@ -0,0 +1,47 @@ +#SETUP: disable:PowerFxV1CompatibilityRules +// Cases to test how shortcut verification work along with behavior functions + +>> Set( t1, [1,2] ) +Table({Value:1},{Value:2}) + +>> IfError(1,Last(Collect(t1,{Value:3})).Value) +1 + +>> 1;t1 +Table({Value:1},{Value:2}) + +>> IfError(1/0,Last(Collect(t1,{Value:3})).Value) +3 + +>> 2;t1 +Table({Value:1},{Value:2},{Value:3}) + +>> IfError(1/0,Last(Collect(t1,{Value:4})).Value,1/0,Last(Collect(t1,{Value:5})).Value) +4 + +>> 3;t1 +Table({Value:1},{Value:2},{Value:3},{Value:4}) + +>> IfError(1/0,Last(Collect(t1,{Value:5})).Value,1/0,Last(Collect(t1,{Value:6})).Value,Last(Collect(t1,{Value:7})).Value) +5 + +>> 4;t1 +Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5}) + +>> IfError(1,Last(Collect(t1,{Value:5})).Value,1/0,Last(Collect(t1,{Value:6})).Value,Last(Collect(t1,{Value:7})).Value) +6 + +>> 5;t1 +Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6}) + +>> IfError(1,Last(Collect(t1,{Value:5})).Value,2,Last(Collect(t1,{Value:6})).Value,Last(Collect(t1,{Value:7})).Value) +7 + +>> 6;t1 +Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7}) + +>> IfError(1,Last(Collect(t1,{Value:5})).Value,2,Last(Collect(t1,{Value:6})).Value) +2 + +>> 7;t1 +Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7}) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/If_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/If_V1CompatDisabled.txt new file mode 100644 index 0000000000..8f56361e47 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/If_V1CompatDisabled.txt @@ -0,0 +1,47 @@ +#SETUP: disable:PowerFxV1CompatibilityRules +// Cases to test how shortcut verification work along with behavior functions + +>> Set( t1, [1,2] ) +Table({Value:1},{Value:2}) + +>> If(false,Last(Collect(t1,{Value:3})).Value,2) +2 + +>> 1;t1 +Table({Value:1},{Value:2}) + +>> If(true,3,Last(Collect(t1,{Value:3})).Value) +3 + +>> 2;t1 +Table({Value:1},{Value:2}) + +>> If(false,Last(Collect(t1,{Value:3})).Value,false,Last(Collect(t1,{Value:4})).Value,2) +2 + +>> 3;t1 +Table({Value:1},{Value:2}) + +>> If(false,Last(Collect(t1,{Value:3})).Value,true,2,Last(Collect(t1,{Value:4})).Value,2) +2 + +>> 3.1;t1 +Table({Value:1},{Value:2}) + +>> If(true,3,true,Last(Collect(t1,{Value:3})).Value,Last(Collect(t1,{Value:4})).Value) +3 + +>> 4;t1 +Table({Value:1},{Value:2}) + +>> If(false,Last(Collect(t1,{Value:3})).Value,false,Last(Collect(t1,{Value:4})).Value,5) +5 + +>> 5;t1 +Table({Value:1},{Value:2}) + +>> If(true,3,true,6,true,Last(Collect(t1,{Value:3})).Value,Last(Collect(t1,{Value:4})).Value) +3 + +>> 6;t1 +Table({Value:1},{Value:2}) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1Compat.txt similarity index 99% rename from src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set.txt rename to src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1Compat.txt index 0ce388d6f7..47db2fcaaa 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1Compat.txt @@ -1,3 +1,4 @@ +#SETUP: PowerFxV1CompatibilityRules // Tests for Set/Clear/Patch/Remove with Copy-on-Write semantics diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1CompatDisabled.txt new file mode 100644 index 0000000000..73049e93f6 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1CompatDisabled.txt @@ -0,0 +1,273 @@ +#SETUP: disable:PowerFxV1CompatibilityRules + +// Tests for Set/Clear/Patch/Remove with Copy-on-Write semantics + +// TABLE MUTATION + +>> Set( a, [1,2,3] ) +Table({Value:1},{Value:2},{Value:3}) + +>> Set( b, a ) +Table({Value:1},{Value:2},{Value:3}) + +>> Patch( a, {Value:2}, {Value:9} ) +{Value:9} + +>> a +Table({Value:1},{Value:9},{Value:3}) + +>> b +Table({Value:1},{Value:2},{Value:3}) + +// TABLE APPEND + +>> Set( aa, [1,2,3] ) +Table({Value:1},{Value:2},{Value:3}) + +>> Set( ab, aa ) +Table({Value:1},{Value:2},{Value:3}) + +>> Collect( ab, {Value:4} ) +Table({Value:1},{Value:2},{Value:3},{Value:4}) + +>> aa +Table({Value:1},{Value:2},{Value:3}) + +>> ab +Table({Value:1},{Value:2},{Value:3},{Value:4}) + +>> Collect( aa, {Value:5} ) +Table({Value:1},{Value:2},{Value:3},{Value:5}) + +>> aa // again +Table({Value:1},{Value:2},{Value:3},{Value:5}) + +>> ab // again +Table({Value:1},{Value:2},{Value:3},{Value:4}) + +// TABLE WITHIN RECORD + +>> Set( ba, [ [1,2,3], [4,5,6] ]) +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +>> Set( bb, Last(ba).Value ) +Table({Value:4},{Value:5},{Value:6}) + +>> Patch( bb, {Value:4}, {Value:8} ) +{Value:8} + +>> ba +Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) + +>> bb +Table({Value:8},{Value:5},{Value:6}) + +// TABLE REMOVE + +>> Set( ca, [1,2,3] ) +Table({Value:1},{Value:2},{Value:3}) + +>> Set( cb, ca ) +Table({Value:1},{Value:2},{Value:3}) + +// ; true on the end so that we get the same result with and without V1Compat +>> Remove( cb, {Value:2} ); true +true + +>> ca // again +Table({Value:1},{Value:2},{Value:3}) + +>> cb // again +Table({Value:1},{Value:3}) + +// ; true on the end so that we get the same result with and without V1Compat +>> Remove( ca, {Value:1} ); true +true + +>> ca // again 2 +Table({Value:2},{Value:3}) + +>> cb // again 2 +Table({Value:1},{Value:3}) + +// TABLE CLEAR + +>> Set( da, [1,2,3] ) +Table({Value:1},{Value:2},{Value:3}) + +>> Set( db, da ) +Table({Value:1},{Value:2},{Value:3}) + +>> da +Table({Value:1},{Value:2},{Value:3}) + +>> db +Table({Value:1},{Value:2},{Value:3}) + +// ; true on the end so that we get the same result with and without V1Compat +>> Clear( da ); true +true + +>> da // again +Table() + +>> db // again +Table({Value:1},{Value:2},{Value:3}) + +// LOTS OF COPIES + +>> Set( ea, [1,2,3] ) +Table({Value:1},{Value:2},{Value:3}) + +>> Set( eb, ea ) +Table({Value:1},{Value:2},{Value:3}) + +>> Set( ec, eb ) +Table({Value:1},{Value:2},{Value:3}) + +>> Set( ed, ec ) +Table({Value:1},{Value:2},{Value:3}) + +>> Patch( ec, {Value:3}, {Value:7} ) +{Value:7} + +>> ea +Table({Value:1},{Value:2},{Value:3}) + +>> eb +Table({Value:1},{Value:2},{Value:3}) + +>> ec +Table({Value:1},{Value:2},{Value:7}) + +>> ed +Table({Value:1},{Value:2},{Value:3}) + +>> Collect( eb, {Value:9} ) +Table({Value:1},{Value:2},{Value:3},{Value:9}) + +>> ea // again +Table({Value:1},{Value:2},{Value:3}) + +>> eb // again +Table({Value:1},{Value:2},{Value:3},{Value:9}) + +>> ec // again +Table({Value:1},{Value:2},{Value:7}) + +>> ed // again +Table({Value:1},{Value:2},{Value:3}) + +// ; true on the end so that we get the same result with and without V1Compat +>> Clear( ea ); true +true + +>> ea // again 2 +Table() + +>> eb // again 2 +Table({Value:1},{Value:2},{Value:3},{Value:9}) + +>> ec // again 2 +Table({Value:1},{Value:2},{Value:7}) + +>> ed // again 2 +Table({Value:1},{Value:2},{Value:3}) + +// MORE NESTED TABLES + +>> Set( fa, Table( { a:[1,2,3], b:[4,5,6], c:"hi", d:true}, { a:[10,20,30], b:[40,50,60], c:"bye", d:false} ) ) +Table({a:Table({Value:1},{Value:2},{Value:3}),b:Table({Value:4},{Value:5},{Value:6}),c:"hi",d:true},{a:Table({Value:10},{Value:20},{Value:30}),b:Table({Value:40},{Value:50},{Value:60}),c:"bye",d:false}) + +>> Set( fb, fa ) +Table({a:Table({Value:1},{Value:2},{Value:3}),b:Table({Value:4},{Value:5},{Value:6}),c:"hi",d:true},{a:Table({Value:10},{Value:20},{Value:30}),b:Table({Value:40},{Value:50},{Value:60}),c:"bye",d:false}) + +>> Set( fc, Index(fa, 2).b ) +Table({Value:40},{Value:50},{Value:60}) + +>> Patch( fc, {Value:50}, {Value:1000} ) +{Value:1000} + +>> fa +Table({a:Table({Value:1},{Value:2},{Value:3}),b:Table({Value:4},{Value:5},{Value:6}),c:"hi",d:true},{a:Table({Value:10},{Value:20},{Value:30}),b:Table({Value:40},{Value:50},{Value:60}),c:"bye",d:false}) + +>> fb +Table({a:Table({Value:1},{Value:2},{Value:3}),b:Table({Value:4},{Value:5},{Value:6}),c:"hi",d:true},{a:Table({Value:10},{Value:20},{Value:30}),b:Table({Value:40},{Value:50},{Value:60}),c:"bye",d:false}) + +>> fc +Table({Value:40},{Value:1000},{Value:60}) + +// NESTED TABLES DEEPLY IN RECORDS + +>> Set( ga, { a: { b: { c: [1,2,3] }}}) +{a:{b:{c:Table({Value:1},{Value:2},{Value:3})}}} + +>> Set( gb, ga.a.b.c ) +Table({Value:1},{Value:2},{Value:3}) + +>> Patch( gb, First(gb), {Value:9} ) +{Value:9} + +>> ga +{a:{b:{c:Table({Value:1},{Value:2},{Value:3})}}} + +>> gb +Table({Value:9},{Value:2},{Value:3}) + +>> Set( gc, ga.a ) +{b:{c:Table({Value:1},{Value:2},{Value:3})}} + +>> Set( gd, gc.b ) +{c:Table({Value:1},{Value:2},{Value:3})} + +>> Set( ge, gd.c ) +Table({Value:1},{Value:2},{Value:3}) + +>> Collect( ge, {Value:88} ) +Table({Value:1},{Value:2},{Value:3},{Value:88}) + +>> ga // again +{a:{b:{c:Table({Value:1},{Value:2},{Value:3})}}} + +>> gb // again +Table({Value:9},{Value:2},{Value:3}) + +>> gc // again +{b:{c:Table({Value:1},{Value:2},{Value:3})}} + +>> gd // again +{c:Table({Value:1},{Value:2},{Value:3})} + +>> ge // again +Table({Value:1},{Value:2},{Value:3},{Value:88}) + +// SET RECORD FROM ANOTHER TABLE + +>> Set( ha, [1,2,3] ) +Table({Value:1},{Value:2},{Value:3}) + +>> Set( hb, Table(Index(ha,2),Last(ha),First(ha))) +Table({Value:2},{Value:3},{Value:1}) + +>> ha +Table({Value:1},{Value:2},{Value:3}) + +>> hb +Table({Value:2},{Value:3},{Value:1}) + +>> Patch( ha, First(ha), {Value:5}) +{Value:5} + +>> Patch( ha, Index(ha,2), {Value:6}) +{Value:6} + +>> Patch( ha, Last(ha), {Value:7}) +{Value:7} + +>> ha // again +Table({Value:5},{Value:6},{Value:7}) + +>> hb // again +Table({Value:2},{Value:3},{Value:1}) + + diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Simple1.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Simple1.txt deleted file mode 100644 index f17c0a7ab9..0000000000 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Simple1.txt +++ /dev/null @@ -1,11 +0,0 @@ ->> Set(x, [1,2,3]) -Table({Value:1},{Value:2},{Value:3}) - -// collect returns the record that was added ->> Collect(x, {Value : 4 }) -{Value:4} - -// Original was mutated ->> x -Table({Value:1},{Value:2},{Value:3},{Value:4}) - diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch.txt index 81d0a8614c..5f38805759 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch.txt @@ -1,3 +1,4 @@ +#SETUP: PowerFxV1CompatibilityRules // Cases to test how shortcut verification work along with behavior functions >> Set( t1, [0,1] ) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt new file mode 100644 index 0000000000..1b0faf947e --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt @@ -0,0 +1,54 @@ +#SETUP: disable:PowerFxV1CompatibilityRules + +// Cases to test how shortcut verification work along with behavior functions + +>> Set( t1, [0,1] ) +Table({Value:0},{Value:1}) + +>> Switch(2,1,Last(Collect(t1,{Value:1})).Value,2,Last(Collect(t1,{Value:2})).Value,3,Last(Collect(t1,{Value:3})).Value) +2 + +>> 1;t1 +Table({Value:0},{Value:1},{Value:2}) + +>> Switch(3, + 1,Last(Collect(t1,{Value:1}).Value), + 2,Last(Collect(t1,{Value:2}).Value), + 3,Last(Collect(t1,{Value:3}).Value), + Last(Collect(t1,{Value:4})).Value) +3 + +>> 2;t1 +Table({Value:0},{Value:1},{Value:2},{Value:3}) + +>> Switch(1, + 1,Last(Collect(t1,{Value:4})).Value, + 2,Last(Collect(t1,{Value:6})).Value, + 3,Last(Collect(t1,{Value:7})).Value, + Last(Collect(t1,{Value:8})).Value) +4 + +>> 3;t1 +Table({Value:0},{Value:1},{Value:2},{Value:3},{Value:4}) + +>> Switch(6, + Last(Collect(t1,{Value:5})).Value,1, + Last(Collect(t1,{Value:6})).Value,12, + 2,Last(Collect(t1,{Value:7})).Value, + Last(Collect(t1,{Value:8})).Value) +12 + +>> 4;t1 +Table({Value:0},{Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6}) + +>> Switch(9, + Last(Collect(t1,{Value:7})).Value,Last(Collect(t1,{Value:20})).Value, + Last(Collect(t1,{Value:8})).Value,2, + 6,8, + Last(Collect(t1,{Value:9})).Value,Last(Collect(t1,{Value:10})).Value, + Last(Collect(t1,{Value:30})).Value) +10 + +>> 5;t1 +Table({Value:0},{Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7},{Value:8},{Value:9},{Value:10}) + From c0f35b8d83bc9e0c026c5956b64e4d32dbdbb65b Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Thu, 22 Feb 2024 22:30:40 -0600 Subject: [PATCH 05/29] Fix. --- .../MutationScripts/Switch_V1CompatDisabled.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt index 1b0faf947e..509b114ec4 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt @@ -12,9 +12,9 @@ Table({Value:0},{Value:1}) Table({Value:0},{Value:1},{Value:2}) >> Switch(3, - 1,Last(Collect(t1,{Value:1}).Value), - 2,Last(Collect(t1,{Value:2}).Value), - 3,Last(Collect(t1,{Value:3}).Value), + 1,Last(Collect(t1,{Value:1})).Value, + 2,Last(Collect(t1,{Value:2})).Value, + 3,Last(Collect(t1,{Value:3})).Value, Last(Collect(t1,{Value:4})).Value) 3 From 0b9465329332d4f84233b464f493e50b734211fc Mon Sep 17 00:00:00 2001 From: "Anderson Ferreira da Silva (from Dev Box)" Date: Fri, 1 Mar 2024 00:23:58 -0600 Subject: [PATCH 06/29] Saving progress. IR translator, removing unnacessary PFxV1Disabled test cases. --- .../Microsoft.PowerFx.Core/IR/CoercionKind.cs | 3 +- .../IR/CoercionMatrix.cs | 6 + .../Microsoft.PowerFx.Core/IR/IRTranslator.cs | 3 + .../IR/Nodes/UnaryOpKind.cs | 3 +- .../Texl/Builtins/Collect.cs | 200 ++++++--- .../Microsoft.PowerFx.Core/Types/DType.cs | 2 +- .../Utils/MutationUtils.cs | 15 + .../Functions/LibraryMutation.cs | 40 +- .../Functions/LibraryUnary.cs | 21 +- .../ExpressionTestCases/Clear.txt | 6 - .../ExpressionTestCases/Clear_V1Compat.txt | 6 + .../ExpressionTestCases/Collect.txt | 39 -- .../ExpressionTestCases/Collect_V1Compat.txt | 30 ++ .../Collect_V1CompatDisabled.txt | 43 -- .../ExpressionTestCases/DecimalBoot.txt | 3 - .../ExpressionTestCases/Patch.txt | 37 -- .../ExpressionTestCases/Patch_V1Compat.txt | 37 ++ .../Patch_V1CompatDisabled.txt | 12 - .../ExpressionTestCases/Remove.txt | 2 +- .../MutationFunctionsTests.cs | 26 ++ .../AndOr_V1CompatDisabled.txt | 18 +- .../Coalesce_CoalesceShortCircuitDisabled.txt | 38 +- .../MutationScripts/Collect.txt | 2 +- .../DeepMutation_V1CompatDisabled.txt | 412 ------------------ .../ForAllMutate_V1CompatDisabled.txt | 107 ----- .../IfError_V1CompatDisabled.txt | 47 -- .../MutationScripts/Set_V1CompatDisabled.txt | 273 ------------ .../Switch_V1CompatDisabled.txt | 54 --- 28 files changed, 330 insertions(+), 1155 deletions(-) delete mode 100644 src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt delete mode 100644 src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt delete mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt delete mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt delete mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError_V1CompatDisabled.txt delete mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1CompatDisabled.txt delete mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/CoercionKind.cs b/src/libraries/Microsoft.PowerFx.Core/IR/CoercionKind.cs index d352851155..0de57192c7 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/CoercionKind.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/CoercionKind.cs @@ -107,6 +107,7 @@ internal enum CoercionKind CurrencyToText, TextToCurrency, CurrencyToBoolean, - BooleanToCurrency, + BooleanToCurrency, + PrimitiveToSingleColumnRecord, } } diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/CoercionMatrix.cs b/src/libraries/Microsoft.PowerFx.Core/IR/CoercionMatrix.cs index e521971068..c75c9b355c 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/CoercionMatrix.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/CoercionMatrix.cs @@ -22,6 +22,12 @@ public static CoercionKind GetCoercionKind(DType fromType, DType toType, bool us if (fromType.IsAggregate && toType.Kind == DKind.DataEntity) { return CoercionKind.AggregateToDataEntity; + } + + // Coercion from a primitive type to a single column record type. + if (fromType.IsPrimitive && toType.IsRecord) + { + return CoercionKind.PrimitiveToSingleColumnRecord; } if (toType.IsLargeImage && (fromType.Kind == DKind.Image || fromType == DType.MinimalLargeImage)) diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs b/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs index 7b8afea7be..f73a53644c 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs @@ -1179,6 +1179,9 @@ private IntermediateNode InjectCoercion(IntermediateNode child, IRTranslatorCont break; case CoercionKind.PenImageToText: unaryOpKind = UnaryOpKind.PenImageToText; + break; + case CoercionKind.PrimitiveToSingleColumnRecord: + unaryOpKind = UnaryOpKind.PrimitiveToSingleColumnRecord; break; case CoercionKind.UntypedToText: return new CallNode(IRContext.NotInSource(FormulaType.Build(toType)), BuiltinFunctionsCore.Text_UO, child); diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/Nodes/UnaryOpKind.cs b/src/libraries/Microsoft.PowerFx.Core/IR/Nodes/UnaryOpKind.cs index 4fa2837382..dfa45df0ef 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/Nodes/UnaryOpKind.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/Nodes/UnaryOpKind.cs @@ -98,7 +98,8 @@ internal enum UnaryOpKind DateToDateTime, BooleanToOptionSet, - AggregateToDataEntity, + AggregateToDataEntity, + PrimitiveToSingleColumnRecord, // Argument pre-processesor in IR Phase. diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 8c9ffc786c..171dbdc772 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -91,15 +91,74 @@ protected CollectFunction(string name, TexlStrings.StringGetter description) return base.GetSignatures(arity); } - public virtual DType GetCollectedType(PowerFx.Features features, DType argType) + public virtual DType GetCollectedType(Features features, DType argType, TexlNode arg, ref Dictionary nodeToCoercedTypeMap) { Contracts.Assert(argType.IsValid); return argType; + } + + public bool TryGetUnifiedCollectedTypeCanvas(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType, ref Dictionary nodeToCoercedTypeMap) + { + Contracts.AssertValue(args); + Contracts.AssertAllValues(args); + Contracts.AssertValue(argTypes); + Contracts.Assert(args.Length == argTypes.Length); + Contracts.AssertValue(errors); + Contracts.Assert(MinArity <= args.Length && args.Length <= MaxArity); + + bool fValid = true; + DType itemType = DType.Invalid; + + var argc = args.Length; + + for (int i = 1; i < argc; i++) + { + DType argType = GetCollectedType(features, argTypes[i], args[i], ref nodeToCoercedTypeMap); + + // The subsequent args should all be aggregates. + if (!argType.IsAggregate) + { + errors.EnsureError(args[i], TexlStrings.ErrBadType_Type, argType.GetKindString()); + fValid = false; + continue; + } + + // Promote the arg type to a table to facilitate unioning. + if (!argType.IsTable) + { + argType = argType.ToTable(); + } + + if (!itemType.IsValid) + { + itemType = argType; + } + else + { + bool fUnionError = false; + itemType = DType.Union(ref fUnionError, itemType, argType, useLegacyDateTimeAccepts: true, features); + if (fUnionError) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrIncompatibleTypes); + fValid = false; + } + } + + // We only support accessing entities in collections if the collection has only 1 argument that contributes to it's type + if (argc != 2 && itemType.ContainsDataEntityType(DPath.Root)) + { + fValid &= DropAllOfKindNested(ref itemType, errors, args[i], DKind.DataEntity); + } + } + + Contracts.Assert(!itemType.IsValid || itemType.IsTable); + collectedType = itemType.IsValid ? itemType.ToTable() : DType.EmptyTable; + return fValid; } // Attempt to get the unified schema of the items being collected by an invocation. - private bool TryGetUnifiedCollectedType(CheckTypesContext context, TexlNode[] args, DType[] argTypes, IErrorContainer errors, out DType collectedType) + private bool TryGetUnifiedCollectedTypeV1(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType, ref Dictionary nodeToCoercedTypeMap) { Contracts.AssertValue(args); Contracts.AssertAllValues(args); @@ -117,7 +176,13 @@ private bool TryGetUnifiedCollectedType(CheckTypesContext context, TexlNode[] ar for (var i = 1; i < argc; i++) { - DType argType = GetCollectedType(context.Features, argTypes[i]); + DType argType = GetCollectedType(features, argTypes[i], args[i], ref nodeToCoercedTypeMap); + + // !!! How is it possible for an argtype to be a primitive and an aggregate at the same time? + //if (argType.DisplayNameProvider == null && argType.Kind == DKind.ObjNull) + //{ + // argType.DisplayNameProvider = datasourceType.DisplayNameProvider; + //} // The subsequent args should all be aggregates. if (!argType.IsAggregate) @@ -128,13 +193,13 @@ private bool TryGetUnifiedCollectedType(CheckTypesContext context, TexlNode[] ar } // Promote the arg type to a table to facilitate unioning. - if (!argType.IsRecord) + if (!argType.IsTable) { - argType = argType.ToRecord(); + argType = argType.ToTable(); } // Checks if all record names exist against table type and if its possible to coerce. - bool checkAggregateNames = argType.CheckAggregateNames(datasourceType, args[i], errors, context.Features, SupportsParamCoercion); + bool checkAggregateNames = argType.CheckAggregateNames(datasourceType, args[i], errors, features, SupportsParamCoercion); fValid = fValid && checkAggregateNames; if (!itemType.IsValid) @@ -144,7 +209,7 @@ private bool TryGetUnifiedCollectedType(CheckTypesContext context, TexlNode[] ar else { var fUnionError = false; - itemType = DType.Union(ref fUnionError, itemType, argType, useLegacyDateTimeAccepts: true, context.Features); + itemType = DType.Union(ref fUnionError, itemType, argType, useLegacyDateTimeAccepts: true, features); if (fUnionError) { errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrIncompatibleTypes); @@ -159,8 +224,8 @@ private bool TryGetUnifiedCollectedType(CheckTypesContext context, TexlNode[] ar } } - Contracts.Assert(!itemType.IsValid || itemType.IsRecord); - collectedType = itemType.IsValid ? itemType : DType.EmptyRecord; + Contracts.Assert(!itemType.IsValid || itemType.IsTable); + collectedType = itemType.IsValid ? itemType : DType.EmptyTable; return fValid; } @@ -182,51 +247,44 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp { errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrInvalidArgs_Func, Name); fValid = false; - } + } + + DType collectedType = null; // Get the unified collected type on the RHS. This will generate appropriate - // document errors for invalid arguments such as unsupported aggregate types. - fValid &= TryGetUnifiedCollectedType(context, args, argTypes, errors, out DType collectedType); - Contracts.Assert(collectedType.IsRecord); - - if (fValid) - { - if (!collectedType.TryGetCoercionSubType(collectionType, out DType coercionType, out var coercionNeeded, context.Features)) - { - fValid = false; - } - else - { - if (coercionNeeded) - { - CollectionUtils.Add(ref nodeToCoercedTypeMap, args[1], coercionType); - } - - var fError = false; - - returnType = DType.Union(ref fError, collectionType.ToRecord(), collectedType, useLegacyDateTimeAccepts: false, context.Features, allowCoerce: true); - - if (fError) - { - fValid = false; - if (!SetErrorForMismatchedColumns(collectionType, collectedType, args[1], errors, context.Features)) - { - errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrTableDoesNotAcceptThisType); - } - } - } - } - - if (context.Features.PowerFxV1CompatibilityRules && argTypes.Length == 2) - { - if (argTypes[1].IsTable && argTypes[1].Kind != DKind.ObjNull) - { - returnType = returnType.ToTable(); - } - } - else - { - returnType = returnType.ToTable(); + // document errors for invalid arguments such as unsupported aggregate types. + if (context.Features.PowerFxV1CompatibilityRules) + { + fValid &= TryGetUnifiedCollectedTypeV1(args, argTypes, errors, context.Features, out collectedType, ref nodeToCoercedTypeMap); + } + else + { + fValid &= TryGetUnifiedCollectedTypeCanvas(args, argTypes, errors, context.Features, out collectedType, ref nodeToCoercedTypeMap); + } + + Contracts.Assert(collectedType.IsTable); + + bool fError = false; + returnType = DType.Union(ref fError, collectionType, collectedType, useLegacyDateTimeAccepts: true, context.Features); + if (fError) + { + fValid = false; + if (!SetErrorForMismatchedColumns(collectionType, collectedType, args[1], errors, context.Features)) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrTableDoesNotAcceptThisType); + } + } + + if (fValid) + { + if (context.Features.PowerFxV1CompatibilityRules && argTypes.Length == 2 && (argTypes[1].IsRecord || argTypes[1].IsPrimitive)) + { + returnType = returnType.ToRecord(); + } + else + { + returnType = returnType.ToTable(); + } } return fValid; @@ -323,8 +381,22 @@ public override bool IsAsyncInvocation(CallNode callNode, TexlBinding binding) return Arg0RequiresAsync(callNode, binding); } - public static DType GetCollectedTypeForGivenArgType(Features features, DType argType) + public static DType GetCollectedTypeForGivenArgType(Features features, DType argType, TexlNode arg, ref Dictionary nodeToCoercedTypeMap) { + var singleColumnRecordType = GetCollectedTypeForGivenArgType(features, argType); + + if (!argType.IsPrimitive) + { + return argType; + } + + CollectionUtils.Add(ref nodeToCoercedTypeMap, arg, singleColumnRecordType); + + return singleColumnRecordType; + } + + public static DType GetCollectedTypeForGivenArgType(Features features, DType argType) + { Contracts.Assert(argType.IsValid); if (!argType.IsPrimitive) @@ -333,7 +405,7 @@ public static DType GetCollectedTypeForGivenArgType(Features features, DType arg } // Passed a scalar; make a record out of it, using a name that depends on the type. - string fieldName = Contracts.VerifyValue(CreateInvariantFieldName(features, argType.Kind)); + var fieldName = Contracts.VerifyValue(CreateInvariantFieldName(features, argType.Kind)); return DType.CreateRecord(new TypedName[] { new TypedName(argType, new DName(fieldName)) }); } @@ -341,21 +413,7 @@ protected static string CreateInvariantFieldName(PowerFx.Features features, DKin { Contracts.Assert(dKind >= DKind._Min && dKind < DKind._Lim); - return GetScalarSingleColumnNameForType(features, dKind); - } - - private static string GetScalarSingleColumnNameForType(Features features, DKind kind) - { - return kind switch - { - DKind.Image or - DKind.Hyperlink or - DKind.Media or - DKind.Blob or - DKind.PenImage => features.ConsistentOneColumnTableResult ? TableValue.ValueName : "Url", - - _ => TableValue.ValueName - }; + return MutationUtils.GetScalarSingleColumnNameForType(features, dKind); } } @@ -373,9 +431,9 @@ public static string GetInvariantNameForRecord(PowerFx.Features features, DKind return CreateInvariantFieldName(features, dKind); } - public override DType GetCollectedType(PowerFx.Features features, DType argType) + public override DType GetCollectedType(Features features, DType argType, TexlNode arg, ref Dictionary nodeToCoercedTypeMap) { - return GetCollectedTypeForGivenArgType(features, argType); + return GetCollectedTypeForGivenArgType(features, argType, arg, ref nodeToCoercedTypeMap); } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs index d37679f0f6..7ad289bd79 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs @@ -144,7 +144,7 @@ public static IEnumerable GetPrimitiveTypes() /// Eventually, all display names should come from this centralized source. /// We should not be using individual DataSource/OptionSet/View references. /// - internal DisplayNameProvider DisplayNameProvider { get; private set; } + internal DisplayNameProvider DisplayNameProvider { get; set; } /// /// NamedValueKind is used only for values of kind NamedValue diff --git a/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs index cf5d4fe3c4..db9c7c198e 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs @@ -10,6 +10,7 @@ using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Syntax; +using Microsoft.PowerFx.Types; namespace Microsoft.PowerFx.Core.Utils { @@ -62,6 +63,20 @@ public static void CheckSemantics(TexlBinding binding, TexlFunction function, Te errors.EnsureError(targetArg, TexlStrings.ErrInvalidArgs_Func, function.Name); return; } + } + + public static string GetScalarSingleColumnNameForType(Features features, DKind kind) + { + return kind switch + { + DKind.Image or + DKind.Hyperlink or + DKind.Media or + DKind.Blob or + DKind.PenImage => features.ConsistentOneColumnTableResult ? TableValue.ValueName : "Url", + + _ => TableValue.ValueName + }; } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index dcbf58846a..9438104bf5 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -8,6 +8,8 @@ using System.Threading.Tasks; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Functions; using Microsoft.PowerFx.Interpreter; using Microsoft.PowerFx.Types; @@ -36,8 +38,12 @@ internal async Task Process(IServiceProvider runtimeServiceProvide { FormulaValue arg0; var argc = args.Length; - var returnIsTable = irContext._type.IsTable; - var features = runtimeServiceProvider.GetService(); + var features = runtimeServiceProvider.GetService(); + + if (!features.PowerFxV1CompatibilityRules) + { + throw new InvalidOperationException("Collect funtion can only be executed if PowerFx V1 feature is active."); + } // Need to check if the Lazy first argument has been evaluated since it may have already been // evaluated in the ClearCollect case. @@ -96,37 +102,27 @@ internal async Task Process(IServiceProvider runtimeServiceProvide } else if (arg is RecordValue) { - var recordValueCopy = (RecordValue)arg.MaybeShallowCopy(); + var recordValueCopy = CompileTimeTypeWrapperRecordValue.AdjustType(tableValue.Type.ToRecord(), (RecordValue)arg.MaybeShallowCopy()); resultRows.Add(await tableValue.AppendAsync(recordValueCopy, cancellationToken).ConfigureAwait(false)); } else if (arg is ErrorValue) { return arg; } - else if (arg is BlankValue && !tableValue.Type._type.IsSingleColumnTable) - { - continue; - } - else - { - // If arg is a scalar value, then we need to create a single column record. - NamedValue namedValue = new NamedValue(tableValue.Type.SingleColumnFieldName, arg); - var singleColumnRecord = FormulaValue.NewRecordFromFields(namedValue); - - resultRows.Add(await tableValue.AppendAsync(singleColumnRecord, cancellationToken).ConfigureAwait(false)); - } - } - - if (!features.PowerFxV1CompatibilityRules) - { - return tableValue; + + // !!! How to handle BlankValue? + //else if (arg is BlankValue && !tableValue.Type._type.IsSingleColumnTable) + //{ + // continue; + //} } if (resultRows.Count == 0) { return FormulaValue.NewBlank(arg0.Type); - } - else if (returnIsTable) + } + + if (irContext._type.IsTable) { return CompileTimeTypeWrapperTableValue.AdjustType(tableValue.Type, new InMemoryTableValue(IRContext.NotInSource(arg0.Type), resultRows)); } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUnary.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUnary.cs index 5d6b44796f..76ce28d25a 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUnary.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUnary.cs @@ -523,6 +523,17 @@ internal static partial class Library checkRuntimeValues: DeferRuntimeValueChecking, returnBehavior: ReturnBehavior.ReturnBlankIfAnyArgIsBlank, targetFunction: UntypedStringToUntypedDecimal) + }, + { + UnaryOpKind.PrimitiveToSingleColumnRecord, + StandardErrorHandling( + functionName: null, // internal function, no user-facing name + expandArguments: NoArgExpansion, + replaceBlankValues: DoNotReplaceBlank, + checkRuntimeTypes: ExactValueTypeOrBlank, + checkRuntimeValues: DeferRuntimeValueChecking, + returnBehavior: ReturnBehavior.AlwaysEvaluateAndReturnResult, + targetFunction: PrimitiveToSingleColumnRecord) }, }; #endregion @@ -1027,7 +1038,13 @@ public static FormulaValue BlankToEmptyString(IRContext irContext, FormulaValue[ } return args[0]; - } -#endregion + } + + public static FormulaValue PrimitiveToSingleColumnRecord(IRContext irContext, FormulaValue[] args) + { + var record = FormulaValue.NewRecordFromFields(new NamedValue("Value", args[0])); + return record; + } + #endregion } } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Clear.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Clear.txt index 2a63593c15..d1edfa0e33 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Clear.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Clear.txt @@ -5,12 +5,6 @@ >> Clear(t1);CountRows(t1) 0 ->> Collect(t1, r2); - Collect(t1, {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}); - Clear(t1); - If(CountRows(t1) = 0, "Cleared", "Failed") -"Cleared" - >> Clear() Errors: Error 0-7: Invalid number of arguments: received 0, expected 1. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Clear_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Clear_V1Compat.txt index dd3efec4b4..2cb3a0469f 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Clear_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Clear_V1Compat.txt @@ -17,3 +17,9 @@ Errors: Error 14-15: Invalid argument type (Decimal). Expecting a Table value in >> Clear(t1);Clear(t2) If(true, {test:1}, "Void value (result of the expression can't be used).") + +>> Collect(t1, r2); + Collect(t1, {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}); + Clear(t1); + If(CountRows(t1) = 0, "Cleared", "Failed") +"Cleared" diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt deleted file mode 100644 index 1292dc2f0a..0000000000 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt +++ /dev/null @@ -1,39 +0,0 @@ -#SETUP: EnableExpressionChaining,MutationFunctionsTestSetup - -// Check MutationFunctionsTestSetup handler (PowerFxEvaluationTests.cs) for documentation. - ->> Collect(t1, r2);CountRows(t1) -2 - ->> Collect(t1, r2);Collect(t1, {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false});CountRows(t1) -3 - ->> Collect(t1, {Field1:3});Last(t1).Field2 -Blank() - ->> Collect(t1, If(1>0, r2));CountRows(t1) -2 - ->> IsError(Collect(t1, If(1/0, r2))) -true - ->> Collect(t1) -Errors: Error 0-11: Invalid number of arguments: received 1, expected 2 or more. - ->> Collect(t1, "x") -Errors: Error 0-16: The function 'Collect' has some invalid arguments.|Error 12-15: Invalid argument type. Cannot use Text values in this context. - ->> Collect(t1, 1) -Errors: Error 0-14: The function 'Collect' has some invalid arguments.|Error 12-13: Invalid argument type. Cannot use Decimal values in this context. - ->> Collect(Foo,r2) -Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-14: The specified column 'Field1' does not exist.|Error 0-15: The function 'Collect' has some invalid arguments. - ->> Collect(Foo,Bar) -Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-15: Name isn't valid. 'Bar' isn't recognized.|Error 0-16: The function 'Collect' has some invalid arguments. - ->> Collect(1/0,Foo) -Errors: Error 12-15: Name isn't valid. 'Foo' isn't recognized. - ->> Collect(t1,{Price:200}).Price -Errors: Error 11-22: The specified column 'Price' does not exist.|Error 0-23: The function 'Collect' has some invalid arguments.|Error 23-29: Name isn't valid. 'Price' isn't recognized. \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt index 0bfe631858..a693f49780 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt @@ -1,5 +1,20 @@ #SETUP: EnableExpressionChaining,MutationFunctionsTestSetup,PowerFxV1CompatibilityRules +>> Collect(t1, r2);CountRows(t1) +2 + +>> Collect(t1, r2);Collect(t1, {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false});CountRows(t1) +3 + +>> Collect(t1, {Field1:3});Last(t1).Field2 +Blank() + +>> Collect(t1, If(1>0, r2));CountRows(t1) +2 + +>> IsError(Collect(t1, If(1/0, r2))) +true + >> Collect(t_name, {name: "textInput1"}) {name:"textInput1"} @@ -44,3 +59,18 @@ Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{F >> Collect(t1, Table(r2,r2), {Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Collect(t1, "x") +Errors: Error 0-16: The function 'Collect' has some invalid arguments.|Error 12-15: Invalid argument type. Cannot use Text values in this context. + +>> Collect(Foo,r2) +Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-14: The specified column 'Field1' does not exist.|Error 0-15: The function 'Collect' has some invalid arguments. + +>> Collect(Foo,Bar) +Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-15: Name isn't valid. 'Bar' isn't recognized.|Error 0-16: The function 'Collect' has some invalid arguments. + +>> Collect(1/0,Foo) +Errors: Error 12-15: Name isn't valid. 'Foo' isn't recognized. + +>> Collect(t1,{Price:200}).Price +Errors: Error 0-23: The function 'Collect' has some invalid arguments.|Error 11-22: The specified column 'Price' does not exist. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt deleted file mode 100644 index 8ffb9284ff..0000000000 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt +++ /dev/null @@ -1,43 +0,0 @@ -#SETUP: EnableExpressionChaining,MutationFunctionsTestSetup,disable:PowerFxV1CompatibilityRules - ->> Collect(FirstN(t_name, 0), {name: "textInput1"}) -Table({name:"textInput1"}) - ->> Collect(t1, {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}) -Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false}) - ->> Last(Collect(t_empty,{Value:200})).Value -200 - ->> Collect(Table({name: "VC"}), {surname: "textInput1"}) -Errors: Error 29-52: The specified column 'surname' does not exist. The column with the most similar name is 'name'.|Error 0-53: The function 'Collect' has some invalid arguments. - ->> Collect(Error({Kind:ErrorKind.Custom}), r2) -Errors: Error 40-42: The specified column 'Field1' does not exist.|Error 0-43: The function 'Collect' has some invalid arguments. - ->> Collect(Error({Kind:ErrorKind.Custom}), Error({Kind:ErrorKind.Div0})) -Error({Kind:ErrorKind.Custom}) - ->> Collect(Blank(), r2) -Errors: Error 17-19: The specified column 'Field1' does not exist.|Error 0-20: The function 'Collect' has some invalid arguments. - ->> Collect(Blank(), Blank()) -Blank() - ->> Collect("", "") -Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context. - ->> IsBlank(Collect(t1, Blank())) -false - ->> Collect(t1, r2, r2) -Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) - ->> Collect(t1, r2, 1; 2; r2) -Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) - ->> Collect(t1, Table(r2,r2)) -Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) - ->> Collect(t1, Table(r2,r2), {Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) -Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/DecimalBoot.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/DecimalBoot.txt index 4bd34fcbe5..69469a8d6b 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/DecimalBoot.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/DecimalBoot.txt @@ -13,9 +13,6 @@ Blank() >> Boolean(Filter([1,2,3], Value > 10)) Table() ->> ForAll(Sequence(5), Collect(t_empty, {Value:ThisRecord.Value})); ForAll(Sequence(CountRows(t_empty)), Collect(t_empty2, Index(t_empty, ThisRecord.Value))); Patch(t_empty, First(t_empty), {Value:99}); Patch(t_empty2, Last(t_empty2), {Value:88}); Concat(t_empty, Value) & "," & Concat(t_empty2, Value) -"992345,123488" - >> ForAll(Boolean([3, 1/0, 0, Sqrt(-1)]), IfError(Text(Value), $"ErrorKind={FirstError.Kind}")) Table({Value:"true"},{Value:"ErrorKind=13"},{Value:"false"},{Value:"ErrorKind=24"}) diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch.txt index 4667e95037..bcd3ba967c 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch.txt @@ -42,10 +42,6 @@ Errors: Error 14-31: The specified column 'Field5' does not exist.|Error 0-32: T >> Patch(t1, First(t1), {Field2:"Venus"}).Field2 "Venus" ->> Collect(t1, r2); - Patch(t1, Last(t1), {Field1:10}, {Field2:"Venus"}, {Field4:true}).Field1 -10 - >> If(Patch(t1, First(t1), {Field4:false}).Field4, 1,0) 0 @@ -98,22 +94,6 @@ Errors: Error 45-78: The type of this argument 'Field3' does not match the expec >> Patch(t1,LookUp(t1,DisplayNameField2="earth"),{Field1:100,Field4:false});t1 Table({Field1:100,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:false}) ->> Collect(t1, r2); - Collect(t1, {Field1:3,Field2:"venus",Field3:DateTime(2030,1,1,0,0,0,0),Field4:true}); - Collect(t1, {Field1:4,Field2:"mars",Field3:DateTime(2031,1,1,0,0,0,0),Field4:false}); - Collect(t1, {Field1:5,Field2:"jupter",Field3:DateTime(2032,1,1,0,0,0,0),Field4:true}); - Collect(t1, {Field1:6,Field2:"saturn",Field3:DateTime(2033,1,1,0,0,0,0),Field4:false}); - Patch(t1, First(Filter(t1, Field1 > 4)), {Field2: "phobos"}); -{Field1:5,Field2:"phobos",Field3:DateTime(2032,1,1,0,0,0,0),Field4:true} - ->> Collect(t1, r2); - Collect(t1, {DisplayNameField1:3,Field2:"venus",Field3:DateTime(2030,1,1,0,0,0,0),DisplayNameField4:true}); - Collect(t1, {DisplayNameField1:4,Field2:"mars",Field3:DateTime(2031,1,1,0,0,0,0),DisplayNameField4:false}); - Collect(t1, {DisplayNameField1:5,Field2:"jupter",Field3:DateTime(2032,1,1,0,0,0,0),DisplayNameField4:true}); - Collect(t1, {DisplayNameField1:6,Field2:"saturn",Field3:DateTime(2033,1,1,0,0,0,0),DisplayNameField4:false}); - Patch(t1, Last(Filter(t1, Field4 = false)), {DisplayNameField2: "phobos"}); -{Field1:6,Field2:"phobos",Field3:DateTime(2033,1,1,0,0,0,0),Field4:false} - >> Patch(Foo, First(t1), {DisplayNameField2:"jupter"}) Errors: Error 6-9: Name isn't valid. 'Foo' isn't recognized.|Error 0-51: The function 'Patch' has some invalid arguments. @@ -129,23 +109,6 @@ Table({a:1,b:Blank()},{a:Blank(),b:"2"}) >> Set(t_an_bs, Table({a:1,b:If(false,"")},{a:Blank(),b:"2"})); Patch(t_an_bs, Last(t_an_bs), {a:2}); t_an_bs Table({a:1,b:Blank()},{a:2,b:"2"}) - -// Base record not found. If the record in its entirety is not matched then it should produce an error. ->> Collect(t1,{Field1:3,Field2:"phobos",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}); - Collect(t1,{Field1:2,Field2:"deimos",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}); - Patch(t1,{Field2:"deimos"},{Field3:DateTime(2030,2,1,0,0,0,0)}) -Error({Kind:ErrorKind.NotFound}) - ->> Collect(t1,{Field1:3,Field2:"phobos",Field3:DateTime(2024,2,1,0,0,0,0),Field4:false}); - Collect(t1,{Field1:2,Field2:"deimos",Field3:DateTime(2025,2,1,0,0,0,0),Field4:false}); - Patch(t1,{Field3:DateTime(2025,2,1,0,0,0,0)},{Field2:"pandora", Field1:55}) -Error({Kind:ErrorKind.NotFound}) - ->> Collect(t1, r2); - Patch(t1, {Field4:true}, {Field2:"phobos"}); - First(t1).DisplayNameField2 -Error({Kind:ErrorKind.NotFound}) - >> Patch(t1, {Field1:1}, {DisplayNameField2:"mars"}); First(t1).Field2 Error({Kind:ErrorKind.NotFound}) diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch_V1Compat.txt index 21402dbe9b..0ff2d85a11 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch_V1Compat.txt @@ -41,3 +41,40 @@ Errors: Error 13-143: The value passed to the 'Patch' function cannot be changed {Properties:{Moon:{Name:"Phobos"}}}, {Planet:"Jupter"}) Errors: Error 13-251: The value passed to the 'Patch' function cannot be changed. + + +>> Collect(t1, r2); + Patch(t1, Last(t1), {Field1:10}, {Field2:"Venus"}, {Field4:true}).Field1 +10 + +>> Collect(t1, r2); + Collect(t1, {Field1:3,Field2:"venus",Field3:DateTime(2030,1,1,0,0,0,0),Field4:true}); + Collect(t1, {Field1:4,Field2:"mars",Field3:DateTime(2031,1,1,0,0,0,0),Field4:false}); + Collect(t1, {Field1:5,Field2:"jupter",Field3:DateTime(2032,1,1,0,0,0,0),Field4:true}); + Collect(t1, {Field1:6,Field2:"saturn",Field3:DateTime(2033,1,1,0,0,0,0),Field4:false}); + Patch(t1, First(Filter(t1, Field1 > 4)), {Field2: "phobos"}); +{Field1:5,Field2:"phobos",Field3:DateTime(2032,1,1,0,0,0,0),Field4:true} + +>> Collect(t1, r2); + Collect(t1, {DisplayNameField1:3,Field2:"venus",Field3:DateTime(2030,1,1,0,0,0,0),DisplayNameField4:true}); + Collect(t1, {DisplayNameField1:4,Field2:"mars",Field3:DateTime(2031,1,1,0,0,0,0),DisplayNameField4:false}); + Collect(t1, {DisplayNameField1:5,Field2:"jupter",Field3:DateTime(2032,1,1,0,0,0,0),DisplayNameField4:true}); + Collect(t1, {DisplayNameField1:6,Field2:"saturn",Field3:DateTime(2033,1,1,0,0,0,0),DisplayNameField4:false}); + Patch(t1, Last(Filter(t1, Field4 = false)), {DisplayNameField2: "phobos"}); +{Field1:6,Field2:"phobos",Field3:DateTime(2033,1,1,0,0,0,0),Field4:false} + +// Base record not found. If the record in its entirety is not matched then it should produce an error. +>> Collect(t1,{Field1:3,Field2:"phobos",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}); + Collect(t1,{Field1:2,Field2:"deimos",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}); + Patch(t1,{Field2:"deimos"},{Field3:DateTime(2030,2,1,0,0,0,0)}) +Error({Kind:ErrorKind.NotFound}) + +>> Collect(t1,{Field1:3,Field2:"phobos",Field3:DateTime(2024,2,1,0,0,0,0),Field4:false}); + Collect(t1,{Field1:2,Field2:"deimos",Field3:DateTime(2025,2,1,0,0,0,0),Field4:false}); + Patch(t1,{Field3:DateTime(2025,2,1,0,0,0,0)},{Field2:"pandora", Field1:55}) +Error({Kind:ErrorKind.NotFound}) + +>> Collect(t1, r2); + Patch(t1, {Field4:true}, {Field2:"phobos"}); + First(t1).DisplayNameField2 +Error({Kind:ErrorKind.NotFound}) \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch_V1CompatDisabled.txt index 4dd4154700..b58a190dd8 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Patch_V1CompatDisabled.txt @@ -13,18 +13,6 @@ >> Patch(Table({Value:1}, If(false, {x:1, Value:2}, {Value:2, z:2}), {Value:3}), {Value:2}, {Value:11}) {Value:11} ->> Collect(t_empty, {Value:1}); Collect(t_empty, {Value:2}); Patch(t_empty, {Value:1}, If(false, {x:1, Value:2}, {Value:11, z:2})) -{Value:11} - ->> Collect(t_empty, {Value:1}); Collect(t_empty, {Value:2}); Patch(t_empty, If(false, {x:1, Value:2}, {Value:1, z:2}), {Value:11}) -{Value:11} - ->> Collect(t_empty, {Value:1}); Collect(t_empty, {Value:2}); Patch(t_empty, If(false, {x:1, Value:2}, {Value:1, z:2}), If(false, {x:1, Value:2}, {Value:11, z:2})) -{Value:11} - ->> Collect(t_empty, {Value:1}); Collect(t_empty, If(false, {x:1, Value:2}, {Value:2, z:2})); Collect(t_empty, {Value:3}); Patch(t_empty, {Value:2}, {Value:11}) -{Value:11} - // field22 is missing >> Patch( Table( diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Remove.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Remove.txt index 98276cdd91..d7c577e2d4 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Remove.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Remove.txt @@ -1,4 +1,4 @@ -#SETUP: EnableExpressionChaining,MutationFunctionsTestSetup +#SETUP: PowerFxV1CompatibilityRules,EnableExpressionChaining,MutationFunctionsTestSetup // Check MutationFunctionsTestSetup handler (PowerFxEvaluationTests.cs) for documentation. diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs index 70a16670ee..e2474b557e 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs @@ -382,6 +382,32 @@ public void SymbolTableEnableMutationFuntionsTest() // Mutation functions is listed. var checkEnabled = engine.Check(expr, symbolTable: symbolTableEnabled); Assert.Contains(checkEnabled.Symbols.Functions.FunctionNames, f => f == "Collect"); + } + + [Theory] + [InlineData("Collect(t, {Value:1})")] + [InlineData("Collect(t, 1)")] + [InlineData("ForAll(Sequence(5), Collect(t, {Value:ThisRecord.Value}))")] + [InlineData("Switch(2,1,Last(Collect(t,{Value:1})).Value,2,Last(Collect(t,{Value:2})).Value,3,Last(Collect(t,{Value:3})).Value)")] + [InlineData("IfError(1/0,Last(Collect(t,{Value:3})).Value)")] + public void CollectPFxV1Disabled(string expression) + { + var engine = new RecalcEngine(new PowerFxConfig(Features.None)); + var t = FormulaValue.NewTable(RecordType.Empty().Add(new NamedFormulaType("Value", FormulaType.Decimal))); + + engine.Config.SymbolTable.EnableMutationFunctions(); + engine.UpdateVariable("t", t); + + var check = engine.Check(expression, options: new ParserOptions() { AllowsSideEffects = true }); + + // Compilation will be successful, but the function will not be executed. + // This is because PA depends on the CheckType to determine if the function is valid. + Assert.True(check.IsSuccess); + + var evaluator = check.GetEvaluator(); + + // Runtime exception + Assert.Throws(() => evaluator.Eval()); } internal class FileObjectRecordValue : InMemoryRecordValue diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/AndOr_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/AndOr_V1CompatDisabled.txt index eb98a43650..37cb02f721 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/AndOr_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/AndOr_V1CompatDisabled.txt @@ -6,7 +6,7 @@ Table({Value:false},{Value:true}) // AND ->> false && Last(Collect(t1, {Value:true})).Value;t1 +>> false && Clear(t1);t1 Table({Value:false},{Value:true}) >> 0.00;t1 @@ -30,7 +30,7 @@ Table({Value:false},{Value:true}) >> 0.03;t1 Table({Value:false},{Value:true}) ->> false And Last(Collect(t1, {Value:true})).Value;t1 +>> false And Clear(t1);t1 Table({Value:false},{Value:true}) >> 0.10;t1 @@ -54,7 +54,7 @@ Table({Value:false},{Value:true}) >> 0.13;t1 Table({Value:false},{Value:true}) ->> And(false, Last(Collect(t1, {Value:true})).Value);t1 +>> And(false, Clear(t1));t1 Table({Value:false},{Value:true}) >> 0.14;t1 @@ -79,7 +79,7 @@ Table({Value:false},{Value:true}) Table({Value:false},{Value:true}) // OR with true, none of these should execute ->> true || Last(Collect(t1, {Value:true})).Value +>> true || With({x:1},Clear(t1);true) true >> -1;t1 @@ -105,10 +105,13 @@ Table({Value:false},{Value:true}) Table({Value:false},{Value:true}) // OR with false, these should all execute ->> false || Last(Collect(t1, {Value:true})).Value +>> false || With({x:1},Clear(t1);true) true >> 1;t1 +Table() + +>> 0;Set( t1, [false,true,true] ) Table({Value:false},{Value:true},{Value:true}) >> false || Patch(t1, First(t1), {Value:true}).Value @@ -129,10 +132,13 @@ Table({Value:false},{Value:true}) >> 3.1;t1 Table({Value:false},{Value:true}) ->> Or(false, Last(Collect(t1, {Value:true})).Value) +>> Or(false, With({x:1},Clear(t1);true)) true >> 4;t1 +Table() + +>> 1;Set( t1, [false,true,true] ) Table({Value:false},{Value:true},{Value:true}) >> Or(false, Patch(t1, First(t1), {Value:true}).Value) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuitDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuitDisabled.txt index 878415c0a3..3a6333f6f9 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuitDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuitDisabled.txt @@ -5,42 +5,48 @@ >> Set( t1, [1,2] ) Table({Value:1},{Value:2}) ->> Coalesce(1,Last(Collect(t1,{Value:3})).Value) +>> Coalesce(1,Clear(t1)) 1 >> 1;t1 +Table() + +>> 1;Set( t1, [1,2,3] ) Table({Value:1},{Value:2},{Value:3}) ->> Coalesce(1,Blank(),Last(Collect(t1,{Value:3.1})).Value) +>> Coalesce(1,Blank(),Clear(t1)) 1 >> 1.1;t1 +Table() + +>> 2;Set( t1, [1,2,3,3.1] ) Table({Value:1},{Value:2},{Value:3},{Value:3.1}) >> Set( bn, If(1<0,1)) Blank() ->> Coalesce( bn, Last(Collect(t1,{Value:3.2})).Value ) -3.2 - ->> 2;t1 -Table({Value:1},{Value:2},{Value:3},{Value:3.1},{Value:3.2}) +>> Coalesce( bn, 1.1 ) +1.1 ->> Coalesce( bn, Last(Collect(t1,{Value:4})).Value, Last(Collect(t1,{Value:5})).Value ) -4 +>> Coalesce( bn, Remove(t1, First(t1));Last(t1).Value, Remove(t1, First(t1));Last(t1).Value ) +3.1 >> 3;t1 -Table({Value:1},{Value:2},{Value:3},{Value:3.1},{Value:3.2},{Value:4},{Value:5}) +Table({Value:3},{Value:3.1}) ->> Coalesce( bn, Blank(), Last(Collect(t1,{Value:5.1})).Value ) -5.1 +>> Coalesce( bn, Blank(), Remove(t1, First(t1));Last(t1).Value ) +3.1 >> 4;t1 -Table({Value:1},{Value:2},{Value:3},{Value:3.1},{Value:3.2},{Value:4},{Value:5},{Value:5.1}) +Table({Value:3.1}) + +>> 3;Set( t1, [1,2,3,3.1] ) +Table({Value:1},{Value:2},{Value:3},{Value:3.1}) ->> Coalesce( bn, Blank(), Last(Collect(t1,{Value:6})).Value, Last(Collect(t1,{Value:7})).Value ) -6 +>> Coalesce( bn, Blank(), Remove(t1, First(t1));First(t1).Value, Remove(t1, First(t1));Last(t1).Value ) +2 >> 5;t1 -Table({Value:1},{Value:2},{Value:3},{Value:3.1},{Value:3.2},{Value:4},{Value:5},{Value:5.1},{Value:6},{Value:7}) +Table({Value:3},{Value:3.1}) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt index 98c150f492..d5bd6b09ea 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt @@ -64,7 +64,7 @@ Errors: Error 16-29: The type of this argument 'Value' does not match the expect Table({Value:1},{Value:2},{Value:3}) >> Collect(temp1,{Value:"run time error"}).Value -Errors: Error 14-38: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 0-39: The function 'Collect' has some invalid arguments.|Error 39-45: Name isn't valid. 'Value' isn't recognized. +Errors: Error 0-39: The function 'Collect' has some invalid arguments.|Error 14-38: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 39-45: Deprecated use of '.'. Please use the 'ShowColumns' function instead. >> Set(partialT1, Table({a:1,b:1},{a:2,b:2})) Table({a:1,b:1},{a:2,b:2}) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt deleted file mode 100644 index b0b96e5c84..0000000000 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/DeepMutation_V1CompatDisabled.txt +++ /dev/null @@ -1,412 +0,0 @@ -#SETUP: disable:PowerFxV1CompatibilityRules -// NESTED TABLES - PATCH - ->> Set( aa, [[1,2,3], [4,5,6]] ) -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - ->> Set( ab, First(aa) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - ->> Patch( ab.Value, {Value:2}, {Value:9}) -{Value:9} - ->> ab -{Value:Table({Value:1},{Value:9},{Value:3})} - ->> Set( ac, Last(aa) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - ->> Patch( ac.Value, {Value:5}, {Value:8} ) -{Value:8} - ->> ac -{Value:Table({Value:4},{Value:8},{Value:6})} - ->> Set( ad, Index(aa,1) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - ->> Set( ae, Index(aa,2) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - ->> Patch( ad.Value, {Value:1}, {Value:10}) -{Value:10} - ->> Patch( ae.Value, {Value:6}, {Value:16}) -{Value:16} - ->> ad -{Value:Table({Value:10},{Value:2},{Value:3})} - ->> ae -{Value:Table({Value:4},{Value:5},{Value:16})} - ->> ab // confirm no changes -{Value:Table({Value:1},{Value:9},{Value:3})} - ->> ac // confirm no changes -{Value:Table({Value:4},{Value:8},{Value:6})} - ->> aa // confirm no changes -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - -// NESTED TABLES - COLLECT - ->> Set( ca, [[1,2,3], [4,5,6]] ) -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - ->> Set( cb, First(ca) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - ->> Collect( cb.Value, {Value:9}) -Table({Value:1},{Value:2},{Value:3},{Value:9}) - ->> cb -{Value:Table({Value:1},{Value:2},{Value:3},{Value:9})} - ->> Set( cc, Last(ca) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - ->> Collect( cc.Value, {Value:8} ) -Table({Value:4},{Value:5},{Value:6},{Value:8}) - ->> cc -{Value:Table({Value:4},{Value:5},{Value:6},{Value:8})} - ->> Set( cd, Index(ca,1) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - ->> Set( ce, Index(ca,2) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - ->> Collect( cd.Value, {Value:10}) -Table({Value:1},{Value:2},{Value:3},{Value:10}) - ->> Collect( ce.Value, {Value:16}) -Table({Value:4},{Value:5},{Value:6},{Value:16}) - ->> cd -{Value:Table({Value:1},{Value:2},{Value:3},{Value:10})} - ->> ce -{Value:Table({Value:4},{Value:5},{Value:6},{Value:16})} - ->> cb // confirm no changes -{Value:Table({Value:1},{Value:2},{Value:3},{Value:9})} - ->> cc // confirm no changes -{Value:Table({Value:4},{Value:5},{Value:6},{Value:8})} - ->> ca // confirm no changes -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - -// NESTED TABLES - CLEARCOLLECT - ->> Set( cca, [[1,2,3], [4,5,6]] ) -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - ->> Set( ccb, First(cca) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - ->> ClearCollect( ccb.Value, {Value:9}) -{Value:9} - ->> ccb -{Value:Table({Value:9})} - ->> Set( ccc, Last(ca) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - ->> ClearCollect( ccc.Value, {Value:8} ) -{Value:8} - ->> ccc -{Value:Table({Value:8})} - ->> Set( ccd, Index(ca,1) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - ->> Set( cce, Index(ca,2) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - ->> ClearCollect( ccd.Value, {Value:10}) -{Value:10} - ->> ClearCollect( cce.Value, {Value:16}) -{Value:16} - ->> ccd -{Value:Table({Value:10})} - ->> cce -{Value:Table({Value:16})} - ->> ccb // confirm no changes -{Value:Table({Value:9})} - ->> ccc // confirm no changes -{Value:Table({Value:8})} - ->> cca // confirm no changes -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - -// NESTED TABLES - CLEAR - ->> Set( cla, [[1,2,3], [4,5,6]] ) -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - ->> Set( clb, First(cla) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - -// ; true on the end so that we get the same result with and without V1Compat ->> Clear( clb.Value ); true -true - ->> clb -{Value:Table()} - ->> Set( clc, Last(cla) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - -// ; true on the end so that we get the same result with and without V1Compat ->> Clear( clc.Value ); true -true - ->> clc -{Value:Table()} - ->> Collect( clb.Value, {Value:91}) -Table({Value:91}) - ->> Collect( clc.Value, {Value:81}) -Table({Value:81}) - ->> Set( cld, Index(ca,1) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - ->> Set( cle, Index(ca,2) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - -// ; true on the end so that we get the same result with and without V1Compat ->> Clear( cld.Value); true -true - -// ; true on the end so that we get the same result with and without V1Compat ->> Clear( cle.Value); true -true - ->> cld -{Value:Table()} - ->> cle -{Value:Table()} - ->> clb // confirm no changes -{Value:Table({Value:91})} - ->> clc // confirm no changes -{Value:Table({Value:81})} - ->> cla // confirm no changes -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - -// NESTED TABLES - REMOVE - ->> Set( ra, [[1,2,3], [4,5,6]] ) -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - ->> Set( rb, First(ra) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - -// ; true on the end so that we get the same result with and without V1Compat ->> Remove( rb.Value, {Value:2} ); true -true - ->> rb -{Value:Table({Value:1},{Value:3})} - ->> Set( rc, Last(ra) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - -// ; true on the end so that we get the same result with and without V1Compat ->> Remove( rc.Value, {Value:4} ); true -true - ->> rc -{Value:Table({Value:5},{Value:6})} - ->> Set( rd, Index(ra,1) ) -{Value:Table({Value:1},{Value:2},{Value:3})} - ->> Set( re, Index(ra,2) ) -{Value:Table({Value:4},{Value:5},{Value:6})} - -// ; true on the end so that we get the same result with and without V1Compat ->> Remove( rd.Value, {Value:3}); true -true - -// ; true on the end so that we get the same result with and without V1Compat ->> Remove( re.Value, {Value:4}); true -true - ->> rd -{Value:Table({Value:1},{Value:2})} - ->> re -{Value:Table({Value:5},{Value:6})} - ->> rb // confirm no changes -{Value:Table({Value:1},{Value:3})} - ->> rc // confirm no changes -{Value:Table({Value:5},{Value:6})} - ->> ra // confirm no changes -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - -// DEEP NESTING IN RECORDS - PATCH - ->> Set( ba, {a: {b: {c: {d: [1,2,3]} } } } ) -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3})}}}} - ->> Set( bb, ba.a.b ) -{c:{d:Table({Value:1},{Value:2},{Value:3})}} - ->> Set( bc, ba.a.b.c ) -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Set( bd, bb.c ) -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Patch( ba.a.b.c.d, {Value:3}, {Value:9} ) -{Value:9} - ->> ba -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:9})}}}} - ->> bb -{c:{d:Table({Value:1},{Value:2},{Value:3})}} - ->> bc -{d:Table({Value:1},{Value:2},{Value:3})} - ->> bd -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Patch( bb.c.d, {Value:2}, {Value:8} ) -{Value:8} - ->> ba // after bb -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:9})}}}} - ->> bb // after bb -{c:{d:Table({Value:1},{Value:8},{Value:3})}} - ->> bc // after bb -{d:Table({Value:1},{Value:2},{Value:3})} - ->> bd // after bb -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Patch( bc.d, {Value:1}, {Value:7}) -{Value:7} - ->> ba // after bc -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:9})}}}} - ->> bb // after bc -{c:{d:Table({Value:1},{Value:8},{Value:3})}} - ->> bc // after bc -{d:Table({Value:7},{Value:2},{Value:3})} - ->> bd // after bc -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Patch( bd.d, { Value: 2}, {Value: 11 }) -{Value:11} - ->> ba // after bd -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:9})}}}} - ->> bb // after bd -{c:{d:Table({Value:1},{Value:8},{Value:3})}} - ->> bc // after bd -{d:Table({Value:7},{Value:2},{Value:3})} - ->> bd // after bd -{d:Table({Value:1},{Value:11},{Value:3})} - -// DEEP NESTING IN RECORDS - COLLECT - ->> Set( dba, {a: {b: {c: {d: [1,2,3]} } } } ) -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3})}}}} - ->> Set( dbb, dba.a.b ) -{c:{d:Table({Value:1},{Value:2},{Value:3})}} - ->> Set( dbc, dba.a.b.c ) -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Set( dbd, dbb.c ) -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Collect( dba.a.b.c.d, {Value:9} ) -Table({Value:1},{Value:2},{Value:3},{Value:9}) - ->> dba -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} - ->> dbb -{c:{d:Table({Value:1},{Value:2},{Value:3})}} - ->> dbc -{d:Table({Value:1},{Value:2},{Value:3})} - ->> dbd -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Collect( dbb.c.d, {Value:8} ) -Table({Value:1},{Value:2},{Value:3},{Value:8}) - ->> dba // after dbb -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} - ->> dbb // after dbb -{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:8})}} - ->> dbc // after dbb -{d:Table({Value:1},{Value:2},{Value:3})} - ->> dbd // after dbb -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Collect( dbc.d, {Value:7}) -Table({Value:1},{Value:2},{Value:3},{Value:7}) - ->> dba // after dbc -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} - ->> dbb // after dbc -{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:8})}} - ->> dbc // after dbc -{d:Table({Value:1},{Value:2},{Value:3},{Value:7})} - ->> dbd // after dbc -{d:Table({Value:1},{Value:2},{Value:3})} - ->> Collect( dbd.d, {Value: 11}) -Table({Value:1},{Value:2},{Value:3},{Value:11}) - ->> dba // after dbd -{a:{b:{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:9})}}}} - ->> dbb // after dbd -{c:{d:Table({Value:1},{Value:2},{Value:3},{Value:8})}} - ->> dbc // after dbd -{d:Table({Value:1},{Value:2},{Value:3},{Value:7})} - ->> dbd // after dbd -{d:Table({Value:1},{Value:2},{Value:3},{Value:11})} diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt deleted file mode 100644 index ddcc241ba7..0000000000 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ForAllMutate_V1CompatDisabled.txt +++ /dev/null @@ -1,107 +0,0 @@ -#SETUP: disable:PowerFxV1CompatibilityRules -// Test mutations on a collection from ForAll() - -// These are all verified in Power Apps -// - ->> Set(t, Sequence(4)) -Table({Value:1},{Value:2},{Value:3},{Value:4}) - ->> Collect(t, { Value : 99}); Concat(t, Value) -"123499" - ->> Set(t, ForAll(Sequence(4), Value*10)) -Table({Value:10},{Value:20},{Value:30},{Value:40}) - ->> Collect(t, { Value : 99}); Concat(t, Value) // After t was changed -"1020304099" - -// Initializing t2 ->> Set(t2, [1]) -Table({Value:1}) - ->> With({t: Sequence(5)}, Set(t2, ForAll(t, Value * 2))); t2 -Table({Value:2},{Value:4},{Value:6},{Value:8},{Value:10}) - -// Inner sequence is unchaged ->> With( { t : Sequence(5)}, - Collect(t2, { Value : 99 }); - Concat(t, Value) & "," & Concat(t2, Value) - ) -"12345,24681099" - -// short circuiting shouldn't execute any loops for a blank or empty table - ->> Set(t3,If(1<0,[1,2,3])) -Blank() - ->> ForAll(t3,Patch(t3,ThisRecord,{Value:4})) -Blank() - ->> 1;t3 -Blank() - -// interpretation of an empty table - ->> Set(t4,[1]) -Table({Value:1}) - -// ; true on the end so that we get the same result with and without V1Compat ->> Remove(t4,First(t4)); true -true - ->> 1;t4 -Table() - -// Should do nothing as t4 is empty ->> ForAll(t4,Patch(t4,ThisRecord,{Value:4})) -Table() - ->> 2;t4 -Table() - ->> Set(t5,[1]) -Table({Value:1}) - -// ForAll shouldn't execute the collect, even once ->> ForAll(t4,Collect(t5,{Value:9.1})) -Table() - -// With shouldn't execute the collect, even once ->> With(First(t4),Collect(t5,{Value:9.2})) -Blank() - -// interpretation of one record - ->> 1;t5 -Table({Value:1}) - ->> Collect(t4,{Value:1}) -Table({Value:1}) - -// ForAll should execute the collect, exactly once ->> ForAll(t4,Collect(t5,{Value:8.1})) -Table({Value:Table({Value:1},{Value:8.1})}) - -// With shouldn execute the collect, exactly once ->> With(First(t4),Collect(t5,{Value:8.2})) -Table({Value:1},{Value:8.1},{Value:8.2}) - ->> 2;t5 -Table({Value:1},{Value:8.1},{Value:8.2}) - -// interpretation of a blank table - ->> Set( t4, Blank() ) -Blank() - -// ForAll shouldn't execute the collect, even once ->> ForAll(t4,Collect(t5,{Value:7.1})) -Blank() - -// With shouldn't execute the collect, even once ->> With(First(t4),Collect(t5,{Value:7.2})) -Blank() - ->> 3;t5 -Table({Value:1},{Value:8.1},{Value:8.2}) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError_V1CompatDisabled.txt deleted file mode 100644 index d36193ffd2..0000000000 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/IfError_V1CompatDisabled.txt +++ /dev/null @@ -1,47 +0,0 @@ -#SETUP: disable:PowerFxV1CompatibilityRules -// Cases to test how shortcut verification work along with behavior functions - ->> Set( t1, [1,2] ) -Table({Value:1},{Value:2}) - ->> IfError(1,Last(Collect(t1,{Value:3})).Value) -1 - ->> 1;t1 -Table({Value:1},{Value:2}) - ->> IfError(1/0,Last(Collect(t1,{Value:3})).Value) -3 - ->> 2;t1 -Table({Value:1},{Value:2},{Value:3}) - ->> IfError(1/0,Last(Collect(t1,{Value:4})).Value,1/0,Last(Collect(t1,{Value:5})).Value) -4 - ->> 3;t1 -Table({Value:1},{Value:2},{Value:3},{Value:4}) - ->> IfError(1/0,Last(Collect(t1,{Value:5})).Value,1/0,Last(Collect(t1,{Value:6})).Value,Last(Collect(t1,{Value:7})).Value) -5 - ->> 4;t1 -Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5}) - ->> IfError(1,Last(Collect(t1,{Value:5})).Value,1/0,Last(Collect(t1,{Value:6})).Value,Last(Collect(t1,{Value:7})).Value) -6 - ->> 5;t1 -Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6}) - ->> IfError(1,Last(Collect(t1,{Value:5})).Value,2,Last(Collect(t1,{Value:6})).Value,Last(Collect(t1,{Value:7})).Value) -7 - ->> 6;t1 -Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7}) - ->> IfError(1,Last(Collect(t1,{Value:5})).Value,2,Last(Collect(t1,{Value:6})).Value) -2 - ->> 7;t1 -Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7}) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1CompatDisabled.txt deleted file mode 100644 index 73049e93f6..0000000000 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Set_V1CompatDisabled.txt +++ /dev/null @@ -1,273 +0,0 @@ -#SETUP: disable:PowerFxV1CompatibilityRules - -// Tests for Set/Clear/Patch/Remove with Copy-on-Write semantics - -// TABLE MUTATION - ->> Set( a, [1,2,3] ) -Table({Value:1},{Value:2},{Value:3}) - ->> Set( b, a ) -Table({Value:1},{Value:2},{Value:3}) - ->> Patch( a, {Value:2}, {Value:9} ) -{Value:9} - ->> a -Table({Value:1},{Value:9},{Value:3}) - ->> b -Table({Value:1},{Value:2},{Value:3}) - -// TABLE APPEND - ->> Set( aa, [1,2,3] ) -Table({Value:1},{Value:2},{Value:3}) - ->> Set( ab, aa ) -Table({Value:1},{Value:2},{Value:3}) - ->> Collect( ab, {Value:4} ) -Table({Value:1},{Value:2},{Value:3},{Value:4}) - ->> aa -Table({Value:1},{Value:2},{Value:3}) - ->> ab -Table({Value:1},{Value:2},{Value:3},{Value:4}) - ->> Collect( aa, {Value:5} ) -Table({Value:1},{Value:2},{Value:3},{Value:5}) - ->> aa // again -Table({Value:1},{Value:2},{Value:3},{Value:5}) - ->> ab // again -Table({Value:1},{Value:2},{Value:3},{Value:4}) - -// TABLE WITHIN RECORD - ->> Set( ba, [ [1,2,3], [4,5,6] ]) -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - ->> Set( bb, Last(ba).Value ) -Table({Value:4},{Value:5},{Value:6}) - ->> Patch( bb, {Value:4}, {Value:8} ) -{Value:8} - ->> ba -Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})}) - ->> bb -Table({Value:8},{Value:5},{Value:6}) - -// TABLE REMOVE - ->> Set( ca, [1,2,3] ) -Table({Value:1},{Value:2},{Value:3}) - ->> Set( cb, ca ) -Table({Value:1},{Value:2},{Value:3}) - -// ; true on the end so that we get the same result with and without V1Compat ->> Remove( cb, {Value:2} ); true -true - ->> ca // again -Table({Value:1},{Value:2},{Value:3}) - ->> cb // again -Table({Value:1},{Value:3}) - -// ; true on the end so that we get the same result with and without V1Compat ->> Remove( ca, {Value:1} ); true -true - ->> ca // again 2 -Table({Value:2},{Value:3}) - ->> cb // again 2 -Table({Value:1},{Value:3}) - -// TABLE CLEAR - ->> Set( da, [1,2,3] ) -Table({Value:1},{Value:2},{Value:3}) - ->> Set( db, da ) -Table({Value:1},{Value:2},{Value:3}) - ->> da -Table({Value:1},{Value:2},{Value:3}) - ->> db -Table({Value:1},{Value:2},{Value:3}) - -// ; true on the end so that we get the same result with and without V1Compat ->> Clear( da ); true -true - ->> da // again -Table() - ->> db // again -Table({Value:1},{Value:2},{Value:3}) - -// LOTS OF COPIES - ->> Set( ea, [1,2,3] ) -Table({Value:1},{Value:2},{Value:3}) - ->> Set( eb, ea ) -Table({Value:1},{Value:2},{Value:3}) - ->> Set( ec, eb ) -Table({Value:1},{Value:2},{Value:3}) - ->> Set( ed, ec ) -Table({Value:1},{Value:2},{Value:3}) - ->> Patch( ec, {Value:3}, {Value:7} ) -{Value:7} - ->> ea -Table({Value:1},{Value:2},{Value:3}) - ->> eb -Table({Value:1},{Value:2},{Value:3}) - ->> ec -Table({Value:1},{Value:2},{Value:7}) - ->> ed -Table({Value:1},{Value:2},{Value:3}) - ->> Collect( eb, {Value:9} ) -Table({Value:1},{Value:2},{Value:3},{Value:9}) - ->> ea // again -Table({Value:1},{Value:2},{Value:3}) - ->> eb // again -Table({Value:1},{Value:2},{Value:3},{Value:9}) - ->> ec // again -Table({Value:1},{Value:2},{Value:7}) - ->> ed // again -Table({Value:1},{Value:2},{Value:3}) - -// ; true on the end so that we get the same result with and without V1Compat ->> Clear( ea ); true -true - ->> ea // again 2 -Table() - ->> eb // again 2 -Table({Value:1},{Value:2},{Value:3},{Value:9}) - ->> ec // again 2 -Table({Value:1},{Value:2},{Value:7}) - ->> ed // again 2 -Table({Value:1},{Value:2},{Value:3}) - -// MORE NESTED TABLES - ->> Set( fa, Table( { a:[1,2,3], b:[4,5,6], c:"hi", d:true}, { a:[10,20,30], b:[40,50,60], c:"bye", d:false} ) ) -Table({a:Table({Value:1},{Value:2},{Value:3}),b:Table({Value:4},{Value:5},{Value:6}),c:"hi",d:true},{a:Table({Value:10},{Value:20},{Value:30}),b:Table({Value:40},{Value:50},{Value:60}),c:"bye",d:false}) - ->> Set( fb, fa ) -Table({a:Table({Value:1},{Value:2},{Value:3}),b:Table({Value:4},{Value:5},{Value:6}),c:"hi",d:true},{a:Table({Value:10},{Value:20},{Value:30}),b:Table({Value:40},{Value:50},{Value:60}),c:"bye",d:false}) - ->> Set( fc, Index(fa, 2).b ) -Table({Value:40},{Value:50},{Value:60}) - ->> Patch( fc, {Value:50}, {Value:1000} ) -{Value:1000} - ->> fa -Table({a:Table({Value:1},{Value:2},{Value:3}),b:Table({Value:4},{Value:5},{Value:6}),c:"hi",d:true},{a:Table({Value:10},{Value:20},{Value:30}),b:Table({Value:40},{Value:50},{Value:60}),c:"bye",d:false}) - ->> fb -Table({a:Table({Value:1},{Value:2},{Value:3}),b:Table({Value:4},{Value:5},{Value:6}),c:"hi",d:true},{a:Table({Value:10},{Value:20},{Value:30}),b:Table({Value:40},{Value:50},{Value:60}),c:"bye",d:false}) - ->> fc -Table({Value:40},{Value:1000},{Value:60}) - -// NESTED TABLES DEEPLY IN RECORDS - ->> Set( ga, { a: { b: { c: [1,2,3] }}}) -{a:{b:{c:Table({Value:1},{Value:2},{Value:3})}}} - ->> Set( gb, ga.a.b.c ) -Table({Value:1},{Value:2},{Value:3}) - ->> Patch( gb, First(gb), {Value:9} ) -{Value:9} - ->> ga -{a:{b:{c:Table({Value:1},{Value:2},{Value:3})}}} - ->> gb -Table({Value:9},{Value:2},{Value:3}) - ->> Set( gc, ga.a ) -{b:{c:Table({Value:1},{Value:2},{Value:3})}} - ->> Set( gd, gc.b ) -{c:Table({Value:1},{Value:2},{Value:3})} - ->> Set( ge, gd.c ) -Table({Value:1},{Value:2},{Value:3}) - ->> Collect( ge, {Value:88} ) -Table({Value:1},{Value:2},{Value:3},{Value:88}) - ->> ga // again -{a:{b:{c:Table({Value:1},{Value:2},{Value:3})}}} - ->> gb // again -Table({Value:9},{Value:2},{Value:3}) - ->> gc // again -{b:{c:Table({Value:1},{Value:2},{Value:3})}} - ->> gd // again -{c:Table({Value:1},{Value:2},{Value:3})} - ->> ge // again -Table({Value:1},{Value:2},{Value:3},{Value:88}) - -// SET RECORD FROM ANOTHER TABLE - ->> Set( ha, [1,2,3] ) -Table({Value:1},{Value:2},{Value:3}) - ->> Set( hb, Table(Index(ha,2),Last(ha),First(ha))) -Table({Value:2},{Value:3},{Value:1}) - ->> ha -Table({Value:1},{Value:2},{Value:3}) - ->> hb -Table({Value:2},{Value:3},{Value:1}) - ->> Patch( ha, First(ha), {Value:5}) -{Value:5} - ->> Patch( ha, Index(ha,2), {Value:6}) -{Value:6} - ->> Patch( ha, Last(ha), {Value:7}) -{Value:7} - ->> ha // again -Table({Value:5},{Value:6},{Value:7}) - ->> hb // again -Table({Value:2},{Value:3},{Value:1}) - - diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt deleted file mode 100644 index 509b114ec4..0000000000 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Switch_V1CompatDisabled.txt +++ /dev/null @@ -1,54 +0,0 @@ -#SETUP: disable:PowerFxV1CompatibilityRules - -// Cases to test how shortcut verification work along with behavior functions - ->> Set( t1, [0,1] ) -Table({Value:0},{Value:1}) - ->> Switch(2,1,Last(Collect(t1,{Value:1})).Value,2,Last(Collect(t1,{Value:2})).Value,3,Last(Collect(t1,{Value:3})).Value) -2 - ->> 1;t1 -Table({Value:0},{Value:1},{Value:2}) - ->> Switch(3, - 1,Last(Collect(t1,{Value:1})).Value, - 2,Last(Collect(t1,{Value:2})).Value, - 3,Last(Collect(t1,{Value:3})).Value, - Last(Collect(t1,{Value:4})).Value) -3 - ->> 2;t1 -Table({Value:0},{Value:1},{Value:2},{Value:3}) - ->> Switch(1, - 1,Last(Collect(t1,{Value:4})).Value, - 2,Last(Collect(t1,{Value:6})).Value, - 3,Last(Collect(t1,{Value:7})).Value, - Last(Collect(t1,{Value:8})).Value) -4 - ->> 3;t1 -Table({Value:0},{Value:1},{Value:2},{Value:3},{Value:4}) - ->> Switch(6, - Last(Collect(t1,{Value:5})).Value,1, - Last(Collect(t1,{Value:6})).Value,12, - 2,Last(Collect(t1,{Value:7})).Value, - Last(Collect(t1,{Value:8})).Value) -12 - ->> 4;t1 -Table({Value:0},{Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6}) - ->> Switch(9, - Last(Collect(t1,{Value:7})).Value,Last(Collect(t1,{Value:20})).Value, - Last(Collect(t1,{Value:8})).Value,2, - 6,8, - Last(Collect(t1,{Value:9})).Value,Last(Collect(t1,{Value:10})).Value, - Last(Collect(t1,{Value:30})).Value) -10 - ->> 5;t1 -Table({Value:0},{Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7},{Value:8},{Value:9},{Value:10}) - From f79b5abf633a0b2667fb74bb425dbe869bcfcdc9 Mon Sep 17 00:00:00 2001 From: "Anderson Ferreira da Silva (from Dev Box)" Date: Mon, 4 Mar 2024 12:42:05 -0600 Subject: [PATCH 07/29] Fixes. Rebuild Collect IR based on arguments. --- .../Microsoft.PowerFx.Core/IR/CoercionKind.cs | 3 +- .../IR/CoercionMatrix.cs | 7 --- .../Microsoft.PowerFx.Core/IR/IRTranslator.cs | 8 --- .../IR/Nodes/UnaryOpKind.cs | 3 +- .../Texl/Builtins/Collect.cs | 61 ++++++++++++++----- .../Functions/LibraryMutation.cs | 20 ++++-- .../Functions/LibraryUnary.cs | 17 ------ 7 files changed, 63 insertions(+), 56 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/CoercionKind.cs b/src/libraries/Microsoft.PowerFx.Core/IR/CoercionKind.cs index 0de57192c7..d352851155 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/CoercionKind.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/CoercionKind.cs @@ -107,7 +107,6 @@ internal enum CoercionKind CurrencyToText, TextToCurrency, CurrencyToBoolean, - BooleanToCurrency, - PrimitiveToSingleColumnRecord, + BooleanToCurrency, } } diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/CoercionMatrix.cs b/src/libraries/Microsoft.PowerFx.Core/IR/CoercionMatrix.cs index c75c9b355c..b22038c721 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/CoercionMatrix.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/CoercionMatrix.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; @@ -22,12 +21,6 @@ public static CoercionKind GetCoercionKind(DType fromType, DType toType, bool us if (fromType.IsAggregate && toType.Kind == DKind.DataEntity) { return CoercionKind.AggregateToDataEntity; - } - - // Coercion from a primitive type to a single column record type. - if (fromType.IsPrimitive && toType.IsRecord) - { - return CoercionKind.PrimitiveToSingleColumnRecord; } if (toType.IsLargeImage && (fromType.Kind == DKind.Image || fromType == DType.MinimalLargeImage)) diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs b/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs index f73a53644c..7754a31408 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs @@ -3,16 +3,11 @@ using System; using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; using System.Linq; -using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Binding; -using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.IR.Nodes; using Microsoft.PowerFx.Core.IR.Symbols; -using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Texl; using Microsoft.PowerFx.Core.Texl.Builtins; using Microsoft.PowerFx.Core.Types; @@ -1179,9 +1174,6 @@ private IntermediateNode InjectCoercion(IntermediateNode child, IRTranslatorCont break; case CoercionKind.PenImageToText: unaryOpKind = UnaryOpKind.PenImageToText; - break; - case CoercionKind.PrimitiveToSingleColumnRecord: - unaryOpKind = UnaryOpKind.PrimitiveToSingleColumnRecord; break; case CoercionKind.UntypedToText: return new CallNode(IRContext.NotInSource(FormulaType.Build(toType)), BuiltinFunctionsCore.Text_UO, child); diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/Nodes/UnaryOpKind.cs b/src/libraries/Microsoft.PowerFx.Core/IR/Nodes/UnaryOpKind.cs index dfa45df0ef..4fa2837382 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/Nodes/UnaryOpKind.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/Nodes/UnaryOpKind.cs @@ -98,8 +98,7 @@ internal enum UnaryOpKind DateToDateTime, BooleanToOptionSet, - AggregateToDataEntity, - PrimitiveToSingleColumnRecord, + AggregateToDataEntity, // Argument pre-processesor in IR Phase. diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 171dbdc772..f8c03c2790 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -9,11 +9,15 @@ using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.Functions.FunctionArgValidators; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.IR.Nodes; +using Microsoft.PowerFx.Core.IR.Symbols; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; -using Microsoft.PowerFx.Syntax; +using Microsoft.PowerFx.Syntax; using Microsoft.PowerFx.Types; +using RecordNode = Microsoft.PowerFx.Core.IR.Nodes.RecordNode; namespace Microsoft.PowerFx.Core.Texl.Builtins { @@ -91,7 +95,7 @@ protected CollectFunction(string name, TexlStrings.StringGetter description) return base.GetSignatures(arity); } - public virtual DType GetCollectedType(Features features, DType argType, TexlNode arg, ref Dictionary nodeToCoercedTypeMap) + public virtual DType GetCollectedType(PowerFx.Features features, DType argType) { Contracts.Assert(argType.IsValid); @@ -114,7 +118,7 @@ public bool TryGetUnifiedCollectedTypeCanvas(TexlNode[] args, DType[] argTypes, for (int i = 1; i < argc; i++) { - DType argType = GetCollectedType(features, argTypes[i], args[i], ref nodeToCoercedTypeMap); + DType argType = GetCollectedType(features, argTypes[i]); // The subsequent args should all be aggregates. if (!argType.IsAggregate) @@ -176,7 +180,7 @@ private bool TryGetUnifiedCollectedTypeV1(TexlNode[] args, DType[] argTypes, IEr for (var i = 1; i < argc; i++) { - DType argType = GetCollectedType(features, argTypes[i], args[i], ref nodeToCoercedTypeMap); + DType argType = GetCollectedType(features, argTypes[i]); // !!! How is it possible for an argtype to be a primitive and an aggregate at the same time? //if (argType.DisplayNameProvider == null && argType.Kind == DKind.ObjNull) @@ -314,12 +318,14 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ } base.CheckSemantics(binding, args, argTypes, errors); - base.ValidateArgumentIsMutable(binding, args[0], errors); + base.ValidateArgumentIsMutable(binding, args[0], errors); - int skip = 1; - - MutationUtils.CheckSemantics(binding, this, args, argTypes, errors); - MutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(skip).ToArray(), argTypes.Skip(skip).ToArray(), errors); + MutationUtils.CheckSemantics(binding, this, args, argTypes, errors); + + if (binding.Features.PowerFxV1CompatibilityRules) + { + MutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(1).ToArray(), argTypes.Skip(1).ToArray(), errors); + } } // This method returns true if there are special suggestions for a particular parameter of the function. @@ -330,7 +336,7 @@ public override bool HasSuggestionsForParam(int argumentIndex) return argumentIndex == 0; } - public override bool TryGetDataSourceNodes(CallNode callNode, TexlBinding binding, out IList dsNodes) + public override bool TryGetDataSourceNodes(PowerFx.Syntax.CallNode callNode, TexlBinding binding, out IList dsNodes) { Contracts.AssertValue(callNode); Contracts.AssertValue(binding); @@ -373,7 +379,7 @@ public override IEnumerable GetIdentifierOfModifiedValue(TexlNode[] return identifiers; } - public override bool IsAsyncInvocation(CallNode callNode, TexlBinding binding) + public override bool IsAsyncInvocation(PowerFx.Syntax.CallNode callNode, TexlBinding binding) { Contracts.AssertValue(callNode); Contracts.AssertValue(binding); @@ -390,7 +396,7 @@ public static DType GetCollectedTypeForGivenArgType(Features features, DType arg return argType; } - CollectionUtils.Add(ref nodeToCoercedTypeMap, arg, singleColumnRecordType); + //CollectionUtils.Add(ref nodeToCoercedTypeMap, arg, singleColumnRecordType); return singleColumnRecordType; } @@ -431,9 +437,36 @@ public static string GetInvariantNameForRecord(PowerFx.Features features, DKind return CreateInvariantFieldName(features, dKind); } - public override DType GetCollectedType(Features features, DType argType, TexlNode arg, ref Dictionary nodeToCoercedTypeMap) + public override DType GetCollectedType(Features features, DType argType) { - return GetCollectedTypeForGivenArgType(features, argType, arg, ref nodeToCoercedTypeMap); + return GetCollectedTypeForGivenArgType(features, argType); + } + + internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) + { + var newArgs = new List() { args[0] }; + + // !!! Blank()? + + foreach (var arg in args.Skip(1)) + { + if (arg.IRContext.ResultType._type.IsPrimitive) + { + newArgs.Add( + new RecordNode( + new IRContext(arg.IRContext.SourceContext, RecordType.Empty().Add(TableValue.ValueName, arg.IRContext.ResultType)), + new Dictionary + { + { TableValue.ValueDName, arg } + })); + } + else + { + newArgs.Add(arg); + } + } + + return base.CreateIRCallNode(node, context, newArgs, scope); } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index 9438104bf5..a0ecd6fdf0 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -108,13 +108,16 @@ internal async Task Process(IServiceProvider runtimeServiceProvide else if (arg is ErrorValue) { return arg; + } + + // !!! This should be moved to IR. + else if (arg is BlankValue) + { + if (tableValue.Type._type.IsSingleColumnTable && tableValue.Type.GetFieldTypes().First().Name.Value == "Value") + { + resultRows.Add(await tableValue.AppendAsync(CreateRecordFromPrimitive(tableValue, arg), cancellationToken).ConfigureAwait(false)); + } } - - // !!! How to handle BlankValue? - //else if (arg is BlankValue && !tableValue.Type._type.IsSingleColumnTable) - //{ - // continue; - //} } if (resultRows.Count == 0) @@ -130,6 +133,11 @@ internal async Task Process(IServiceProvider runtimeServiceProvide { return CompileTimeTypeWrapperRecordValue.AdjustType(tableValue.Type.ToRecord(), (RecordValue)resultRows.First().ToFormulaValue()); } + } + + private RecordValue CreateRecordFromPrimitive(TableValue tableValue, FormulaValue arg) + { + return FormulaValue.NewRecordFromFields(tableValue.Type.ToRecord(), new NamedValue("Value", arg)); } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUnary.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUnary.cs index 76ce28d25a..2832047b46 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUnary.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUnary.cs @@ -523,17 +523,6 @@ internal static partial class Library checkRuntimeValues: DeferRuntimeValueChecking, returnBehavior: ReturnBehavior.ReturnBlankIfAnyArgIsBlank, targetFunction: UntypedStringToUntypedDecimal) - }, - { - UnaryOpKind.PrimitiveToSingleColumnRecord, - StandardErrorHandling( - functionName: null, // internal function, no user-facing name - expandArguments: NoArgExpansion, - replaceBlankValues: DoNotReplaceBlank, - checkRuntimeTypes: ExactValueTypeOrBlank, - checkRuntimeValues: DeferRuntimeValueChecking, - returnBehavior: ReturnBehavior.AlwaysEvaluateAndReturnResult, - targetFunction: PrimitiveToSingleColumnRecord) }, }; #endregion @@ -1039,12 +1028,6 @@ public static FormulaValue BlankToEmptyString(IRContext irContext, FormulaValue[ return args[0]; } - - public static FormulaValue PrimitiveToSingleColumnRecord(IRContext irContext, FormulaValue[] args) - { - var record = FormulaValue.NewRecordFromFields(new NamedValue("Value", args[0])); - return record; - } #endregion } } From a547230a0b6ff590b85cc4391bd97c8e1e545e8e Mon Sep 17 00:00:00 2001 From: Anderson Ferreira da Silva Date: Tue, 5 Mar 2024 10:15:58 -0600 Subject: [PATCH 08/29] Push after changing git user.name --- .../Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index a0ecd6fdf0..8948a3965d 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -110,7 +110,7 @@ internal async Task Process(IServiceProvider runtimeServiceProvide return arg; } - // !!! This should be moved to IR. + // !!!! This should be moved to IR. else if (arg is BlankValue) { if (tableValue.Type._type.IsSingleColumnTable && tableValue.Type.GetFieldTypes().First().Name.Value == "Value") From bcb871fac84f51f639036db1ca5988e3d15d4304 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Tue, 5 Mar 2024 16:34:12 -0600 Subject: [PATCH 09/29] PR feedback. --- .../Texl/Builtins/Collect.cs | 53 +++++++++---------- .../Microsoft.PowerFx.Core/Types/DType.cs | 2 +- .../MutationFunctionsTests.cs | 4 +- 3 files changed, 27 insertions(+), 32 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index f8c03c2790..617f29d177 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -8,6 +8,7 @@ using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Functions.DLP; using Microsoft.PowerFx.Core.Functions.FunctionArgValidators; using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.IR.Nodes; @@ -42,7 +43,9 @@ internal class CollectFunction : BuiltinFunction public override bool CanSuggestInputColumns => true; - public override bool MutatesArg0 => true; + public override bool MutatesArg0 => true; + + public override RequiredDataSourcePermissions FunctionPermission => RequiredDataSourcePermissions.Create; /// /// Since Arg1 and Arg2 depends on type of Arg1 return false for them. @@ -81,7 +84,12 @@ protected CollectFunction(string name, TexlStrings.StringGetter description) } public override IEnumerable GetSignatures() - { + { + // !!! TODO + //yield return new[] { CanvasStringResources.CollectArg1, CanvasStringResources.CollectArg2 }; + //yield return new[] { CanvasStringResources.CollectArg1, CanvasStringResources.CollectArg2, CanvasStringResources.CollectArg2 }; + //yield return new[] { CanvasStringResources.CollectArg1, CanvasStringResources.CollectArg2, CanvasStringResources.CollectArg2, CanvasStringResources.CollectArg2 }; + yield return new[] { TexlStrings.CollectDataSourceArg, TexlStrings.CollectItemArg }; } @@ -95,14 +103,14 @@ protected CollectFunction(string name, TexlStrings.StringGetter description) return base.GetSignatures(arity); } - public virtual DType GetCollectedType(PowerFx.Features features, DType argType) + public virtual DType GetCollectedType(Features features, DType argType) { Contracts.Assert(argType.IsValid); return argType; } - public bool TryGetUnifiedCollectedTypeCanvas(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType, ref Dictionary nodeToCoercedTypeMap) + public bool TryGetUnifiedCollectedTypeCanvas(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType) { Contracts.AssertValue(args); Contracts.AssertAllValues(args); @@ -157,12 +165,12 @@ public bool TryGetUnifiedCollectedTypeCanvas(TexlNode[] args, DType[] argTypes, } Contracts.Assert(!itemType.IsValid || itemType.IsTable); - collectedType = itemType.IsValid ? itemType.ToTable() : DType.EmptyTable; + collectedType = itemType.IsValid ? itemType : DType.EmptyTable; return fValid; } // Attempt to get the unified schema of the items being collected by an invocation. - private bool TryGetUnifiedCollectedTypeV1(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType, ref Dictionary nodeToCoercedTypeMap) + private bool TryGetUnifiedCollectedTypeV1(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType) { Contracts.AssertValue(args); Contracts.AssertAllValues(args); @@ -259,11 +267,11 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp // document errors for invalid arguments such as unsupported aggregate types. if (context.Features.PowerFxV1CompatibilityRules) { - fValid &= TryGetUnifiedCollectedTypeV1(args, argTypes, errors, context.Features, out collectedType, ref nodeToCoercedTypeMap); + fValid &= TryGetUnifiedCollectedTypeV1(args, argTypes, errors, context.Features, out collectedType); } else { - fValid &= TryGetUnifiedCollectedTypeCanvas(args, argTypes, errors, context.Features, out collectedType, ref nodeToCoercedTypeMap); + fValid &= TryGetUnifiedCollectedTypeCanvas(args, argTypes, errors, context.Features, out collectedType); } Contracts.Assert(collectedType.IsTable); @@ -295,7 +303,8 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp } public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors) - { + { + // !!! TODO DType dataSourceType = argTypes[0]; bool isConnected = binding.EntityScope != null && binding.EntityScope.TryGetDataSource(args[0], out IExternalDataSource dataSourceInfo) @@ -340,12 +349,14 @@ public override bool TryGetDataSourceNodes(PowerFx.Syntax.CallNode callNode, Tex { Contracts.AssertValue(callNode); Contracts.AssertValue(binding); - - dsNodes = new List(); + if (callNode.Args.Count != 2) - { + { + dsNodes = null; return false; - } + } + + dsNodes = new List(); var args = Contracts.VerifyValue(callNode.Args.Children); var arg1 = Contracts.VerifyValue(args[1]); @@ -385,20 +396,6 @@ public override bool IsAsyncInvocation(PowerFx.Syntax.CallNode callNode, TexlBin Contracts.AssertValue(binding); return Arg0RequiresAsync(callNode, binding); - } - - public static DType GetCollectedTypeForGivenArgType(Features features, DType argType, TexlNode arg, ref Dictionary nodeToCoercedTypeMap) - { - var singleColumnRecordType = GetCollectedTypeForGivenArgType(features, argType); - - if (!argType.IsPrimitive) - { - return argType; - } - - //CollectionUtils.Add(ref nodeToCoercedTypeMap, arg, singleColumnRecordType); - - return singleColumnRecordType; } public static DType GetCollectedTypeForGivenArgType(Features features, DType argType) @@ -446,8 +443,6 @@ internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node { var newArgs = new List() { args[0] }; - // !!! Blank()? - foreach (var arg in args.Skip(1)) { if (arg.IRContext.ResultType._type.IsPrimitive) diff --git a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs index 7ad289bd79..d37679f0f6 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs @@ -144,7 +144,7 @@ public static IEnumerable GetPrimitiveTypes() /// Eventually, all display names should come from this centralized source. /// We should not be using individual DataSource/OptionSet/View references. /// - internal DisplayNameProvider DisplayNameProvider { get; set; } + internal DisplayNameProvider DisplayNameProvider { get; private set; } /// /// NamedValueKind is used only for values of kind NamedValue diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs index e2474b557e..85f5b49fe0 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs @@ -4,8 +4,8 @@ using System; using System.Collections.Generic; using System.Data; -using System.Diagnostics.Contracts; using System.Linq; +using System.Threading; using Microsoft.PowerFx.Core.IR; using Microsoft.PowerFx.Core.Tests; using Microsoft.PowerFx.Core.Tests.Helpers; @@ -407,7 +407,7 @@ public void CollectPFxV1Disabled(string expression) var evaluator = check.GetEvaluator(); // Runtime exception - Assert.Throws(() => evaluator.Eval()); + Assert.ThrowsAsync(async () => await evaluator.EvalAsync(CancellationToken.None).ConfigureAwait(false)).ConfigureAwait(false); } internal class FileObjectRecordValue : InMemoryRecordValue From c8a82821a562bb146784a8a8e4f0d4b0a3521a2e Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Tue, 5 Mar 2024 16:55:48 -0600 Subject: [PATCH 10/29] Removing deade code. --- .../Functions/LibraryMutation.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index 8948a3965d..3e455cf675 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -108,15 +108,6 @@ internal async Task Process(IServiceProvider runtimeServiceProvide else if (arg is ErrorValue) { return arg; - } - - // !!!! This should be moved to IR. - else if (arg is BlankValue) - { - if (tableValue.Type._type.IsSingleColumnTable && tableValue.Type.GetFieldTypes().First().Name.Value == "Value") - { - resultRows.Add(await tableValue.AppendAsync(CreateRecordFromPrimitive(tableValue, arg), cancellationToken).ConfigureAwait(false)); - } } } From 3249eb451ffaa9d7af019ca821f043dbc5c7b714 Mon Sep 17 00:00:00 2001 From: Anderson Ferreira da Silva Date: Thu, 7 Mar 2024 14:52:34 -0600 Subject: [PATCH 11/29] Adding code from https://github.com/microsoft/Power-Fx/pull/2246 since this PR depends on it. --- .../Microsoft.PowerFx.Core/Binding/Binder.cs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs b/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs index 6b3cd5f810..2c512d7373 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs @@ -2873,12 +2873,20 @@ public override void Visit(FirstNameNode node) isConstantNamedFormula = formula.IsConstant; } } - else if (lookupInfo.Kind == BindKind.Data) - { - if (lookupInfo.Data is IExternalCdsDataSource or IExternalTabularDataSource) - { - _txb.SetMutable(node, true); - } + else if (lookupInfo.Kind == BindKind.Data) + { + if (lookupInfo.Data is IExternalCdsDataSource or IExternalTabularDataSource) + { + _txb.SetMutable(node, true); + } + else if (lookupInfo.Data is IExternalDataSource ds) + { + _txb.SetMutable(node, ds.IsWritable); + } + } + else if (lookupInfo.Kind == BindKind.ScopeCollection) + { + _txb.SetMutable(node, true); } Contracts.Assert(lookupInfo.Kind != BindKind.LambdaField); From 90b12e228a7522754e0092c68bdc611fcbe6e4bc Mon Sep 17 00:00:00 2001 From: Anderson Ferreira da Silva Date: Thu, 7 Mar 2024 19:49:53 -0600 Subject: [PATCH 12/29] New string resources. --- .../Localization/Strings.cs | 4 ++-- .../Texl/Builtins/Collect.cs | 18 ++++----------- .../Functions/Mutation/CollectFunction.cs | 2 +- src/strings/PowerFxResources.en-US.resx | 22 ++++++++++++------- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs index 02ff7e2f43..6b64c2cf65 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs @@ -492,8 +492,8 @@ internal static class TexlStrings public static StringGetter PatchChangeRecordsArg = (b) => StringResources.Get("PatchChangeRecordsArg", b); public static StringGetter AboutCollect = (b) => StringResources.Get("AboutCollect", b); - public static StringGetter CollectDataSourceArg = (b) => StringResources.Get("CollectDataSourceArg", b); - public static StringGetter CollectRecordArg = (b) => StringResources.Get("CollectRecordArg", b); + public static StringGetter CollectArg1 = (b) => StringResources.Get("CollectArg1", b); + public static StringGetter CollectArg2 = (b) => StringResources.Get("CollectArg2", b); public static StringGetter CollectItemArg = (b) => StringResources.Get("CollectItemArg", b); public static StringGetter AboutCollect_data_source = (b) => StringResources.Get("AboutCollect_data_source", b); diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 617f29d177..ece1ee1d6a 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -85,19 +85,16 @@ protected CollectFunction(string name, TexlStrings.StringGetter description) public override IEnumerable GetSignatures() { - // !!! TODO - //yield return new[] { CanvasStringResources.CollectArg1, CanvasStringResources.CollectArg2 }; - //yield return new[] { CanvasStringResources.CollectArg1, CanvasStringResources.CollectArg2, CanvasStringResources.CollectArg2 }; - //yield return new[] { CanvasStringResources.CollectArg1, CanvasStringResources.CollectArg2, CanvasStringResources.CollectArg2, CanvasStringResources.CollectArg2 }; - - yield return new[] { TexlStrings.CollectDataSourceArg, TexlStrings.CollectItemArg }; + yield return new[] { TexlStrings.CollectArg1, TexlStrings.CollectArg2 }; + yield return new[] { TexlStrings.CollectArg1, TexlStrings.CollectArg2, TexlStrings.CollectArg2 }; + yield return new[] { TexlStrings.CollectArg1, TexlStrings.CollectArg2, TexlStrings.CollectArg2, TexlStrings.CollectArg2 }; } public override IEnumerable GetSignatures(int arity) { if (arity > 2) { - return GetGenericSignatures(arity, TexlStrings.CollectDataSourceArg, TexlStrings.CollectItemArg); + return GetGenericSignatures(arity, TexlStrings.CollectArg1, TexlStrings.CollectArg2); } return base.GetSignatures(arity); @@ -190,12 +187,6 @@ private bool TryGetUnifiedCollectedTypeV1(TexlNode[] args, DType[] argTypes, IEr { DType argType = GetCollectedType(features, argTypes[i]); - // !!! How is it possible for an argtype to be a primitive and an aggregate at the same time? - //if (argType.DisplayNameProvider == null && argType.Kind == DKind.ObjNull) - //{ - // argType.DisplayNameProvider = datasourceType.DisplayNameProvider; - //} - // The subsequent args should all be aggregates. if (!argType.IsAggregate) { @@ -304,7 +295,6 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors) { - // !!! TODO DType dataSourceType = argTypes[0]; bool isConnected = binding.EntityScope != null && binding.EntityScope.TryGetDataSource(args[0], out IExternalDataSource dataSourceInfo) diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs index 8454c1a181..03fb8e3265 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs @@ -93,7 +93,7 @@ public CollectFunction() public override IEnumerable GetSignatures() { - yield return new[] { TexlStrings.CollectDataSourceArg, TexlStrings.CollectRecordArg }; + yield return new[] { TexlStrings.CollectArg1, TexlStrings.CollectArg2 }; } public virtual DType GetCollectedType(DType argType) diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index 61f52be50c..d5bad82997 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -3109,16 +3109,22 @@ Display text representing the Contrast value of TeamsTheme enum (TeamsTheme_Contrast_Name). The possible values for this enumeration are: Default, Dark, Contrast. - Adds records to a data source. - Description of 'Clear' function. + Adds one or more items to the specified 'collection'. The items can be from a different table or collection (e.g. Collect(collection, source_collection)), or one or more records Collect(collection, {key1: val1, key2: val2, ...}, ...). + Description of 'Collect' function. - - data_source - function_parameter - First parameter for the Collect function. The data source that you want to add data to. + + collection + function_parameter - First parameter for the Collect function - the name of the collection that will be created / appended to. - - record - function_parameter - Second parameter for the Collect function. The record to be added. + + item + function_parameter - Second parameter for the Collect function - the item which will be added to the collection. + + + A new or existing collection to augment. + + + A record or table to collect. A record will be appended to the collection. A table will have its rows appended to the collection. Deletes all the records of a collection. The columns of the collection will remain. From 07c9a3032b0bff3e228dd1bec444f2295d96639e Mon Sep 17 00:00:00 2001 From: Anderson Ferreira da Silva Date: Fri, 8 Mar 2024 12:52:34 -0600 Subject: [PATCH 13/29] Fixing suggestions for Collect function. --- .../Texl/Builtins/Collect.cs | 19 +++++++++++++------ .../ExpressionTestCases/Collect_V1Compat.txt | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index ece1ee1d6a..b95764b6c1 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -51,14 +51,21 @@ internal class CollectFunction : BuiltinFunction /// Since Arg1 and Arg2 depends on type of Arg1 return false for them. /// public override bool TryGetTypeForArgSuggestionAt(int argIndex, out DType type) - { - if (argIndex == 1 || argIndex == 2) + { + if (argIndex == 0) + { + type = DType.EmptyTable; + return true; + } + else if (argIndex == 1 || argIndex == 2) { type = default; return false; - } - - return base.TryGetTypeForArgSuggestionAt(argIndex, out type); + } + else + { + return base.TryGetTypeForArgSuggestionAt(argIndex, out type); + } } public override bool ArgMatchesDatasourceType(int argNum) @@ -79,7 +86,7 @@ public CollectFunction() } protected CollectFunction(string name, TexlStrings.StringGetter description) - : base(name, description, FunctionCategories.Behavior, DType.EmptyRecord, 0, 2, int.MaxValue, DType.EmptyTable) + : base(name, description, FunctionCategories.Behavior, DType.EmptyTable, 0, 2, int.MaxValue) { } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt index a693f49780..bdbdc33b65 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt @@ -43,7 +43,7 @@ Errors: Error 17-19: The specified column 'Field1' does not exist.|Error 8-15: T Errors: Error 8-15: The value passed to the 'Collect' function cannot be changed. >> Collect("", "") -Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed. +Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: The function 'Collect' has some invalid arguments.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed. >> IsBlank(Collect(t1, Blank())) true From 5f93aed7400e73edc7dfd5c4b76fa49b9b05deb9 Mon Sep 17 00:00:00 2001 From: Anderson Ferreira da Silva Date: Mon, 11 Mar 2024 00:04:39 -0500 Subject: [PATCH 14/29] Moving ISuggestionAwareFunction interface to Core. --- .../Functions/ISuggestionAwareFunction.cs | 14 ++++++++++++++ .../Texl/Builtins/Collect.cs | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/libraries/Microsoft.PowerFx.Core/Functions/ISuggestionAwareFunction.cs diff --git a/src/libraries/Microsoft.PowerFx.Core/Functions/ISuggestionAwareFunction.cs b/src/libraries/Microsoft.PowerFx.Core/Functions/ISuggestionAwareFunction.cs new file mode 100644 index 0000000000..5a7705c27c --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Core/Functions/ISuggestionAwareFunction.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.PowerFx.Core.Functions +{ + /// + /// Interface used for TexlFunctions to override the default behavior for intellisense suggestions. + /// + internal interface ISuggestionAwareFunction + { + // Returns true if it's valid to suggest ThisItem for this function as an argument. + bool CanSuggestThisItem { get; } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index b95764b6c1..944d66fdfd 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -23,7 +23,7 @@ namespace Microsoft.PowerFx.Core.Texl.Builtins { // Collect(collection:*[...], item1:![...]|*[...], ...) - internal class CollectFunction : BuiltinFunction + internal class CollectFunction : BuiltinFunction, ISuggestionAwareFunction { public override bool AffectsCollectionSchemas => true; From 026e9297c271fa7e2dabeee5a9a8fe8779b658f1 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Sun, 14 Apr 2024 23:33:32 -0500 Subject: [PATCH 15/29] Merging. --- .../Environment/PowerFxConfigExtensions.cs | 1 - .../Functions/LibraryMutation.cs | 1 + .../Functions/Mutation/CollectFunction.cs | 5 +- .../Functions/Mutation/MutationUtils.cs | 51 +++++++++++++++++++ .../ExpressionTestCases/Collect_V1Compat.txt | 10 ++-- .../Coalesce_CoalesceShortCircuit.txt | 1 + .../MutationScripts/Collect.txt | 4 +- 7 files changed, 63 insertions(+), 10 deletions(-) create mode 100644 src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs index 8d7742ef3c..9fe0995867 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs @@ -38,7 +38,6 @@ public static void EnableSetFunction(this PowerFxConfig powerFxConfig) public static void EnableMutationFunctions(this SymbolTable symbolTable) { symbolTable.AddFunction(new RecalcEngineSetFunction()); - symbolTable.AddFunction(new CollectFunction()); symbolTable.AddFunction(new PatchImpl()); symbolTable.AddFunction(new PatchSingleRecordImpl()); symbolTable.AddFunction(new PatchAggregateImpl()); diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index c673cf9c8a..3f31a7f040 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -13,6 +13,7 @@ using Microsoft.PowerFx.Functions; using Microsoft.PowerFx.Interpreter; using Microsoft.PowerFx.Types; +using MutationUtils = Microsoft.PowerFx.Interpreter.MutationUtils; namespace Microsoft.PowerFx.Core.Texl.Builtins { diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs index 03fb8e3265..563cb997ed 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs @@ -16,7 +16,8 @@ using Microsoft.PowerFx.Functions; using Microsoft.PowerFx.Syntax; using Microsoft.PowerFx.Types; -using static Microsoft.PowerFx.Core.Localization.TexlStrings; +using static Microsoft.PowerFx.Core.Localization.TexlStrings; +using CoreMutationUtils = Microsoft.PowerFx.Core.Utils.MutationUtils; namespace Microsoft.PowerFx.Interpreter { @@ -230,7 +231,7 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ int skip = 1; - MutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(skip).ToArray(), argTypes.Skip(skip).ToArray(), errors); + CoreMutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(skip).ToArray(), argTypes.Skip(skip).ToArray(), errors); } // This method returns true if there are special suggestions for a particular parameter of the function. diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs new file mode 100644 index 0000000000..b970442c79 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.PowerFx.Core.App.ErrorContainers; +using Microsoft.PowerFx.Core.Entities; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Localization; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Syntax; +using Microsoft.PowerFx.Types; + +namespace Microsoft.PowerFx.Interpreter +{ + internal class MutationUtils + { + /// + /// Merges all records into a single record. Collisions are resolved by last-one-wins. + /// + /// + /// + public static DValue MergeRecords(IEnumerable records) + { + var mergedFields = new Dictionary(); + + foreach (FormulaValue fv in records) + { + if (fv is ErrorValue errorValue) + { + return DValue.Of(errorValue); + } + + if (fv is BlankValue) + { + continue; + } + + if (fv is RecordValue recordValue) + { + foreach (var field in recordValue.Fields) + { + mergedFields[field.Name] = field.Value; + } + } + } + + return DValue.Of(FormulaValue.NewRecordFromFields(mergedFields.Select(kvp => new NamedValue(kvp.Key, kvp.Value)))); + } + } +} diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt index f6e449fd2e..8c2aec15c3 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt @@ -43,7 +43,7 @@ Errors: Error 17-19: The specified column 'Field1' does not exist.|Error 8-15: T Errors: Error 8-15: The value passed to the 'Collect' function cannot be changed. >> Collect("", "") -Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: The function 'Collect' has some invalid arguments.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed. +Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 8-10: The function 'Collect' has some invalid arguments.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed. >> IsBlank(Collect(t1, Blank())) true @@ -61,16 +61,16 @@ Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{F Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) >> Collect(t1, "x") -Errors: Error 0-16: The function 'Collect' has some invalid arguments.|Error 12-15: Invalid argument type. Cannot use Text values in this context. +Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 12-15: Invalid argument type. Cannot use Text values in this context. >> Collect(Foo,r2) -Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-14: The specified column 'Field1' does not exist.|Error 0-15: The function 'Collect' has some invalid arguments. +Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 0-7: The function 'Collect' has some invalid arguments.|Error 12-14: The specified column 'Field1' does not exist. >> Collect(Foo,Bar) -Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-15: Name isn't valid. 'Bar' isn't recognized.|Error 0-16: The function 'Collect' has some invalid arguments. +Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-15: Name isn't valid. 'Bar' isn't recognized.|Error 0-7: The function 'Collect' has some invalid arguments. >> Collect(1/0,Foo) Errors: Error 12-15: Name isn't valid. 'Foo' isn't recognized. >> Collect(t1,{Price:200}).Price -Errors: Error 0-23: The function 'Collect' has some invalid arguments.|Error 11-22: The specified column 'Price' does not exist. +Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 11-22: The specified column 'Price' does not exist.|Error 23-29: Deprecated use of '.'. Please use the 'ShowColumns' function instead. diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuit.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuit.txt index 84aac719ef..6390620666 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuit.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Coalesce_CoalesceShortCircuit.txt @@ -1,3 +1,4 @@ +#SETUP: PowerFxV1CompatibilityRules // Cases to test how shortcut verification work along with behavior functions >> Set( t1, [1,2] ) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt index 4a5222210e..f62eff269d 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt @@ -49,7 +49,7 @@ Blank() Table({Value:1},{Value:2},{Value:3}) >> Collect(temp1,{Value:"200"}).Value -Errors: Error 14-27: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 0-28: The function 'Collect' has some invalid arguments. +Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 14-27: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 28-34: Deprecated use of '.'. Please use the 'ShowColumns' function instead. >> Collect( temp1, { Value:"11"+0 } ) {Value:11} @@ -64,7 +64,7 @@ Errors: Error 16-29: The type of this argument 'Value' does not match the expect Table({Value:1},{Value:2},{Value:3}) >> Collect(temp1,{Value:"run time error"}).Value -Errors: Error 0-39: The function 'Collect' has some invalid arguments.|Error 14-38: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 39-45: Deprecated use of '.'. Please use the 'ShowColumns' function instead. +Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 14-38: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 39-45: Deprecated use of '.'. Please use the 'ShowColumns' function instead. >> Set(partialT1, Table({a:1,b:1},{a:2,b:2})) Table({a:1,b:1},{a:2,b:2}) From b427b2dae7642c7e2ec6cec4394d0e96689f7f31 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Mon, 15 Apr 2024 14:42:50 -0500 Subject: [PATCH 16/29] PR feedback. --- .../Texl/Builtins/Collect.cs | 4 +- .../Functions/LibraryMutation.cs | 25 +++-------- .../ParsedExpression.cs | 8 ---- .../Microsoft.PowerFx.Core.Tests/TexlTests.cs | 44 ++++++++++++++++--- 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 944d66fdfd..890bb7d413 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -114,7 +114,7 @@ public virtual DType GetCollectedType(Features features, DType argType) return argType; } - public bool TryGetUnifiedCollectedTypeCanvas(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType) + private bool TryGetUnifiedCollectedTypeCanvas(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType) { Contracts.AssertValue(args); Contracts.AssertAllValues(args); @@ -404,7 +404,7 @@ public static DType GetCollectedTypeForGivenArgType(Features features, DType arg return argType; } - // Passed a scalar; make a record out of it, using a name that depends on the type. + // Passed a scalar; make a record out of it. var fieldName = Contracts.VerifyValue(CreateInvariantFieldName(features, argType.Kind)); return DType.CreateRecord(new TypedName[] { new TypedName(argType, new DName(fieldName)) }); } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index 3f31a7f040..ca6eb93a12 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -232,34 +232,28 @@ public async Task InvokeAsync(FormulaType irContext, FormulaValue[ } } - internal class CollectImpl : CollectFunction, IAsyncTexlFunction5 + internal class CollectImpl : CollectFunction, IAsyncTexlFunction3 { - public async Task InvokeAsync(IServiceProvider runtimeServiceProvider, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) + public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { - return await new CollectProcess().Process(runtimeServiceProvider, irContext, args, cancellationToken).ConfigureAwait(false); + return await new CollectProcess().Process(irContext, args, cancellationToken).ConfigureAwait(false); } } - internal class CollectScalarImpl : CollectScalarFunction, IAsyncTexlFunction5 + internal class CollectScalarImpl : CollectScalarFunction, IAsyncTexlFunction3 { - public async Task InvokeAsync(IServiceProvider runtimeServiceProvider, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) + public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { - return await new CollectProcess().Process(runtimeServiceProvider, irContext, args, cancellationToken).ConfigureAwait(false); + return await new CollectProcess().Process(irContext, args, cancellationToken).ConfigureAwait(false); } } internal class CollectProcess { - internal async Task Process(IServiceProvider runtimeServiceProvider, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) + internal async Task Process(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { FormulaValue arg0; var argc = args.Length; - var features = runtimeServiceProvider.GetService(); - - if (!features.PowerFxV1CompatibilityRules) - { - throw new InvalidOperationException("Collect funtion can only be executed if PowerFx V1 feature is active."); - } // Need to check if the Lazy first argument has been evaluated since it may have already been // evaluated in the ClearCollect case. @@ -341,10 +335,5 @@ internal async Task Process(IServiceProvider runtimeServiceProvide return CompileTimeTypeWrapperRecordValue.AdjustType(tableValue.Type.ToRecord(), (RecordValue)resultRows.First().ToFormulaValue()); } } - - private RecordValue CreateRecordFromPrimitive(TableValue tableValue, FormulaValue arg) - { - return FormulaValue.NewRecordFromFields(tableValue.Type.ToRecord(), new NamedValue("Value", arg)); - } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs b/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs index 5409eb8379..1e31dcb8cb 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs @@ -87,7 +87,6 @@ internal static IExpressionEvaluator GetEvaluator(this CheckResult result, Stack _allSymbols = result.Symbols, _parameterSymbolTable = result.Parameters, _additionalFunctions = result.Engine.Config.AdditionalFunctions, - _features = result.Engine.Config.Features }; return expr; @@ -105,7 +104,6 @@ internal class ParsedExpression : IExpressionEvaluator internal ReadOnlySymbolTable _allSymbols; internal ReadOnlySymbolTable _parameterSymbolTable; internal IReadOnlyDictionary _additionalFunctions; - internal Features _features; internal ParsedExpression(IntermediateNode irnode, ScopeSymbol topScope, StackDepthCounter stackMarker, CultureInfo cultureInfo = null) { @@ -135,12 +133,6 @@ public async Task EvalAsync(CancellationToken cancellationToken, I hasInnerServices = true; } - if (_features != null) - { - innerServices.AddService(_features); - hasInnerServices = true; - } - RuntimeConfig runtimeConfig2 = new RuntimeConfig { Values = symbolValues, diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/TexlTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests/TexlTests.cs index 92e9e3fe70..d6b6ea6294 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/TexlTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests/TexlTests.cs @@ -5,25 +5,19 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Reflection.Metadata; using Microsoft.CodeAnalysis; -using Microsoft.PowerFx.Core.App.Controls; -using Microsoft.PowerFx.Core.Binding; -using Microsoft.PowerFx.Core.Binding.BindInfo; using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.Functions.Delegation; using Microsoft.PowerFx.Core.Functions.Delegation.DelegationMetadata; -using Microsoft.PowerFx.Core.Glue; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Parser; using Microsoft.PowerFx.Core.Tests.Helpers; using Microsoft.PowerFx.Core.Texl; using Microsoft.PowerFx.Core.Texl.Builtins; -using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Syntax; -using Microsoft.PowerFx.Tests; using Microsoft.PowerFx.Types; using Xunit; @@ -134,6 +128,42 @@ public void TexlDateTableFunctions_Float(string expression, string expectedType) Assert.True(result.IsSuccess); } + [Theory] + + // Collect + [InlineData("Collect(t1, {a:1})", "![a:w, b:s]")] + [InlineData("Collect(t1, {a:1},{a:2})", "*[a:w, b:s]")] + [InlineData("Collect(t1, Table({a:1}))", "*[a:w, b:s]")] + [InlineData("Collect(t2, 2)", "![Value:w]")] + [InlineData("Collect(t2, 2, 3, 4)", "*[Value:w]")] + [InlineData("Collect(t1, {a:1})", "*[a:w, b:s]", false)] + [InlineData("Collect(t1, {a:1},{a:2})", "*[a:w, b:s]", false)] + [InlineData("Collect(t1, Table({a:1}))", "*[a:w, b:s]", false)] + [InlineData("Collect(t2, 2)", "*[Value:w]", false)] + [InlineData("Collect(t2, 2, 3, 4)", "*[Value:w]", false)] + public void TexlMutationFunctionsV1Tests(string expression, string expectedType, bool isPFxV1 = true) + { + var engine = new Engine(new PowerFxConfig(isPFxV1 ? Features.PowerFxV1 : Features.None)); + var options = new ParserOptions() { AllowsSideEffects = true }; + + DType.TryParse("*[a:w,b:s]", out var expectedDType); + DType.TryParse("*[Value:w]", out var expectedDTypeScalar); + + engine.Config.SymbolTable.AddFunction(new CollectFunction()); + engine.Config.SymbolTable.AddFunction(new CollectScalarFunction()); + engine.Config.SymbolTable.AddFunction(new PatchFunction()); + engine.Config.SymbolTable.AddFunction(new PatchSingleRecordFunction()); + engine.Config.SymbolTable.AddFunction(new PatchAggregateFunction()); + engine.Config.SymbolTable.AddFunction(new PatchAggregateSingleTableFunction()); + engine.Config.SymbolTable.AddVariable("t1", FormulaType.Build(expectedDType), mutable: true); + engine.Config.SymbolTable.AddVariable("t2", FormulaType.Build(expectedDTypeScalar), mutable: true); + + var check = engine.Check(expression, options); + + Assert.True(check.IsSuccess); + Assert.Equal(expectedType, check.Binding.ResultType.ToString()); + } + [Theory] [InlineData("DateDiff([Date(2000,1,1)],[Date(2001,1,1)],\"years\")", "*[Result:w]")] [InlineData("DateDiff(Date(2000,1,1),[Date(2001,1,1)],\"years\")", "*[Result:w]")] From 925051cbbbee1e489dc0a1b8ed79cf96a9610405 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Mon, 15 Apr 2024 14:47:16 -0500 Subject: [PATCH 17/29] Conflict. --- src/Directory.Build.props | 1 - src/strings/PowerFxResources.en-US.resx | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Directory.Build.props b/src/Directory.Build.props index b82b021623..454f024aa3 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -9,7 +9,6 @@ - diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index c7973279ad..e3b806ad74 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -3121,6 +3121,10 @@ Adds one or more items to the specified 'collection'. The items can be from a different table or collection (e.g. Collect(collection, source_collection)), or one or more records Collect(collection, {key1: val1, key2: val2, ...}, ...). Description of 'Collect' function. + + data_source + function_parameter - First parameter for the Collect function. The data source that you want to add data to. Translate this string. When translating, maintain as a single word (i.e., do not add spaces). + collection function_parameter - First parameter for the Collect function - the name of the collection that will be created / appended to. From 0351bb2fd32dc059464833fc804115ea71b3bc90 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Thu, 18 Apr 2024 09:37:12 -0500 Subject: [PATCH 18/29] Conflict merge. --- .../ExpressionTestCases/Collect_V1Compat.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt index 6925b10ca0..4732b709f7 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt @@ -43,7 +43,7 @@ Errors: Error 17-19: The specified column 'Field1' does not exist.|Error 0-7: Th Errors: Error 8-15: The value passed to the 'Collect' function cannot be changed. >> Collect("", "") -Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 8-10: The function 'Collect' has some invalid arguments.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed. +Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 8-10: The function 'Collect' has some invalid arguments.|Error 12-14: Invalid argument type. Cannot use Text values in this context. >> IsBlank(Collect(t1, Blank())) true From 7b215341125b377c264bc35dafc8132cb3fb144c Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Thu, 9 May 2024 11:55:39 -0500 Subject: [PATCH 19/29] Merging changes. --- src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 890bb7d413..94b956b278 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -73,7 +73,7 @@ public override bool ArgMatchesDatasourceType(int argNum) return argNum >= 1; } - public override bool IsLazyEvalParam(int index, Features features) + public override bool IsLazyEvalParam(TexlNode node, int index, Features features) { // First argument to mutation functions is Lazy for datasources that are copy-on-write. // If there are any side effects in the arguments, we want those to have taken place before we make the copy. From bde1d849ff444a6cd2232be4daefdf5093de5f2b Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Tue, 21 May 2024 22:35:58 -0500 Subject: [PATCH 20/29] Implementing ClearCollect/ClearCollectScalar. --- .../Localization/Strings.cs | 2 - .../Texl/Builtins/Collect.cs | 62 +++- .../Utils/MutationUtils.cs | 30 ++ .../Environment/PowerFxConfigExtensions.cs | 3 +- .../Functions/LibraryMutation.cs | 36 +- .../Mutation/ClearCollectFunction.cs | 52 --- .../Functions/Mutation/CollectFunction.cs | 310 ------------------ src/strings/PowerFxResources.en-US.resx | 18 +- .../MutationScripts/ClearCollect.txt | 32 +- 9 files changed, 139 insertions(+), 406 deletions(-) delete mode 100644 src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/ClearCollectFunction.cs delete mode 100644 src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs index 437562bcc0..65725aa31c 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs @@ -506,8 +506,6 @@ internal static class TexlStrings public static StringGetter AboutCollect_item = (b) => StringResources.Get("AboutCollect_item", b); public static StringGetter AboutClearCollect = (b) => StringResources.Get("AboutClearCollect", b); - public static StringGetter ClearCollectDataSourceArg = (b) => StringResources.Get("ClearCollectDataSourceArg", b); - public static StringGetter ClearCollectRecordArg = (b) => StringResources.Get("ClearCollectRecordArg", b); public static StringGetter AboutRemove = (b) => StringResources.Get("AboutRemove", b); public static StringGetter RemoveDataSourceArg = (b) => StringResources.Get("RemoveDataSourceArg", b); diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 94b956b278..8b4d61331b 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -438,27 +438,53 @@ public override DType GetCollectedType(Features features, DType argType) internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) { - var newArgs = new List() { args[0] }; + return base.CreateIRCallNode(node, context, MutationUtils.CreateIRCallNodeCollect(node, context, args, scope), scope); + } + } + + // ClearCollect(collection:*[...], item1:![...]|*[...], ...) + internal class ClearCollectFunction : CollectFunction + { + public override bool AllowedWithinNondeterministicOperationOrder => false; + + public ClearCollectFunction() + : base("ClearCollect", TexlStrings.AboutClearCollect) + { + } + + public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors) + { + Contracts.AssertValue(args); + Contracts.AssertAllValues(args); + Contracts.AssertValue(argTypes); + Contracts.Assert(args.Length == argTypes.Length); + Contracts.AssertValue(errors); + Contracts.Assert(MinArity <= args.Length && args.Length <= MaxArity); + + base.CheckSemantics(binding, args, argTypes, errors); - foreach (var arg in args.Skip(1)) + if (binding.EntityScope != null && + binding.EntityScope.TryGetDataSource(args[0], out IExternalDataSource dataSourceInfo) && + !dataSourceInfo.IsClearable) { - if (arg.IRContext.ResultType._type.IsPrimitive) - { - newArgs.Add( - new RecordNode( - new IRContext(arg.IRContext.SourceContext, RecordType.Empty().Add(TableValue.ValueName, arg.IRContext.ResultType)), - new Dictionary - { - { TableValue.ValueDName, arg } - })); - } - else - { - newArgs.Add(arg); - } + errors.EnsureError(args[0], TexlStrings.ErrInvalidDataSourceForFunction); } + } + } - return base.CreateIRCallNode(node, context, newArgs, scope); - } + // ClearCollect(collection:*[...], item1, ...) + internal class ClearCollectScalarFunction : ClearCollectFunction + { + public override bool IsScalar => true; + + public override DType GetCollectedType(PowerFx.Features features, DType argType) + { + return GetCollectedTypeForGivenArgType(features, argType); + } + + internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) + { + return base.CreateIRCallNode(node, context, MutationUtils.CreateIRCallNodeCollect(node, context, args, scope), scope); + } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs index e37f3713b5..a476a41319 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs @@ -8,10 +8,15 @@ using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Core.IR.Nodes; +using Microsoft.PowerFx.Core.IR.Symbols; using Microsoft.PowerFx.Core.Localization; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Syntax; using Microsoft.PowerFx.Types; +using CallNode = Microsoft.PowerFx.Syntax.CallNode; +using RecordNode = Microsoft.PowerFx.Core.IR.Nodes.RecordNode; namespace Microsoft.PowerFx.Core.Utils { @@ -79,5 +84,30 @@ DKind.Blob or _ => TableValue.ValueName }; } + + public static List CreateIRCallNodeCollect(CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) + { + var newArgs = new List() { args[0] }; + + foreach (var arg in args.Skip(1)) + { + if (arg.IRContext.ResultType._type.IsPrimitive) + { + newArgs.Add( + new RecordNode( + new IRContext(arg.IRContext.SourceContext, RecordType.Empty().Add(TableValue.ValueName, arg.IRContext.ResultType)), + new Dictionary + { + { TableValue.ValueDName, arg } + })); + } + else + { + newArgs.Add(arg); + } + } + + return newArgs; + } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs index 59ec93422e..b307af9826 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs @@ -44,7 +44,8 @@ public static void EnableMutationFunctions(this SymbolTable symbolTable) symbolTable.AddFunction(new PatchAggregateSingleTableImpl()); symbolTable.AddFunction(new RemoveFunction()); symbolTable.AddFunction(new ClearImpl()); - symbolTable.AddFunction(new ClearCollectFunction()); + symbolTable.AddFunction(new ClearCollectImpl()); + symbolTable.AddFunction(new ClearCollectScalarImpl()); symbolTable.AddFunction(new CollectImpl()); symbolTable.AddFunction(new CollectScalarImpl()); } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index 92e80d0dec..fc3242f536 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -337,7 +337,7 @@ internal async Task Process(FormulaType irContext, FormulaValue[] } } - // Clear(collection_or_table) + // Clear(collection_or_table) internal class ClearImpl : ClearFunction, IAsyncTexlFunction3 { public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) @@ -365,4 +365,38 @@ public async Task InvokeAsync(FormulaType irContext, FormulaValue[ } } } + + // ClearCollect(table_or_collection, table|record, ...) + internal class ClearCollectImpl : ClearCollectFunction, IAsyncTexlFunction3 + { + public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) + { + if (args[0] is LambdaFormulaValue arg0lazy) + { + args[0] = await arg0lazy.EvalAsync().ConfigureAwait(false); + } + + var clearFunction = new ClearImpl(); + + var cleared = await clearFunction.InvokeAsync(FormulaType.Void, args, cancellationToken).ConfigureAwait(false); + + if (cleared is ErrorValue) + { + return cleared; + } + + var collectFunction = new CollectImpl(); + + return await collectFunction.InvokeAsync(irContext, args, cancellationToken).ConfigureAwait(false); + } + } + + // ClearCollect(table_or_collection, scalar, ...) + internal class ClearCollectScalarImpl : ClearCollectScalarFunction, IAsyncTexlFunction3 + { + public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) + { + return await new ClearCollectImpl().InvokeAsync(irContext, args, cancellationToken).ConfigureAwait(false); + } + } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/ClearCollectFunction.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/ClearCollectFunction.cs deleted file mode 100644 index d26575112a..0000000000 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/ClearCollectFunction.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.PowerFx.Core.Functions; -using Microsoft.PowerFx.Core.Localization; -using Microsoft.PowerFx.Core.Texl.Builtins; -using Microsoft.PowerFx.Interpreter; -using Microsoft.PowerFx.Types; -using static Microsoft.PowerFx.Core.Localization.TexlStrings; - -namespace Microsoft.PowerFx.Functions -{ - /// - /// The ClearCollect function is a combination of Clear + Collect. - /// - internal class ClearCollectFunction : CollectFunction, IAsyncTexlFunction - { - public override bool IsSelfContained => false; - - public ClearCollectFunction() - : base("ClearCollect", TexlStrings.AboutClearCollect) - { - } - - public override IEnumerable GetSignatures() - { - yield return new[] { TexlStrings.ClearCollectDataSourceArg, TexlStrings.ClearCollectRecordArg }; - } - - public override async Task InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken) - { - if (args[0] is LambdaFormulaValue arg0lazy) - { - args[0] = await arg0lazy.EvalAsync().ConfigureAwait(false); - } - - var clearFunction = new ClearImpl(); - - var cleared = await clearFunction.InvokeAsync(FormulaType.Void, args, cancellationToken).ConfigureAwait(false); - - if (cleared is ErrorValue) - { - return cleared; - } - - return await base.InvokeAsync(args, cancellationToken).ConfigureAwait(false); - } - } -} diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs deleted file mode 100644 index cbcc66be19..0000000000 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/CollectFunction.cs +++ /dev/null @@ -1,310 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.PowerFx.Core.App.ErrorContainers; -using Microsoft.PowerFx.Core.Binding; -using Microsoft.PowerFx.Core.Errors; -using Microsoft.PowerFx.Core.Functions; -using Microsoft.PowerFx.Core.IR; -using Microsoft.PowerFx.Core.Localization; -using Microsoft.PowerFx.Core.Types; -using Microsoft.PowerFx.Core.Utils; -using Microsoft.PowerFx.Functions; -using Microsoft.PowerFx.Syntax; -using Microsoft.PowerFx.Types; -using static Microsoft.PowerFx.Core.Localization.TexlStrings; -using CoreMutationUtils = Microsoft.PowerFx.Core.Utils.MutationUtils; - -namespace Microsoft.PowerFx.Interpreter -{ - // The CollectFunction class was copied from PowerApss. - // Implementation of a Set function which just chains to - // RecalcEngine.UpdateVariable(). - // Set has no return value. - // Whereas PowerApps' Set() will implicitly define arg0, - // this Set() requires arg0 was already defined and has a type. - // - // Called as: - // Set(var,newValue) - internal class CollectFunction : BuiltinFunction, IAsyncTexlFunction - { - public override bool ManipulatesCollections => true; - - public override bool ModifiesValues => true; - - public override bool IsSelfContained => false; - - public override bool RequiresDataSourceScope => true; - - protected virtual bool IsScalar => false; - - public override bool CanSuggestInputColumns => true; - - public override bool TryGetTypeForArgSuggestionAt(int argIndex, out DType type) - { - if (argIndex == 1) - { - type = default; - return false; - } - - return base.TryGetTypeForArgSuggestionAt(argIndex, out type); - } - - public override bool ArgMatchesDatasourceType(int argNum) - { - return argNum >= 1; - } - - public override bool MutatesArg0 => true; - - public override bool IsLazyEvalParam(TexlNode node, int index, Features features) - { - // First argument to mutation functions is Lazy for datasources that are copy-on-write. - // If there are any side effects in the arguments, we want those to have taken place before we make the copy. - return index == 0; - } - - /// - /// Initializes a new instance of the class. - /// To be consumed by ClearCollect function. - /// - protected CollectFunction(string name, TexlStrings.StringGetter description) - : base(name, description, FunctionCategories.Behavior, DType.EmptyRecord, 0, 2, 2, DType.EmptyTable, DType.EmptyRecord) - { - } - - public CollectFunction() - : base( - "Collect", - TexlStrings.AboutCollect, - FunctionCategories.Behavior, - DType.EmptyRecord, - 0, - 2, - 2, // Not handling multiple arguments for now - DType.EmptyTable, - DType.EmptyRecord) - { - } - - public override IEnumerable GetSignatures() - { - yield return new[] { TexlStrings.CollectArg1, TexlStrings.CollectArg2 }; - } - - public virtual DType GetCollectedType(DType argType) - { - Contracts.Assert(argType.IsValid); - - return argType; - } - - // Attempt to get the unified schema of the items being collected by an invocation. - private bool TryGetUnifiedCollectedType(CheckTypesContext context, TexlNode[] args, DType[] argTypes, IErrorContainer errors, out DType collectedType) - { - Contracts.AssertValue(args); - Contracts.AssertAllValues(args); - Contracts.AssertValue(argTypes); - Contracts.Assert(args.Length == argTypes.Length); - Contracts.AssertValue(errors); - Contracts.Assert(MinArity <= args.Length && args.Length <= MaxArity); - - var fValid = true; - DType itemType = DType.Invalid; - - var argc = args.Length; - - for (var i = 1; i < argc; i++) - { - DType argType = GetCollectedType(argTypes[i]); - - // The subsequent args should all be aggregates. - if (!argType.IsAggregate) - { - errors.EnsureError(args[i], TexlStrings.ErrBadType_Type, argType.GetKindString()); - fValid = false; - continue; - } - - // Promote the arg type to a table to facilitate unioning. - if (!argType.IsRecord) - { - argType = argType.ToRecord(); - } - - // Checks if all record names exist against table type and if its possible to coerce. - bool checkAggregateNames = argType.CheckAggregateNames(argTypes[0], args[i], errors, context.Features, SupportsParamCoercion); - fValid = fValid && checkAggregateNames; - - if (!itemType.IsValid) - { - itemType = argType; - } - else - { - var fUnionError = false; - itemType = DType.Union(ref fUnionError, itemType, argType, useLegacyDateTimeAccepts: true, context.Features); - if (fUnionError) - { - errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrIncompatibleTypes); - fValid = false; - } - } - - // We only support accessing entities in collections if the collection has only 1 argument that contributes to it's type - if (argc != 2 && itemType.ContainsDataEntityType(DPath.Root)) - { - fValid &= DropAllOfKindNested(ref itemType, errors, args[i], DKind.DataEntity); - } - } - - Contracts.Assert(!itemType.IsValid || itemType.IsRecord); - collectedType = itemType.IsValid ? itemType : DType.EmptyRecord; - return fValid; - } - - // Typecheck an invocation of Collect. - public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DType[] argTypes, IErrorContainer errors, out DType returnType, out Dictionary nodeToCoercedTypeMap) - { - Contracts.AssertValue(args); - Contracts.AssertAllValues(args); - Contracts.AssertValue(argTypes); - Contracts.Assert(args.Length == argTypes.Length); - Contracts.AssertValue(errors); - Contracts.Assert(MinArity <= args.Length && args.Length <= MaxArity); - - var fValid = base.CheckTypes(context, args, argTypes, errors, out returnType, out nodeToCoercedTypeMap); - - // Need a collection for the 1st arg - DType collectionType = argTypes[0]; - if (!collectionType.IsTable) - { - errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrInvalidArgs_Func, Name); - fValid = false; - } - - // Get the unified collected type on the RHS. This will generate appropriate - // document errors for invalid arguments such as unsupported aggregate types. - fValid &= TryGetUnifiedCollectedType(context, args, argTypes, errors, out DType collectedType); - Contracts.Assert(collectedType.IsRecord); - - if (fValid) - { - if (!collectedType.TryGetCoercionSubType(collectionType, out DType coercionType, out var coercionNeeded, context.Features)) - { - fValid = false; - } - else - { - if (coercionNeeded) - { - CollectionUtils.Add(ref nodeToCoercedTypeMap, args[1], coercionType); - } - - var fError = false; - - returnType = DType.Union(ref fError, collectionType.ToRecord(), collectedType, useLegacyDateTimeAccepts: false, context.Features, allowCoerce: true); - - if (fError) - { - fValid = false; - if (!SetErrorForMismatchedColumns(collectionType, collectedType, args[1], errors, context.Features)) - { - errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrTableDoesNotAcceptThisType); - } - } - } - } - - return fValid; - } - - public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors) - { - base.CheckSemantics(binding, args, argTypes, errors); - base.ValidateArgumentIsMutable(binding, args[0], errors); - - int skip = 1; - - CoreMutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(skip).ToArray(), argTypes.Skip(skip).ToArray(), errors); - } - - // This method returns true if there are special suggestions for a particular parameter of the function. - public override bool HasSuggestionsForParam(int argumentIndex) - { - Contracts.Assert(argumentIndex >= 0); - - return argumentIndex == 0; - } - - public override bool IsAsyncInvocation(CallNode callNode, TexlBinding binding) - { - Contracts.AssertValue(callNode); - Contracts.AssertValue(binding); - - return Arg0RequiresAsync(callNode, binding); - } - - public virtual async Task InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken) - { - FormulaValue arg0; - - // Need to check if the Lazy first argument has been evaluated since it may have already been - // evaluated in the ClearCollect case. - if (args[0] is LambdaFormulaValue arg0lazy) - { - arg0 = await arg0lazy.EvalAsync().ConfigureAwait(false); - } - else - { - arg0 = args[0]; - } - - var arg1 = args[1]; - - // PA returns arg0. - // PFx returns arg1 for now except when arg0 is anything but TableValue, return arg0 or RuntimeTypeMismatch error. - if (arg0 is BlankValue) - { - return arg0; - } - else if (arg0 is ErrorValue) - { - return arg0; - } - - if (arg0 is not TableValue) - { - return CommonErrors.RuntimeTypeMismatch(IRContext.NotInSource(arg0.Type)); - } - - // If arg0 is valid, then return arg1. - if (arg1 is BlankValue) - { - return arg1; - } - else if (arg1 is ErrorValue) - { - return arg1; - } - - if (arg1 is not RecordValue) - { - return CommonErrors.RuntimeTypeMismatch(IRContext.NotInSource(arg1.Type)); - } - - var tableValue = arg0 as TableValue; - var recordValueToAppend = (RecordValue)arg1.MaybeShallowCopy(); - - cancellationToken.ThrowIfCancellationRequested(); - var result = await tableValue.AppendAsync(recordValueToAppend, cancellationToken).ConfigureAwait(false); - - return result.ToFormulaValue(); - } - } -} diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index bcc6bb1dcd..b241f07e2c 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -3168,20 +3168,14 @@ A record or table to collect. A record will be appended to the collection. A table will have its rows appended to the collection. - Deletes all the records from a collection. And then adds a different set of records to the same collection. - Description of 'Clear' function. - - - data_source - function_parameter - First parameter for the Clear function. The data source that you want to clear from all elements. Translate this string. When translating, maintain as a single word (i.e., do not add spaces). + Clears the collection first and adds one or more items to the specified 'collection'. The items can be from a different table or collection (e.g. ClearCollect(collection, source_collection)), or one or more records ClearCollect(collection, {key1: val1, key2: val2, ...}, ...). + Description of 'ClearCollect' function. - - data_source - function_parameter - First parameter for the ClearCollect function. The data source that contains the records that you want to modify or will contain the records that you want to create. Translate this string. When translating, maintain as a single word (i.e., do not add spaces). + + A new or existing collection to augment. - - record - function_parameter - Second parameter for the ClearCollect function. The record to be added. + + A record or table to collect. A record will be appended to the collection. A table will have its rows appended to the collection. Applies the updates to the specified row and changes the row in the specified datasource. Returns the updated row. diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ClearCollect.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ClearCollect.txt index 32b1908906..d023a28026 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ClearCollect.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/ClearCollect.txt @@ -32,25 +32,25 @@ Table({Value:1},{Value:2},{Value:3}) Table({Value:88}) >> ClearCollect(t1) -Errors: Error 0-16: Invalid number of arguments: received 1, expected 2. +Errors: Error 0-16: Invalid number of arguments: received 1, expected 2 or more. >> ClearCollect(t1, r1, r1) -Errors: Error 0-24: Invalid number of arguments: received 3, expected 2. +Table({a:99},{a:99}) >> ClearCollect(t1, r1, 1; 2; r1) -Errors: Error 0-30: Invalid number of arguments: received 3, expected 2. +Table({a:99},{a:99}) >> ClearCollect(t1, "x") -Errors: Error 17-20: Invalid argument type (Text). Expecting a Record value instead.|Error 17-20: Invalid argument type. Cannot use Text values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. +Errors: Error 17-20: Invalid argument type. Cannot use Text values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. >> ClearCollect(t1, 1) -Errors: Error 17-18: Invalid argument type (Decimal). Expecting a Record value instead.|Error 17-18: Invalid argument type. Cannot use Decimal values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. +Errors: Error 17-18: Invalid argument type. Cannot use Decimal values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. >> ClearCollect(t1, Float(1)) -Errors: Error 17-25: Invalid argument type (Number). Expecting a Record value instead.|Error 17-25: Invalid argument type. Cannot use Number values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. +Errors: Error 17-25: Invalid argument type. Cannot use Number values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. >> ClearCollect(t1, true) -Errors: Error 17-21: Invalid argument type (Boolean). Expecting a Record value instead.|Error 17-21: Invalid argument type. Cannot use Boolean values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. +Errors: Error 17-21: Invalid argument type. Cannot use Boolean values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. >> ClearCollect(Table({name: "VC"}), {surname: "textInput1"}) Errors: Error 34-57: The specified column 'surname' does not exist. The column with the most similar name is 'name'.|Error 0-12: The function 'ClearCollect' has some invalid arguments. @@ -68,10 +68,10 @@ Errors: Error 13-16: Name isn't valid. 'Foo' isn't recognized.|Error 17-19: The Errors: Error 13-16: Name isn't valid. 'Foo' isn't recognized.|Error 17-20: Name isn't valid. 'Bar' isn't recognized.|Error 0-12: The function 'ClearCollect' has some invalid arguments. >> ClearCollect(1/0,Foo) -Errors: Error 17-20: Name isn't valid. 'Foo' isn't recognized.|Error 14-15: Invalid argument type (Decimal). Expecting a Table value instead.|Error 0-12: The function 'ClearCollect' has some invalid arguments. +Errors: Error 17-20: Name isn't valid. 'Foo' isn't recognized.|Error 14-15: The function 'ClearCollect' has some invalid arguments.|Error 0-12: The function 'ClearCollect' has some invalid arguments. >> ClearCollect(Float(1)/0,Foo) -Errors: Error 24-27: Name isn't valid. 'Foo' isn't recognized.|Error 21-22: Invalid argument type (Number). Expecting a Table value instead.|Error 0-12: The function 'ClearCollect' has some invalid arguments. +Errors: Error 24-27: Name isn't valid. 'Foo' isn't recognized.|Error 21-22: The function 'ClearCollect' has some invalid arguments.|Error 0-12: The function 'ClearCollect' has some invalid arguments. >> ClearCollect(Error({Kind:ErrorKind.Custom}), r1) Errors: Error 45-47: The specified column 'a' does not exist.|Error 0-12: The function 'ClearCollect' has some invalid arguments. @@ -86,7 +86,19 @@ Errors: Error 22-24: The specified column 'a' does not exist.|Error 0-12: The fu Errors: Error 13-20: The value passed to the 'ClearCollect' function cannot be changed. >> ClearCollect("", "") -Errors: Error 13-15: Invalid argument type (Text). Expecting a Table value instead.|Error 17-19: Invalid argument type (Text). Expecting a Record value instead.|Error 17-19: Invalid argument type. Cannot use Text values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. +Errors: Error 13-15: The function 'ClearCollect' has some invalid arguments.|Error 17-19: Invalid argument type. Cannot use Text values in this context.|Error 0-12: The function 'ClearCollect' has some invalid arguments. >> ClearCollect(If(false,Blank()),r1) Errors: Error 31-33: The specified column 'a' does not exist.|Error 0-12: The function 'ClearCollect' has some invalid arguments. + +>> Set(t2, [1,2,3]) +Table({Value:1},{Value:2},{Value:3}) + +>> ClearCollect(t2, 99) +{Value:99} + +>> 0;t2 +Table({Value:99}) + +>> ClearCollect(t2, 8,88,888) +Table({Value:8},{Value:88},{Value:888}) \ No newline at end of file From 5a04764a1a26a95b4b9ede275b1cc96613ebad33 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Fri, 31 May 2024 12:29:58 -0500 Subject: [PATCH 21/29] Fixing Collect return type. --- src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs | 4 ++-- .../ExpressionTestCases/Collect_V1Compat.txt | 2 +- .../MutationScripts/Collect.txt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 8b4d61331b..ed1fa39fc8 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -284,8 +284,8 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrTableDoesNotAcceptThisType); } } - - if (fValid) + + if (returnType.IsAggregate) { if (context.Features.PowerFxV1CompatibilityRules && argTypes.Length == 2 && (argTypes[1].IsRecord || argTypes[1].IsPrimitive)) { diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Collect_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Collect_V1Compat.txt index 4732b709f7..dd086987ca 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Collect_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Collect_V1Compat.txt @@ -73,4 +73,4 @@ Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-15: Name Errors: Error 12-15: Name isn't valid. 'Foo' isn't recognized. >> Collect(t1,{Price:200}).Price -Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 11-22: The specified column 'Price' does not exist.|Error 23-29: Deprecated use of '.'. Please use the 'ShowColumns' function instead. \ No newline at end of file +Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 11-22: The specified column 'Price' does not exist. \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/Collect.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/Collect.txt index f62eff269d..0aa2a38fad 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/Collect.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/Collect.txt @@ -49,7 +49,7 @@ Blank() Table({Value:1},{Value:2},{Value:3}) >> Collect(temp1,{Value:"200"}).Value -Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 14-27: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 28-34: Deprecated use of '.'. Please use the 'ShowColumns' function instead. +Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 14-27: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'. >> Collect( temp1, { Value:"11"+0 } ) {Value:11} @@ -64,7 +64,7 @@ Errors: Error 16-29: The type of this argument 'Value' does not match the expect Table({Value:1},{Value:2},{Value:3}) >> Collect(temp1,{Value:"run time error"}).Value -Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 14-38: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 39-45: Deprecated use of '.'. Please use the 'ShowColumns' function instead. +Errors: Error 0-7: The function 'Collect' has some invalid arguments.|Error 14-38: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'. >> Set(partialT1, Table({a:1,b:1},{a:2,b:2})) Table({a:1,b:1},{a:2,b:2}) From c9a72b4865b283e4519c76481a6b177ed68e6cc9 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Tue, 4 Jun 2024 17:38:46 -0500 Subject: [PATCH 22/29] Fixes. --- .../Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs | 8 ++++++++ .../Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs | 3 ++- .../MutationFunctionsTests.cs | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs b/src/libraries/Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs index baf98c1ce2..5d0fb7af38 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs @@ -22,6 +22,8 @@ internal class CheckTypesContext public bool NumberIsFloat { get; } + public bool AnalysisMode { get; } + public CheckTypesContext(Features features, INameResolver nameResolver, string entityName, string propertyName, bool allowsSideEffects, bool numberIsFloat) { Features = features; @@ -31,5 +33,11 @@ public CheckTypesContext(Features features, INameResolver nameResolver, string e AllowsSideEffects = allowsSideEffects; NumberIsFloat = numberIsFloat; } + + public CheckTypesContext(Features features, INameResolver nameResolver, string entityName, string propertyName, bool allowsSideEffects, bool numberIsFloat, bool analysisMode) + : this(features, nameResolver, entityName, propertyName, allowsSideEffects, numberIsFloat) + { + AnalysisMode = analysisMode; + } } } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs index 055c110bce..ab22d08a83 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs @@ -1,10 +1,11 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Threading.Tasks; using Microsoft.CodeAnalysis; using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Functions; diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationFunctionsTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationFunctionsTests.cs index f4140b27c0..40d9f0969a 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationFunctionsTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationFunctionsTests.cs @@ -412,7 +412,7 @@ public void CollectPFxV1Disabled(string expression) var evaluator = check.GetEvaluator(); // Runtime exception - Assert.ThrowsAsync(async () => await evaluator.EvalAsync(CancellationToken.None).ConfigureAwait(false)).ConfigureAwait(false); + Assert.ThrowsAsync(async () => await evaluator.EvalAsync(CancellationToken.None).ConfigureAwait(false)).ConfigureAwait(true); } [Theory] From d9b236b62a92a846d91f86f98b3954680a99a525 Mon Sep 17 00:00:00 2001 From: Anderson Ferreira da Silva Date: Wed, 5 Jun 2024 15:00:11 -0500 Subject: [PATCH 23/29] Adding AnalysisMode flag. --- src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs | 3 ++- .../Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs | 1 + src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs b/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs index 30585e1ce6..688ba287e0 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs @@ -346,7 +346,8 @@ private TexlBinding( entityName: EntityName, propertyName: Property?.InvariantName ?? string.Empty, allowsSideEffects: bindingConfig.AllowsSideEffects, - numberIsFloat: bindingConfig.NumberIsFloat); + numberIsFloat: bindingConfig.NumberIsFloat, + analysisMode: false); } /// diff --git a/src/libraries/Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs b/src/libraries/Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs index 5d0fb7af38..0f74254b41 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Functions/CheckTypesContext.cs @@ -32,6 +32,7 @@ public CheckTypesContext(Features features, INameResolver nameResolver, string e PropertyName = propertyName; AllowsSideEffects = allowsSideEffects; NumberIsFloat = numberIsFloat; + AnalysisMode = true; } public CheckTypesContext(Features features, INameResolver nameResolver, string entityName, string propertyName, bool allowsSideEffects, bool numberIsFloat, bool analysisMode) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index ed1fa39fc8..10ae39da61 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -263,7 +263,7 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp // Get the unified collected type on the RHS. This will generate appropriate // document errors for invalid arguments such as unsupported aggregate types. - if (context.Features.PowerFxV1CompatibilityRules) + if (context.Features.PowerFxV1CompatibilityRules && !context.AnalysisMode) { fValid &= TryGetUnifiedCollectedTypeV1(args, argTypes, errors, context.Features, out collectedType); } From 1e1872d3af3dd1d6dfcb78a2abc6fbd31c419833 Mon Sep 17 00:00:00 2001 From: Anderson Ferreira da Silva Date: Thu, 6 Jun 2024 11:11:32 -0500 Subject: [PATCH 24/29] Adding AnalysisMode to binder config --- src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs | 2 +- .../Microsoft.PowerFx.Core/Binding/BindingConfig.cs | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs b/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs index 688ba287e0..20ef07d1d6 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs @@ -347,7 +347,7 @@ private TexlBinding( propertyName: Property?.InvariantName ?? string.Empty, allowsSideEffects: bindingConfig.AllowsSideEffects, numberIsFloat: bindingConfig.NumberIsFloat, - analysisMode: false); + analysisMode: bindingConfig.AnalysisMode); } /// diff --git a/src/libraries/Microsoft.PowerFx.Core/Binding/BindingConfig.cs b/src/libraries/Microsoft.PowerFx.Core/Binding/BindingConfig.cs index beaf512797..9e5b4ad2ae 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Binding/BindingConfig.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Binding/BindingConfig.cs @@ -19,13 +19,16 @@ internal class BindingConfig public bool UseThisRecordForRuleScope { get; } - public bool NumberIsFloat { get; } + public bool NumberIsFloat { get; } + + public bool AnalysisMode { get; } - public BindingConfig(bool allowsSideEffects = false, bool useThisRecordForRuleScope = false, bool numberIsFloat = false) + public BindingConfig(bool allowsSideEffects = false, bool useThisRecordForRuleScope = false, bool numberIsFloat = false, bool analysisMode = false) { AllowsSideEffects = allowsSideEffects; UseThisRecordForRuleScope = useThisRecordForRuleScope; - NumberIsFloat = numberIsFloat; + NumberIsFloat = numberIsFloat; + AnalysisMode = analysisMode; } } } From ed1a1acf9b828579eb5b57309ef012722b663bb9 Mon Sep 17 00:00:00 2001 From: Greg Lindhorst Date: Wed, 5 Jun 2024 20:56:19 -0700 Subject: [PATCH 25/29] Deep mutations of records, squashed (#2457) Adds support for deep mutation through the Set function. `Set( a, {b:1} )` doesn't change and will assign a reference to the record `{b:1}` to the variable `a`. After `a` has been established, it now supports mutation through `Set( a.b, 2 )`. This is an optional facility that is disabled by default and there are no semantic changes. When creating variables the host has the option to enable this with `SymbolProperties.CanSetMutate`. This also fixes two related bugs. First, really deep mutations were not being properly copied on write, where `t` should be unaffected by the Patch https://github.com/microsoft/Power-Fx/issues/2450: ``` >> Set( deep, [[[[1,2,3],[4,5,6]]]] ) deep: [[[[1, 2, 3], [4, 5, 6]]]] >> Set( t, deep ) t: [[[[1, 2, 3], [4, 5, 6]]]] >> Patch( First(First(First(deep).Value).Value).Value, {Value:2}, {Value:99} ) {Value:99} >> deep [[[[1, 99, 3], [4, 5, 6]]]] >> t [[[[1, 99, 3], [4, 5, 6]]]] ``` And this should not return a casting error, instead returning Blank() https://github.com/microsoft/Power-Fx/issues/2451: ``` >> Patch(Blank(),Blank(),Blank()) Unable to cast object of type 'Microsoft.PowerFx.Types.BlankType' to type 'Microsoft.PowerFx.Types.RecordType'. ``` --- .../Microsoft.PowerFx.Core/Binding/Binder.cs | 38 ++ .../Functions/TexlFunction.cs | 29 +- .../Microsoft.PowerFx.Core/IR/IRTranslator.cs | 2 +- .../Public/Config/DeferredSymbolTable.cs | 3 +- .../Public/Config/SymbolProperties.cs | 22 +- .../Public/Config/SymbolTable.cs | 3 +- .../Config/SymbolTableOverRecordType.cs | 9 +- .../Public/Values/CollectionTableValue.cs | 52 ++- .../Public/Values/TableValue.cs | 108 +++++- .../Texl/Builtins/Clear.cs | 2 +- .../Texl/Builtins/Patch.cs | 2 +- .../Environment/PowerFxConfigExtensions.cs | 2 +- .../EvalVisitor.cs | 22 +- .../Functions/LibraryTable.cs | 17 +- .../Functions/Mutation/RemoveFunction.cs | 2 +- .../Functions/SetFunction.cs | 77 ++-- .../RecalcEngine.cs | 22 +- src/strings/PowerFxResources.en-US.resx | 2 +- .../ExpressionTestCases/Clear_V1Compat.txt | 6 + .../FileExpressionEvaluationTests.cs | 5 + .../AutomatedLowCodePlugins.txt | 181 +++++++++ .../MutationScripts/DeepMutation_V1Compat.txt | 71 ++++ .../Set_DeepMutation_V1Compat.txt | 342 ++++++++++++++++++ .../PowerFxEvaluationTests.cs | 45 ++- .../Scenarios/DataTableMarshallerProvider.cs | 2 +- .../SetFunctionTests.cs | 52 ++- .../SymbolValueTests.cs | 19 +- .../YamlTexlFunction.cs | 2 - 28 files changed, 1038 insertions(+), 101 deletions(-) create mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/AutomatedLowCodePlugins.txt create mode 100644 src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/Set_DeepMutation_V1Compat.txt diff --git a/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs b/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs index 20ef07d1d6..eb8278e9d4 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Binding/Binder.cs @@ -84,6 +84,7 @@ internal sealed partial class TexlBinding private readonly BitArray _isConstant; private readonly BitArray _isSelfContainedConstant; private readonly BitArray _isMutable; + private readonly BitArray _isSetMutable; // Whether a node supports its rowscoped param exempted from delegation check. e.g. The 3rd argument in AddColumns function private readonly BitArray _supportsRowScopedParamDelegationExempted; @@ -305,6 +306,7 @@ private TexlBinding( _isContextual = new BitArray(idLim); _isConstant = new BitArray(idLim); _isMutable = new BitArray(idLim); + _isSetMutable = new BitArray(idLim); _isSelfContainedConstant = new BitArray(idLim); _lambdaScopingMap = new ScopeUseSet[idLim]; _isDelegatable = new BitArray(idLim); @@ -553,6 +555,15 @@ private void SetMutable(TexlNode node, bool isMutable) _isMutable.Set(node.Id, isMutable); } + private void SetSetMutable(TexlNode node, bool isSetMutable) + { + Contracts.AssertValue(node); + Contracts.AssertIndex(node.Id, _typeMap.Length); + Contracts.Assert(isSetMutable || !_isSetMutable.Get(node.Id)); + + _isSetMutable.Set(node.Id, isSetMutable); + } + private void SetSelfContainedConstant(TexlNode node, bool isConstant) { Contracts.AssertValue(node); @@ -1333,6 +1344,14 @@ public bool IsMutable(TexlNode node) return _isMutable.Get(node.Id); } + public bool IsSetMutable(TexlNode node) + { + Contracts.AssertValue(node); + Contracts.AssertIndex(node.Id, _isSetMutable.Length); + + return _isSetMutable.Get(node.Id); + } + public bool IsSelfContainedConstant(TexlNode node) { Contracts.AssertValue(node); @@ -2891,6 +2910,7 @@ public override void Visit(FirstNameNode node) { var nameSymbol = lookupInfo.Data as NameSymbol; _txb.SetMutable(node, nameSymbol?.Props.CanMutate ?? false); + _txb.SetSetMutable(node, nameSymbol?.Props.CanSetMutate ?? false); if (lookupInfo.Data is IExternalNamedFormula formula) { isConstantNamedFormula = formula.IsConstant; @@ -3724,6 +3744,7 @@ public override void PostVisit(DottedNameNode node) // An `a.b` expression will be mutable if `a` is mutable _txb.SetMutable(node, _txb.IsMutable(node.Left)); + _txb.SetSetMutable(node, _txb.IsSetMutable(node.Left)); _txb.SetConstant(node, isConstant); _txb.SetSelfContainedConstant(node, leftType.IsEnum || (leftType.IsAggregate && _txb.IsSelfContainedConstant(node.Left))); @@ -4692,6 +4713,23 @@ private void FinalizeCall(CallNode node) _txb.SetMutable(node, mutable); } + // Propagate mutability if supported by the function + if (func.PropagatesMutability && node.Args.Count > 0 && _txb.IsSetMutable(node.Args.ChildNodes[0])) + { + var firstChildNode = node.Args.ChildNodes[0]; + + // Propagate mutability if it is *not* a connected data source + var mutable = true; + if (firstChildNode is FirstNameNode first && + _nameResolver?.Lookup(first.Ident.Name, out var lookupInfo) == true && + lookupInfo.Kind == BindKind.Data) + { + mutable = false; + } + + _txb.SetSetMutable(node, mutable); + } + // Invalid datasources always result in error if (func.IsBehaviorOnly && !_txb.BindingConfig.AllowsSideEffects) { diff --git a/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs b/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs index 699357d08b..6ce885bff3 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Functions/TexlFunction.cs @@ -126,8 +126,16 @@ internal abstract class TexlFunction : IFunction /// public virtual bool HasPreciseErrors => false; - // Returns true if the function will mutate the value of argument 0, as is the case with Patch, Collect, Remove, etc. - public virtual bool MutatesArg0 => false; + /// + /// Returns true if the function will mutate the argument, as is the case of Patch, Collect, Remove, etc. + /// Set can also mutate, but needs to make a decision based on the argument's node. + /// For example, Set(x,{a:1}) is not a mutate and has a single FirstName node for the first argument, + /// while Set(x.a,1) is a mutate and has a more complex node for the first argument. + /// This function covers both CanMutate and CanSetMutate scenarios which is checked in CheckTypes/CheckSemantics. + /// + /// Index of the argument. + /// Argument at that index. + public virtual bool MutatesArg(int argIndex, TexlNode arg) => false; public virtual RequiredDataSourcePermissions FunctionPermission => RequiredDataSourcePermissions.None; @@ -247,6 +255,23 @@ protected void ValidateArgumentIsMutable(TexlBinding binding, TexlNode arg, IErr } } + /// + /// Adds an error to the container if the given argument is immutable. + /// + /// + /// + /// + protected void ValidateArgumentIsSetMutable(TexlBinding binding, TexlNode arg, IErrorContainer errors) + { + if (binding.Features.PowerFxV1CompatibilityRules && !binding.IsSetMutable(arg)) + { + errors.EnsureError( + arg, + new ErrorResourceKey("ErrorResource_MutationFunctionCannotBeUsedWithImmutableValue"), + this.Name); + } + } + /// /// Indicates whether the function sets a value. /// diff --git a/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs b/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs index 2baddcb945..094426be3d 100644 --- a/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs +++ b/src/libraries/Microsoft.PowerFx.Core/IR/IRTranslator.cs @@ -352,7 +352,7 @@ public override IntermediateNode Visit(TexlCallNode node, IRTranslatorContext co for (var i = 0; i < carg; ++i) { var arg = node.Args.Children[i]; - var argContext = i == 0 && func.MutatesArg0 ? new IRTranslatorContext(context, isMutation: true) : context; + var argContext = func.MutatesArg(i, arg) ? new IRTranslatorContext(context, isMutation: true) : context; var supportColumnNamesAsIdentifiers = _features.SupportColumnNamesAsIdentifiers; if (supportColumnNamesAsIdentifiers && func.ParameterCanBeIdentifier(arg, i, context.Binding.Features)) diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Config/DeferredSymbolTable.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Config/DeferredSymbolTable.cs index 2702d1c2b7..55cbb4989d 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Config/DeferredSymbolTable.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Config/DeferredSymbolTable.cs @@ -145,7 +145,8 @@ private NameLookupInfo AddUnderLock(DName logical, DName display, FormulaType ty var props = new SymbolProperties { CanMutate = true, - CanSet = false + CanSet = false, + CanSetMutate = false }; var data = new NameSymbol(logical, props) diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolProperties.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolProperties.cs index fcd2e36d53..e7f119fff4 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolProperties.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolProperties.cs @@ -17,13 +17,29 @@ namespace Microsoft.PowerFx { public class SymbolProperties { - // Can this symbol be reassigned with a Set() function. + // When a symbol table variable is created, these attributes can be specified. + // A host can override these settings as they see fit. + // + // Examples by category: + // + // CanSet CanMutate CanSetMutate + // ================================================================ + // Data sources false true false + // Scope variables false false false + // Global variables (1) true true false + // Low code plugin NewRecord false false true + // Low code plugin OldRecord false false false + // + // (1) CanSetMutate for global variables is false by default today, but we may change this default in the future. + + // Can this symbol be reassigned with a Set function. For example: Set( a, {b:3} ) public bool CanSet { get; init; } - // Can this symbol be passed to Collect, Patch() for mutation + // Can this symbol be passed to Collect, Patch, Remove, etc. for mutation. For example: Collect( t, {b:3} ) // This only applies to symbols of a tabular type. public bool CanMutate { get; init; } - // Is Copyable? + // Can this symbol be deep mutated with Set. For example: Set( a.b, 3 ) + public bool CanSetMutate { get; init; } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTable.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTable.cs index 285b4a3314..3b3992f68b 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTable.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTable.cs @@ -101,7 +101,8 @@ public ISymbolSlot AddVariable(string name, FormulaType type, bool mutable = fal var props = new SymbolProperties { CanSet = mutable, - CanMutate = mutable + CanMutate = mutable, + CanSetMutate = false }; return AddVariable(name, type, props, displayName); } diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTableOverRecordType.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTableOverRecordType.cs index 705f913657..af768405b0 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTableOverRecordType.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Config/SymbolTableOverRecordType.cs @@ -40,7 +40,8 @@ public SymbolTableOverRecordType(RecordType type, ReadOnlySymbolTable parent = n var data = new NameSymbol(TexlBinding.ThisRecordDefaultName, new SymbolProperties { CanMutate = false, - CanSet = false + CanSet = false, + CanSetMutate = false }) { Owner = this, @@ -70,7 +71,8 @@ internal SymbolTableOverRecordType(RecordType type, ReadOnlySymbolTable parent = var data = new NameSymbol(TexlBinding.ThisRecordDefaultName, new SymbolProperties { CanMutate = false, - CanSet = false + CanSet = false, + CanSetMutate = false }) { Owner = this, @@ -224,7 +226,8 @@ private NameLookupInfo Create(string logicalName, FormulaType type) data = new NameSymbol(logicalName, new SymbolProperties { CanSet = _mutable, - CanMutate = false + CanMutate = false, + CanSetMutate = false }) { Owner = this, diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Values/CollectionTableValue.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Values/CollectionTableValue.cs index dd65060166..69eb9d6050 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Values/CollectionTableValue.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Values/CollectionTableValue.cs @@ -126,14 +126,8 @@ public override async Task> ClearAsync(CancellationToken ca protected override bool TryGetIndex(int index1, out DValue record) { var index0 = index1 - 1; - if (_sourceIndex != null) + if (_sourceIndex != null && index0 >= 0 && index0 < _sourceCount.Count) { - if (index0 < 0 || index0 >= _sourceCount.Count) - { - record = null; - return false; - } - var item = _sourceIndex[index0]; record = Marshal(item); return true; @@ -144,6 +138,50 @@ record = Marshal(item); } } + protected override bool TryGetIndex(int index1, out DValue record, bool mutationCopy) + { + if (!mutationCopy) + { + return TryGetIndex(index1, out record); + } + else + { + var index0 = index1 - 1; + if (_sourceIndex != null && _sourceMutableIndex != null && index0 >= 0 && index0 < _sourceCount.Count) + { + RecordValue rec = Marshal(_sourceIndex[index0]).Value; + RecordValue copyRecord = (RecordValue)rec.MaybeShallowCopy(); + _sourceMutableIndex[index0] = MarshalInverse(copyRecord); + record = DValue.Of(copyRecord); + return true; + } + else + { + return base.TryGetIndex(index1, out record, mutationCopy); + } + } + } + + public override DValue First(bool mutationCopy = false) + { + if (TryGetIndex(1, out var record, mutationCopy: mutationCopy)) + { + return record; + } + + return DValue.Of(FormulaValue.NewBlank()); + } + + public override DValue Last(bool mutationCopy = false) + { + if (TryGetIndex(Count(), out var record, mutationCopy: mutationCopy)) + { + return record; + } + + return DValue.Of(FormulaValue.NewBlank()); + } + public override async Task> RemoveAsync(IEnumerable recordsToRemove, bool all, CancellationToken cancellationToken) { var ret = false; diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Values/TableValue.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Values/TableValue.cs index a1ef038ec6..186e285748 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Values/TableValue.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Values/TableValue.cs @@ -99,7 +99,7 @@ public virtual int Count() /// /// Lookup the record at the given 1-based index, or return an error value if out of range. /// - /// 1-based index. + /// 1-based index. /// The record or an errorValue. public DValue Index(int index1) { @@ -108,24 +108,96 @@ public DValue Index(int index1) return record; } - return DValue.Of(ArgumentOutOfRange(IRContext)); - } - - // Index() does standard error messaging and then call TryGetIndex(). - protected virtual bool TryGetIndex(int index1, out DValue record) + return DValue.Of(ArgumentOutOfRangeError(IRContext)); + } + + /// + /// With mutation support, lookup the record at the given 1-based index, or return an error value if out of range. + /// + /// 1-based index. + /// copies the element, and the table entry pointing to it, when in a mutation context. + /// The record or an errorValue. + public DValue Index(int index1, bool mutationCopy) { - var index0 = index1 - 1; - if (index0 < 0) + if (TryGetIndex(index1, out var record, mutationCopy)) { - record = null; - return false; + return record; } - record = Rows.ElementAtOrDefault(index0); - return record != null; + return DValue.Of(ArgumentOutOfRangeError(IRContext)); + } + + // Index() does standard error messaging and then call TryGetIndex(). + // Can't mutate through this entry point. + // It is OK to just override this overload if the table is not mutable. + protected virtual bool TryGetIndex(int index1, out DValue record) + { + var index0 = index1 - 1; + if (index0 < 0) + { + record = null; + return false; + } + + record = Rows.ElementAtOrDefault(index0); + return record != null; + } + + // Index() does standard error messaging and then call TryGetIndex(). + // This needs to be overriden to support mutation. + protected virtual bool TryGetIndex(int index1, out DValue record, bool mutationCopy) + { + if (!mutationCopy) + { + return TryGetIndex(index1, out record); + } + else + { + throw new NotImplementedException(); + } + } + + /// + /// Lookup the first record, or return blank if the table is empty. + /// + /// copies the element, and the table entry pointing to it, when in a mutation context. + /// The record or blank. + public virtual DValue First(bool mutationCopy = false) + { + if (mutationCopy) + { + return DValue.Of(ImmutableTableError(IRContext)); + } + + return Rows.FirstOrDefault() ?? DValue.Of(FormulaValue.NewBlank()); + } + + /// + /// Lookup the last record, or return blank if the table is empty. + /// + /// copies the element, and the table entry pointing to it, when in a mutation context. + /// The record or blank. + public virtual DValue Last(bool mutationCopy = false) + { + if (mutationCopy) + { + return DValue.Of(ImmutableTableError(IRContext)); + } + + return Rows.LastOrDefault() ?? DValue.Of(FormulaValue.NewBlank()); + } + + private static ErrorValue ImmutableTableError(IRContext irContext) + { + return new ErrorValue(irContext, new ExpressionError() + { + Message = "Table is immutable", + Span = irContext.SourceContext, + Kind = ErrorKind.InvalidArgument + }); } - private static ErrorValue ArgumentOutOfRange(IRContext irContext) + private static ErrorValue ArgumentOutOfRangeError(IRContext irContext) { return new ErrorValue(irContext, new ExpressionError() { @@ -135,7 +207,7 @@ private static ErrorValue ArgumentOutOfRange(IRContext irContext) }); } - private static ErrorValue NotImplemented(IRContext irContext, [CallerMemberName] string methodName = null) + private static ErrorValue NotImplementedError(IRContext irContext, [CallerMemberName] string methodName = null) { return new ErrorValue(irContext, new ExpressionError() { @@ -151,17 +223,17 @@ private static ErrorValue NotImplemented(IRContext irContext, [CallerMemberName] // Async because derived classes may back this with a network call. public virtual async Task> AppendAsync(RecordValue record, CancellationToken cancellationToken) { - return DValue.Of(NotImplemented(IRContext)); + return DValue.Of(NotImplementedError(IRContext)); } public virtual async Task> RemoveAsync(IEnumerable recordsToRemove, bool all, CancellationToken cancellationToken) { - return DValue.Of(NotImplemented(IRContext)); + return DValue.Of(NotImplementedError(IRContext)); } public virtual async Task> ClearAsync(CancellationToken cancellationToken) { - return DValue.Of(NotImplemented(IRContext)); + return DValue.Of(NotImplementedError(IRContext)); } /// @@ -173,7 +245,7 @@ public virtual async Task> ClearAsync(CancellationToken can /// protected virtual async Task> PatchCoreAsync(RecordValue baseRecord, RecordValue changeRecord, CancellationToken cancellationToken) { - return DValue.Of(NotImplemented(IRContext)); + return DValue.Of(NotImplementedError(IRContext)); } /// diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Clear.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Clear.cs index 1ede184e44..f29da6bf76 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Clear.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Clear.cs @@ -26,7 +26,7 @@ internal class ClearFunction : BuiltinFunction public override bool SupportsParamCoercion => false; - public override bool MutatesArg0 => true; + public override bool MutatesArg(int argIndex, TexlNode arg) => argIndex == 0; public ClearFunction() : base("Clear", TexlStrings.AboutClear, FunctionCategories.Behavior, DType.Unknown, 0, 1, 1, DType.EmptyTable) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs index 45eab62d3f..f993279e90 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Patch.cs @@ -283,7 +283,7 @@ internal abstract class PatchAsyncFunctionCore : PatchAndValidateRecordFunctionB // Return true if this function affects datasource query options. public override bool AffectsDataSourceQueryOptions => true; - public override bool MutatesArg0 => true; + public override bool MutatesArg(int argIndex, TexlNode arg) => argIndex == 0; public override RequiredDataSourcePermissions FunctionPermission => RequiredDataSourcePermissions.Create | RequiredDataSourcePermissions.Update; diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs index b307af9826..891306015b 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs @@ -23,7 +23,7 @@ public static void AddFunction(this SymbolTable symbolTable, ReflectionFunction } /// - /// Enable a Set() function which allows scripts to do . + /// Enable a Set() function which allows scripts to do . /// /// public static void EnableSetFunction(this PowerFxConfig powerFxConfig) diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs b/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs index 9874d9708a..5e256748e3 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs @@ -165,8 +165,24 @@ private async Task TryHandleSet(CallNode node, EvalVisitorContext var arg0 = node.Args[0]; var arg1 = node.Args[1]; - var newValue = await arg1.Accept(this, context).ConfigureAwait(false); - + var newValue = await arg1.Accept(this, context).ConfigureAwait(false); + + if (arg0.IRContext.IsMutation && arg0 is RecordFieldAccessNode rfan) + { + var arg0value = await rfan.From.Accept(this, context).ConfigureAwait(false); + + if (arg0value is RecordValue rv && arg0value is IMutationCopyField) + { + ((IMutationCopyField)rv).ShallowCopyFieldInPlace(rfan.Field); + rv.UpdateField(rfan.Field, newValue); + return node.IRContext.ResultType._type.Kind == DKind.Boolean ? FormulaValue.New(true) : FormulaValue.NewVoid(); + } + else + { + return CommonErrors.UnreachableCodeError(node.IRContext); + } + } + // Binder has already ensured this is a first name node as well as mutable symbol. if (arg0 is ResolvedObjectNode obj) { @@ -180,7 +196,7 @@ private async Task TryHandleSet(CallNode node, EvalVisitorContext // This may happen if the runtime symbols are missing a value and we failed to update. } - } + } // Fail? return CommonErrors.UnreachableCodeError(node.IRContext); diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryTable.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryTable.cs index e4f33b556b..ad1f52d9e5 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryTable.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryTable.cs @@ -59,13 +59,13 @@ public static FormulaValue First(IRContext irContext, TableValue[] args) { } } - - return arg0.Rows.FirstOrDefault()?.ToFormulaValue() ?? new BlankValue(irContext); + + return arg0.First(mutationCopy: irContext.IsMutation).ToFormulaValue(); } public static FormulaValue Last(IRContext irContext, TableValue[] args) { - return args[0].Rows.LastOrDefault()?.ToFormulaValue() ?? new BlankValue(irContext); + return args[0].Last(mutationCopy: irContext.IsMutation).ToFormulaValue(); } public static FormulaValue FirstN(IRContext irContext, FormulaValue[] args) @@ -526,7 +526,7 @@ public static FormulaValue IndexTable(IRContext irContext, FormulaValue[] args) var arg1 = (NumberValue)args[1]; var rowIndex = (int)arg1.Value; - return arg0.Index(rowIndex).ToFormulaValue(); + return arg0.Index(rowIndex, mutationCopy: irContext.IsMutation).ToFormulaValue(); } public static FormulaValue Shuffle(IServiceProvider services, IRContext irContext, FormulaValue[] args) @@ -1161,7 +1161,14 @@ private static FormulaValue Refresh(EvalVisitor runner, EvalVisitorContext conte public static FormulaValue PatchRecord(IRContext irContext, FormulaValue[] args) { - return CompileTimeTypeWrapperRecordValue.AdjustType((RecordType)FormulaType.Build(irContext.ResultType._type), (RecordValue)MutationUtils.MergeRecords(args).ToFormulaValue()); + if (irContext.ResultType is BlankType) + { + return new BlankValue(irContext); + } + else + { + return CompileTimeTypeWrapperRecordValue.AdjustType((RecordType)FormulaType.Build(irContext.ResultType._type), (RecordValue)MutationUtils.MergeRecords(args).ToFormulaValue()); + } } public static async ValueTask Summarize(EvalVisitor runner, EvalVisitorContext context, IRContext irContext, FormulaValue[] args) diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/RemoveFunction.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/RemoveFunction.cs index b7c089ec59..e0bb724273 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/RemoveFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/RemoveFunction.cs @@ -35,7 +35,7 @@ public override bool ArgMatchesDatasourceType(int argNum) return argNum >= 1; } - public override bool MutatesArg0 => true; + public override bool MutatesArg(int argIndex, TexlNode arg) => argIndex == 0; public override bool IsLazyEvalParam(TexlNode node, int index, Features features) { diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs index bf38025ff7..fc0f632e3c 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/SetFunction.cs @@ -26,7 +26,10 @@ namespace Microsoft.PowerFx.Interpreter internal class RecalcEngineSetFunction : TexlFunction { // Set() is a behavior function. - public override bool IsSelfContained => false; + public override bool IsSelfContained => false; + + // Set() of a simple identifier is not a mutation through a reference (a mutate), but rather changing the reference (a true set). + public override bool MutatesArg(int argIndex, TexlNode arg) => argIndex == 0 && arg.Kind != NodeKind.FirstName; public override IEnumerable GetSignatures() { @@ -70,52 +73,58 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ Contracts.AssertValue(errors); Contracts.Assert(MinArity <= args.Length && args.Length <= MaxArity); - var arg0 = argTypes[0]; - - var firstName = args[0].AsFirstName(); + var arg0 = argTypes[0]; + var arg1 = argTypes[1]; - if (firstName != null) + // Type check + if (!(arg0.Accepts(arg1, exact: true, useLegacyDateTimeAccepts: false, usePowerFxV1CompatibilityRules: binding.Features.PowerFxV1CompatibilityRules) || + (arg0.IsNumeric && arg1.IsNumeric))) { - var info = binding.GetInfo(firstName); - if (info.Data is NameSymbol nameSymbol && nameSymbol.Props.CanSet) - { - // We have a variable. type check - var arg1 = argTypes[1]; + errors.EnsureError(DocumentErrorSeverity.Critical, args[1], ErrBadType_ExpectedType_ProvidedType, arg0.GetKindString(), arg1.GetKindString()); + return; + } - if (!(arg0.Accepts(arg1, exact: true, useLegacyDateTimeAccepts: false, usePowerFxV1CompatibilityRules: binding.Features.PowerFxV1CompatibilityRules) || - (arg0.IsNumeric && arg1.IsNumeric))) + if (arg1.AggregateHasExpandedType()) + { + if (binding.Features.SkipExpandableSetSemantics) + { + errors.EnsureError(DocumentErrorSeverity.Warning, args[1], WrnSetExpandableType); + return; + } + else + { + if (arg1.IsTable) { - errors.EnsureError(DocumentErrorSeverity.Critical, args[1], ErrBadType_ExpectedType_ProvidedType, arg0.GetKindString(), arg1.GetKindString()); + errors.EnsureError(DocumentErrorSeverity.Critical, args[1], ErrSetVariableWithRelationshipNotAllowTable); return; } - if (arg1.AggregateHasExpandedType()) + if (arg1.IsRecord) { - if (binding.Features.SkipExpandableSetSemantics) - { - errors.EnsureError(DocumentErrorSeverity.Warning, args[1], WrnSetExpandableType); - return; - } - else - { - if (arg1.IsTable) - { - errors.EnsureError(DocumentErrorSeverity.Critical, args[1], ErrSetVariableWithRelationshipNotAllowTable); - return; - } - - if (arg1.IsRecord) - { - errors.EnsureError(DocumentErrorSeverity.Critical, args[1], ErrSetVariableWithRelationshipNotAllowRecord); - return; - } - } + errors.EnsureError(DocumentErrorSeverity.Critical, args[1], ErrSetVariableWithRelationshipNotAllowRecord); + return; } + } + } + + var firstName = args[0].AsFirstName(); - // Success + if (firstName != null) + { + // Variable reference assignment, for example Set( x, 3 ) + var info = binding.GetInfo(firstName); + if (info.Data is NameSymbol nameSymbol && nameSymbol.Props.CanSet) + { + // We have a variable, success return; } } + else if (binding.Features.PowerFxV1CompatibilityRules) + { + // Deep mutation, for example Set( x.a, 4 ) + base.ValidateArgumentIsSetMutable(binding, args[0], errors); + return; + } errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrNeedValidVariableName_Arg, Name, args[0]); return; diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs b/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs index fe1acf1ee5..7f16630650 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs @@ -128,11 +128,22 @@ public void UpdateVariable(string name, TimeSpan value) } /// - /// Create or update a named variable to a value. + /// Create or update a named variable to a value, with custom CanSet/Mutate attributes. /// /// variable name. This can be used in other formulas. - /// constant value. + /// constant value. The variable will take the type of this value on create. public void UpdateVariable(string name, FormulaValue value) + { + UpdateVariable(name, value, new SymbolProperties { CanMutate = true, CanSet = true }); + } + + /// + /// Create or update a named variable to a value, with custom CanSet/Mutate attributes. + /// + /// variable name. This can be used in other formulas. + /// constant value. The variable will take the type of this value on create. + /// symbol properties. This is only used on the initial create of the variable. + public void UpdateVariable(string name, FormulaValue value, SymbolProperties newVarProps) { var x = value; @@ -151,7 +162,12 @@ public void UpdateVariable(string name, FormulaValue value) else { // New - var slot = _symbolTable.AddVariable(name, value.Type, mutable: true); + if (newVarProps == null) + { + newVarProps = new SymbolProperties { CanMutate = true, CanSet = true }; + } + + var slot = _symbolTable.AddVariable(name, value.Type, newVarProps); Formulas[slot.SlotIndex] = RecalcFormulaInfo.NewVariable(slot, name, x.IRContext.ResultType); _symbolValues.Set(slot, value); diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index 8491989c8a..eeb32789ed 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -4106,7 +4106,7 @@ {0} - Entity that is about to be created/renamed, {1} - Name of the entity, {2} - Type of entity that is already using the name. - The first argument of '{0}' should be a valid variable name, and cannot conflict with any existing control, screen, collection, or data source names. Found type '{1}' + '{1}' can't be modified with '{0}' or conflicts with the name of an existing scope variable, data source, or other object. Error Message diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Clear_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Clear_V1Compat.txt index 2bceebc8a5..d11c7f3690 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Clear_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Clear_V1Compat.txt @@ -12,6 +12,12 @@ Errors: Error 6-7: Invalid argument type (Decimal). Expecting a Table value inst >> Clear(1/0) Errors: Error 7-8: Invalid argument type (Decimal). Expecting a Table value instead.|Error 0-5: The function 'Clear' has some invalid arguments. +>> Clear(If(false,[1,2,3])) +Errors: Error 6-23: The value passed to the 'Clear' function cannot be changed. + +>> Clear(If(false,[1,2,3],Error("bad table"))) +Errors: Error 6-42: The value passed to the 'Clear' function cannot be changed. + >> IsError(Clear(1)) Errors: Error 14-15: Invalid argument type (Decimal). Expecting a Table value instead.|Error 8-13: The function 'Clear' has some invalid arguments. diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/FileExpressionEvaluationTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/FileExpressionEvaluationTests.cs index 5d0c295274..524433dffa 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/FileExpressionEvaluationTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/FileExpressionEvaluationTests.cs @@ -233,6 +233,11 @@ private void RunMutationTestFile(string file, Features features, string setup) testRunner.AddFile(TestRunner.ParseSetupString(setup), path); + if (testRunner.Tests.Count > 0 && testRunner.Tests[0].SetupHandlerName.Contains("MutationFunctionsTestSetup")) + { + ExpressionEvaluationTests.MutationFunctionsTestSetup(engine, false); + } + var result = testRunner.RunTests(); if (result.Fail > 0) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/AutomatedLowCodePlugins.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/AutomatedLowCodePlugins.txt new file mode 100644 index 0000000000..a53a247f64 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/AutomatedLowCodePlugins.txt @@ -0,0 +1,181 @@ +#SETUP: PowerFxV1CompatibilityRules,DecimalSupport,MutationFunctionsTestSetup + +>> NewRecord +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> OldRecord +{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false} + +>> NewRecord_SetEnabled_SetMutateEnabled +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +// Can compare fields, can't compare records + +>> NewRecord.Field1 = OldRecord.Field1 +false + +>> NewRecord.Field4 = Not( OldRecord.Field4 ) +true + +>> NewRecord = OldRecord; NewRecord // can't compare records, "; NewRecord" avoid the REPL from thinking it is named formula +Errors: Error 10-11: Incompatible types for comparison. These types can't be compared: Record, Record. + +// NewRecord can be mutated but not set + +>> Set( NR, NewRecord ) +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( NewRecord.Field1, 20 ); NewRecord +{Field1:20,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( NewRecord.Field2, "venus"); NewRecord +{Field1:20,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> NR // hasn't been modified from on write +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( NewRecord, OldRecord ) +Errors: Error 5-14: 'NewRecord' can't be modified with 'Set' or conflicts with the name of an existing scope variable, data source, or other object. + +// OldRecord can't be mutated or set + +>> Set( OldRecord.Field1, 30 ); NewRecord +Errors: Error 14-21: The value passed to the 'Set' function cannot be changed. + +>> Set( OldRecord.Field2, "mars"); NewRecord +Errors: Error 14-21: The value passed to the 'Set' function cannot be changed. + +>> Set( OldRecord, NewRecord ) +Errors: Error 5-14: 'OldRecord' can't be modified with 'Set' or conflicts with the name of an existing scope variable, data source, or other object. + +// Copies of NewRecord and OldRecord can't be mutated (today, we may relax this limitation in the future) + +>> Set( x, NewRecord ) +{Field1:20,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( x.Field1, 40 ) +Errors: Error 6-13: The value passed to the 'Set' function cannot be changed. + +>> Set( y, OldRecord ) +{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false} + +>> Set( y.Field1, 50 ) +Errors: Error 6-13: The value passed to the 'Set' function cannot be changed. + +// Data source, can't use Set to mutate, Patch needs to be used + +>> DataSource +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> Patch( DataSource, First(DataSource), {Field2: "saturn"} ) +{Field1:1,Field2:"saturn",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( DataSource, Table(First(DataSource)) ) +Errors: Error 5-15: 'DataSource' can't be modified with 'Set' or conflicts with the name of an existing scope variable, data source, or other object. + +>> Set( DataSource.Field2, [{Name:"saturn"}] ) +Errors: Error 15-22: Deprecated use of '.'. Please use the 'ShowColumns' function instead. + +>> Set( First(DataSource).Field2, "saturn" ) +Errors: Error 22-29: The value passed to the 'Set' function cannot be changed. + +// USE IN SCOPE VARIABLES + +// Scope variables cannot be Set + +>> With( NewRecord, Field1 ) +20 + +>> With( NewRecord, Set( Field1, 2 ) ) +Errors: Error 22-28: 'Field1' can't be modified with 'Set' or conflicts with the name of an existing scope variable, data source, or other object. + +>> With( NewRecord, Set( Field1, Field1 ) ) +Errors: Error 22-28: 'Field1' can't be modified with 'Set' or conflicts with the name of an existing scope variable, data source, or other object. + +>> With( NewRecord, Set( NewRecord.Field1, 22 ) ); NewRecord +{Field1:22,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> ForAll( [NewRecord, NewRecord], Field1 ) +Table({Value:22},{Value:22}) + +>> ForAll( [NewRecord, NewRecord], Set( Field1, 2 ) ) +Errors: Error 37-43: 'Field1' can't be modified with 'Set' or conflicts with the name of an existing scope variable, data source, or other object. + +>> ForAll( [NewRecord, NewRecord], Set( Field1, Field1 ) ) +Errors: Error 37-43: 'Field1' can't be modified with 'Set' or conflicts with the name of an existing scope variable, data source, or other object. + +>> ForAll( [NewRecord, NewRecord], Set( ThisRecord.Field1, 3 ) ) +Errors: Error 47-54: The value passed to the 'Set' function cannot be changed. + +>> ForAll( [NewRecord, NewRecord], Set( NewRecord.Field1, 23 ) ); NewRecord +{Field1:23,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +// The following three tests are effecitvely the same, with different levels of explicitness +// In all three cases, we are reading the scope variable Field1 that was copied at the point With started and using that value throughout +// The double additions don't matter since the source is always the same. +>> With( NewRecord, Set( NewRecord.Field1, Field1+1); Set( NewRecord.Field1, Field1+1) ); NewRecord.Field1 +24 + +>> With( NewRecord, Set( NewRecord.Field1, ThisRecord.Field1+1); Set( NewRecord.Field1, ThisRecord.Field1+1) ); NewRecord.Field1 +25 + +>> With( NewRecord As ThisRecord, Set( NewRecord.Field1, ThisRecord.Field1+1); Set( NewRecord.Field1, ThisRecord.Field1+1) ); NewRecord.Field1 +26 + +// In the future, we may allow variables to set mutate, which would use copy on write + +>> NewRecord // before NewRecord_SetEnabled_SetMutateEnabled changes +{Field1:26,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( NewRecord_SetEnabled_SetMutateEnabled, NewRecord ) +{Field1:26,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( NewRecord_SetEnabled_SetMutateEnabled.Field2, "saturn" ) +If(true, {test:1}, "Void value (result of the expression can't be used).") + +>> Set( NewRecord_SetEnabled_SetMutateEnabled.Field3, DateTime(2024,3,1,0,0,0,0) ) +If(true, {test:1}, "Void value (result of the expression can't be used).") + +>> Set( NewRecord_SetEnabled_SetMutateEnabled.Field4, Blank() ) +If(true, {test:1}, "Void value (result of the expression can't be used).") + +>> NewRecord_SetEnabled_SetMutateEnabled // after mutations +{Field1:26,Field2:"saturn",Field3:DateTime(2024,3,1,0,0,0,0),Field4:Blank()} + +>> NewRecord // unchanged (copy on write prevented it from being updated) +{Field1:26,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( z, NewRecord ) +{Field1:26,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( z, Patch( NewRecord, { Field2: "neptune", Field3: DateTime(2024,4,1,0,0,0,0), Field4: false } ) ) +{Field1:26,Field2:"neptune",Field3:DateTime(2024,4,1,0,0,0,0),Field4:false} + +>> NewRecord // unchanged after z copy (not a mutation, not a copy on write) +{Field1:26,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +// Check data type compatibility, should be the same as any other variable + +>> Set( NewRecord.Field1, Decimal(51) ); NewRecord +{Field1:51,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( NewRecord.Field1, Float(53) ); NewRecord +{Field1:53,Field2:"venus",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( NewRecord.Field1, true ); NewRecord +Errors: Error 23-27: Invalid argument type (Boolean). Expecting a Decimal value instead. + +>> Set( NewRecord.Field1, "53" ); NewRecord +Errors: Error 23-27: Invalid argument type (Text). Expecting a Decimal value instead. + +>> Set( n, Decimal(100) ) +100 + +>> Set( n, Float(101) ) +101 + +>> Set( n, true ) +Errors: Error 8-12: Invalid argument type (Boolean). Expecting a Decimal value instead. + +>> Set( n, "102" ) +Errors: Error 8-13: Invalid argument type (Text). Expecting a Decimal value instead. diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/DeepMutation_V1Compat.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/DeepMutation_V1Compat.txt index 3553409788..b9763cec62 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/DeepMutation_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/DeepMutation_V1Compat.txt @@ -410,3 +410,74 @@ Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value >> dbd // after dbd {d:Table({Value:1},{Value:2},{Value:3},{Value:11})} + +// REALLY DEEP MUTATIONS + +>> Set( deep, [[[[1,2,3],[4,5,6]]]] ) +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})})}) + +>> Set( deep_copy1, deep ) +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})})}) + +>> Patch( First(First(First(deep).Value).Value).Value, {Value:2}, {Value:99} ) +{Value:99} + +>> deep // after first update +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})})}) + +>> deep_copy1 +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})})}) + +>> Set( deep_copy2, deep ) +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})})}) + +>> Patch( Last(Last(Last(deep).Value).Value).Value, {Value:6}, {Value:98} ) +{Value:98} + +>> deep // after last update +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:98})})})}) + +>> deep_copy1 // unchanged +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})})}) + +>> deep_copy2 +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})})}) + +>> Set( deep_copy3, deep ) +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:98})})})}) + +>> Patch( Index(Index(Index(deep,1).Value,1).Value,2).Value, {Value:4}, {Value:97} ) +{Value:97} + +>> deep // after index update +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:97},{Value:5},{Value:98})})})}) + +>> deep_copy1 // still unchanged +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})})}) + +>> deep_copy2 // unchanged +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})})}) + +>> deep_copy3 +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:98})})})}) + +>> Patch( Last(Index(First(deep_copy1).Value,1).Value).Value, {Value:5}, {Value:96} ) +{Value:96} + +>> deep_copy1 // local mod +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:96},{Value:6})})})}) + +>> Patch( Last(First(Index(deep_copy2,1).Value).Value).Value, {Value:4}, {Value:95} ) +{Value:95} + +>> deep_copy2 // local mod +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:95},{Value:5},{Value:6})})})}) + +>> Patch( Index(First(First(deep_copy3).Value).Value,1).Value, {Value:1}, {Value:94} ) +{Value:94} + +>> deep_copy3 // local mod +Table({Value:Table({Value:Table({Value:Table({Value:94},{Value:99},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:98})})})}) + +>> deep // unchanged after copy mutations +Table({Value:Table({Value:Table({Value:Table({Value:1},{Value:99},{Value:3})},{Value:Table({Value:97},{Value:5},{Value:98})})})}) diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/Set_DeepMutation_V1Compat.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/Set_DeepMutation_V1Compat.txt new file mode 100644 index 0000000000..99841b69dd --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationScripts/Set_DeepMutation_V1Compat.txt @@ -0,0 +1,342 @@ +#SETUP: PowerFxV1CompatibilityRules,MutationFunctionsTestSetup + +// Newly declared vars are output as part of the testing infrastructure. The Set function is actually returning Void. + +// BASICS by default, variables are mutable but not set mutable + +>> Set( x, {a:1} ) +{a:1} + +>> Set( x.a, 3 ) +Errors: Error 6-8: The value passed to the 'Set' function cannot be changed. + +>> Set( y, [1,2,3] ) +Table({Value:1},{Value:2},{Value:3}) + +>> Set( First(y).Value, 4 ) +Errors: Error 13-19: The value passed to the 'Set' function cannot be changed. + +>> Patch( y, First(y), {Value:5}) +{Value:5} + +>> y +Table({Value:5},{Value:2},{Value:3}) + +// LITERALS can never be mutated + +>> Set( {a:1,b:2}.a, 3 ) +Errors: Error 14-16: The value passed to the 'Set' function cannot be changed. + +>> Set( First([1,2,3]).Value, 4 ) +Errors: Error 19-25: The value passed to the 'Set' function cannot be changed. + +>> Set( First(Table({a:1},{a:2},{a:3})).a, 5 ) +Errors: Error 36-38: The value passed to the 'Set' function cannot be changed. + +>> Patch( [1,2,3], {Value:1}, {Value:4} ) +Errors: Error 7-14: The value passed to the 'Patch' function cannot be changed. + +// SET MUTATION + +>> Set( r1_copy1, r1_SetMutateEnabled ); r1_copy1 +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( r1_SetMutateEnabled.Field1, 25 ); r1_SetMutateEnabled +{Field1:25,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> r1_copy1 // umodified due to copy on write +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( r1_copy2, r1_MutateEnabled_SetMutateEnabled ); r1_copy2 +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( r1_MutateEnabled_SetMutateEnabled.Field1, 26 ); r1_MutateEnabled_SetMutateEnabled +{Field1:26,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> r1_copy2 // umodified due to copy on write +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( r1_copy3, r1_MutateEnabled ); r1_copy3 +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( r1_MutateEnabled.Field1, 27 ); r1_MutateEnabled_SetMutateEnabled +Errors: Error 21-28: The value passed to the 'Set' function cannot be changed. + +>> r1_MutateEnabled // unmodified due to the error +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> r1_copy3 // umodified due to copy on write +{Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Set( t1_copy1, t1_SetMutateEnabled ); t1_copy1 +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> Set( First(t1_SetMutateEnabled).Field1, 29 ); t1_SetMutateEnabled +Table({Field1:29,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> Set( t1_copy2, t1_MutateEnabled_SetMutateEnabled ); t1_copy2 +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> Set( First(t1_MutateEnabled_SetMutateEnabled).Field1, 28 ); t1_MutateEnabled_SetMutateEnabled +Table({Field1:28,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> Set( t1_copy3, t1_MutateEnabled ); t1_copy3 +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> Set( First(t1_MutateEnabled).Field1, 21 ); t1_MutateEnabled +Errors: Error 28-35: The value passed to the 'Set' function cannot be changed. + +// PATCH/CLEAR/REMOVE/ETC MUTATION + +>> Patch( t1_SetMutateEnabled, First(t1_SetMutateEnabled), {Field2: "neptune"}) +Errors: Error 7-26: The value passed to the 'Patch' function cannot be changed. + +>> Collect( t1_SetMutateEnabled, r2 ) +Errors: Error 9-28: The value passed to the 'Collect' function cannot be changed. + +>> t1_SetMutateEnabled // so far +Table({Field1:29,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> Patch( t1_MutateEnabled_SetMutateEnabled, First(t1_MutateEnabled_SetMutateEnabled), {Field2: "neptune"} ) +{Field1:28,Field2:"neptune",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Collect( t1_MutateEnabled_SetMutateEnabled, r2 ) +{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false} + +>> t1_MutateEnabled_SetMutateEnabled +Table({Field1:28,Field2:"neptune",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Patch( t1_MutateEnabled, First(t1_MutateEnabled), {Field2: "neptune"} ) +{Field1:1,Field2:"neptune",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true} + +>> Collect( t1_MutateEnabled, r2 ) +{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false} + +>> t1_MutateEnabled +Table({Field1:1,Field2:"neptune",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> t1_copy1 // umodified due to copy on write +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> t1_copy2 // umodified due to copy on write +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> t1_copy3 // umodified due to copy on write +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +// FUNCTIONS THAT PROPAGATE MUTABILITY + +>> Set( t1_copy4, t1_MutateEnabled_SetMutateEnabled ) +Table({Field1:28,Field2:"neptune",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Set( First(t1_MutateEnabled_SetMutateEnabled).Field3, DateTime(1999,1,1,0,0,0,0) ); t1_MutateEnabled_SetMutateEnabled +Table({Field1:28,Field2:"neptune",Field3:DateTime(1999,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Set( Last(t1_MutateEnabled_SetMutateEnabled).Field4, Blank() ); t1_MutateEnabled_SetMutateEnabled +Table({Field1:28,Field2:"neptune",Field3:DateTime(1999,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:Blank()}) + +>> Set( Index(t1_MutateEnabled_SetMutateEnabled,2).Field2, "mars" ); t1_MutateEnabled_SetMutateEnabled +Table({Field1:28,Field2:"neptune",Field3:DateTime(1999,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"mars",Field3:DateTime(2022,2,1,0,0,0,0),Field4:Blank()}) + +// FUNCTIONS THAT DON'T PROPAGATE MUTABILITY + +>> Set( Index(FirstN(t1_MutateEnabled_SetMutateEnabled,3),2).Field1, 6 ) +Errors: Error 57-64: The value passed to the 'Set' function cannot be changed. + +>> Set( Index(LastN(t1_MutateEnabled_SetMutateEnabled,3),2).Field1, 6 ) +Errors: Error 56-63: The value passed to the 'Set' function cannot be changed. + +>> Set( Index(Filter(t1_MutateEnabled_SetMutateEnabled,Field1>0),2).Field1, 6 ) +Errors: Error 64-71: The value passed to the 'Set' function cannot be changed. + +>> Set( Index(Sort(t1_MutateEnabled_SetMutateEnabled,Field1),2).Field1, 6 ) +Errors: Error 60-67: The value passed to the 'Set' function cannot be changed. + +>> Set( Index(SortByColumns(t1_MutateEnabled_SetMutateEnabled,Field1),2).Field1, 6 ) +Errors: Error 69-76: The value passed to the 'Set' function cannot be changed. + +>> Set( First(FirstN(t1_MutateEnabled_SetMutateEnabled,3)).Field1, 6 ) +Errors: Error 55-62: The value passed to the 'Set' function cannot be changed. + +>> Set( First(LastN(t1_MutateEnabled_SetMutateEnabled,3)).Field1, 6 ) +Errors: Error 54-61: The value passed to the 'Set' function cannot be changed. + +>> Set( First(Filter(t1_MutateEnabled_SetMutateEnabled,Field1>0)).Field1, 6 ) +Errors: Error 62-69: The value passed to the 'Set' function cannot be changed. + +>> Set( First(Sort(t1_MutateEnabled_SetMutateEnabled,Field1)).Field1, 6 ) +Errors: Error 58-65: The value passed to the 'Set' function cannot be changed. + +>> Set( First(SortByColumns(t1_MutateEnabled_SetMutateEnabled,Field1)).Field1, 6 ) +Errors: Error 67-74: The value passed to the 'Set' function cannot be changed. + +>> Set( Last(FirstN(t1_MutateEnabled_SetMutateEnabled,3)).Field1, 6 ) +Errors: Error 54-61: The value passed to the 'Set' function cannot be changed. + +>> Set( Last(LastN(t1_MutateEnabled_SetMutateEnabled,3)).Field1, 6 ) +Errors: Error 53-60: The value passed to the 'Set' function cannot be changed. + +>> Set( Last(Filter(t1_MutateEnabled_SetMutateEnabled,Field1>0)).Field1, 6 ) +Errors: Error 61-68: The value passed to the 'Set' function cannot be changed. + +>> Set( LookUp(t1_MutateEnabled_SetMutateEnabled,Field1>0).Field1, 6 ) +Errors: Error 55-62: The value passed to the 'Set' function cannot be changed. + +>> Set( Last(Sort(t1_MutateEnabled_SetMutateEnabled,Field1)).Field1, 6 ) +Errors: Error 57-64: The value passed to the 'Set' function cannot be changed. + +>> Set( Last(SortByColumns(t1_MutateEnabled_SetMutateEnabled,Field1)).Field1, 6 ) +Errors: Error 66-73: The value passed to the 'Set' function cannot be changed. + +>> Set( Last(Table(r1,r2)).Field1, 6 ) +Errors: Error 23-30: The value passed to the 'Set' function cannot be changed. + +>> Set( Last([r1,r2]).Field1, 6 ) +Errors: Error 18-25: The value passed to the 'Set' function cannot be changed. + +>> t1_MutateEnabled_SetMutateEnabled // all mods +Table({Field1:28,Field2:"neptune",Field3:DateTime(1999,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"mars",Field3:DateTime(2022,2,1,0,0,0,0),Field4:Blank()}) + +>> t1_copy2 // unchanged due to copy on write, recheck +Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true}) + +>> t1_copy4 // unchanged due to copy on write +Table({Field1:28,Field2:"neptune",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +// DEEP, DEEP RECORD MUTATION + +// rwr1_SetMutateEnabled + +>> Set( rwr1_copy1, rwr1_SetMutateEnabled ) +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"common"}},Field3:false} + +>> Set( rwr1_SetMutateEnabled.Field2.Field2_3.Field2_3_2, "very common" ); rwr1_SetMutateEnabled +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"very common"}},Field3:false} + +>> Set( rwr1_copy2, rwr1_SetMutateEnabled ) +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"very common"}},Field3:false} + +>> Set( rwr1_SetMutateEnabled.Field2.Field2_3.Field2_3_1, 2112 ); rwr1_SetMutateEnabled +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:2112,Field2_3_2:"very common"}},Field3:false} + +>> rwr1_copy1 // unchanged due to copy on write +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"common"}},Field3:false} + +>> rwr1_copy2 // unchanged due to copy on write +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"very common"}},Field3:false} + +// rwr1_MutateEnabled + +>> Set( rwr1_copy4, rwr1_MutateEnabled ) +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"common"}},Field3:false} + +>> Set( rwr1_MutateEnabled.Field2.Field2_3.Field2_3_2, "very common" ); rwr1_MutateEnabled +Errors: Error 39-50: The value passed to the 'Set' function cannot be changed. + +>> Set( rwr1_MutateEnabled.Field2.Field2_3.Field2_3_1, 2112 ); rwr1_MutateEnabled +Errors: Error 39-50: The value passed to the 'Set' function cannot be changed. + +>> rwr1_copy4 // unchanged due to errors +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"common"}},Field3:false} + +// rwr1_MutateEnabled_SetMutateEnabled + +>> Set( rwr1_copy5, rwr1_MutateEnabled_SetMutateEnabled ) +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"common"}},Field3:false} + +>> Set( rwr1_MutateEnabled_SetMutateEnabled.Field2.Field2_3.Field2_3_2, "super common" ); rwr1_MutateEnabled_SetMutateEnabled +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"super common"}},Field3:false} + +>> Set( rwr1_copy6, rwr1_MutateEnabled_SetMutateEnabled ) +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"super common"}},Field3:false} + +>> Set( rwr1_MutateEnabled_SetMutateEnabled.Field2.Field2_3.Field2_3_1, 4224 ); rwr1_MutateEnabled_SetMutateEnabled +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:4224,Field2_3_2:"super common"}},Field3:false} + +>> rwr1_copy5 // unchanged due to copy on write +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"common"}},Field3:false} + +>> rwr1_copy6 // unchanged due to copy on write +{Field1:1,Field2:{Field2_1:121,Field2_2:"2_2",Field2_3:{Field2_3_1:1231,Field2_3_2:"super common"}},Field3:false} + +// DEEP, DEEP RECORD MUTATION WITH TABLE + +// rwt1_MutateEnabled_SetMutateEnabled + +>> rwt1_MutateEnabled_SetMutateEnabled +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> Set( rwt1_copy1, rwt1_MutateEnabled_SetMutateEnabled ) +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> Set( First(rwt1_MutateEnabled_SetMutateEnabled.Field2.Field2_4).Field2, "mercury" ); rwt1_MutateEnabled_SetMutateEnabled +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"mercury",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> Collect( rwt1_MutateEnabled_SetMutateEnabled.Field2.Field2_4, r2 ); rwt1_MutateEnabled_SetMutateEnabled +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"mercury",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false})}} + +>> Set( Last(rwt1_MutateEnabled_SetMutateEnabled.Field2.Field2_4).Field2, "jupiter" ); rwt1_MutateEnabled_SetMutateEnabled +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"mercury",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"jupiter",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false})}} + +>> Patch( rwt1_MutateEnabled_SetMutateEnabled.Field2.Field2_4, LookUp(rwt1_MutateEnabled_SetMutateEnabled.Field2.Field2_4, Field2 = "jupiter"), { Field2: "stars" } ) +{Field1:2,Field2:"stars",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false} + +>> rwt1_MutateEnabled_SetMutateEnabled // after changes +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"mercury",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"stars",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false})}} + +>> rwt1_copy1 +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +// rwt1_SetMutateEnabled + +>> rwt1_SetMutateEnabled +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> Set( rwt1_copy2, rwt1_SetMutateEnabled ) +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> Set( First(rwt1_SetMutateEnabled.Field2.Field2_4).Field2, "mercury" ); rwt1_SetMutateEnabled +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"mercury",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> Collect( rwt1_SetMutateEnabled.Field2.Field2_4, r2 ); rwt1_SetMutateEnabled +Errors: Error 37-46: The value passed to the 'Collect' function cannot be changed. + +>> Set( Last(rwt1_SetMutateEnabled.Field2.Field2_4).Field2, "jupiter" ); rwt1_SetMutateEnabled +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"jupiter",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> Patch( rwt1_SetMutateEnabled.Field2.Field2_4, LookUp(rwt1_SetMutateEnabled.Field2.Field2_4, Field2 = "jupiter"), { Field2: "stars" } ) +Errors: Error 35-44: The value passed to the 'Patch' function cannot be changed. + +>> rwt1_SetMutateEnabled // after changes +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"jupiter",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> rwt1_copy2 +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +// rwt1_MutateEnabled + +>> rwt1_MutateEnabled +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> Set( rwt1_copy3, rwt1_MutateEnabled ) +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + +>> Set( First(rwt1_MutateEnabled.Field2.Field2_4).Field2, "mercury" ); rwt1_MutateEnabled +Errors: Error 46-53: The value passed to the 'Set' function cannot be changed. + +>> Collect( rwt1_MutateEnabled.Field2.Field2_4, r2 ); rwt1_MutateEnabled +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false})}} + +>> Set( Last(rwt1_MutateEnabled.Field2.Field2_4).Field2, "jupiter" ); rwt1_MutateEnabled +Errors: Error 45-52: The value passed to the 'Set' function cannot be changed. + +>> Patch( rwt1_MutateEnabled.Field2.Field2_4, LookUp(rwt1_MutateEnabled.Field2.Field2_4, Field2 = "moon"), { Field2: "stars" } ) +{Field1:2,Field2:"stars",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false} + +>> rwt1_MutateEnabled // after changes +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true},{Field1:2,Field2:"stars",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false})}} + +>> rwt1_copy3 +{Field1:3,Field2:{Field2_1:321,Field2_2:"2_2",Field2_4:Table({Field1:1,Field2:"earth",Field3:DateTime(2022,1,1,0,0,0,0),Field4:true})}} + + diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/PowerFxEvaluationTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/PowerFxEvaluationTests.cs index e3b7f9ff7c..afa31bcb1f 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/PowerFxEvaluationTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/PowerFxEvaluationTests.cs @@ -11,10 +11,13 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using System.Xml.Linq; using Microsoft.PowerFx.Core; using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Functions; using Microsoft.PowerFx.Core.Tests; +using Microsoft.PowerFx.Core.Tests.AssociatedDataSourcesTests; +using Microsoft.PowerFx.Core.Tests.Helpers; using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Types.Enums; using Microsoft.PowerFx.Core.Utils; @@ -590,7 +593,7 @@ private static object OptionSetSortTestSetup(PowerFxConfig config, SymbolTable s return null; } - private static void MutationFunctionsTestSetup(RecalcEngine engine, bool numberIsFloat) + internal static void MutationFunctionsTestSetup(RecalcEngine engine, bool numberIsFloat) { /* * Record r1 => {![Field1:n, Field2:s, Field3:d, Field4:b]} @@ -654,6 +657,14 @@ private static void MutationFunctionsTestSetup(RecalcEngine engine, bool numberI "DisplayNameField2_3")), "DisplayNameField2")) .Add(new NamedFormulaType("Field3", FormulaType.Boolean, "DisplayNameField3")); + + var recordWithTableType = RecordType.Empty() + .Add(new NamedFormulaType("Field1", numberType, "DisplayNameField1")) + .Add(new NamedFormulaType("Field2", RecordType.Empty() + .Add(new NamedFormulaType("Field2_1", numberType, "DisplayNameField2_1")) + .Add(new NamedFormulaType("Field2_2", FormulaType.String, "DisplayNameField2_2")) + .Add(new NamedFormulaType("Field2_4", FormulaValue.NewTable(r1.Type).Type, "DisplayNameField2_4")), + "DisplayNameField2")); #pragma warning restore SA1117 // Parameters should be on same line or separate lines var recordWithRecordFields1 = new List() @@ -704,9 +715,21 @@ private static void MutationFunctionsTestSetup(RecalcEngine engine, bool numberI new NamedValue("Field3", FormulaValue.New(true)) }; + var recordWithTableFields1 = new List() + { + new NamedValue("Field1", newNumber(3)), + new NamedValue("Field2", FormulaValue.NewRecordFromFields(new List() + { + new NamedValue("Field2_1", newNumber(321)), + new NamedValue("Field2_2", FormulaValue.New("2_2")), + new NamedValue("Field2_4", FormulaValue.NewTable(r1.Type, r1)), + })) + }; + var recordWithRecord1 = FormulaValue.NewRecordFromFields(recordWithRecordType, recordWithRecordFields1); var recordWithRecord2 = FormulaValue.NewRecordFromFields(recordWithRecordType, recordWithRecordFields2); var recordWithRecord3 = FormulaValue.NewRecordFromFields(recordWithRecordType, recordWithRecordFields3); + var recordWithTable1 = FormulaValue.NewRecordFromFields(recordWithTableType, recordWithTableFields1); var t2 = FormulaValue.NewTable(recordWithRecordType, new List() { @@ -729,6 +752,26 @@ private static void MutationFunctionsTestSetup(RecalcEngine engine, bool numberI engine.UpdateVariable("rwr3", recordWithRecord3); engine.UpdateVariable("r_empty", rEmpty); + // Low code plugin testing + engine.UpdateVariable("NewRecord", r1, new SymbolProperties { CanMutate = false, CanSet = false, CanSetMutate = true }); + engine.UpdateVariable("OldRecord", r2, new SymbolProperties { CanMutate = false, CanSet = false, CanSetMutate = false }); + engine.UpdateVariable("DataSource", t1, new SymbolProperties { CanMutate = true, CanSet = false, CanSetMutate = false }); + engine.UpdateVariable("NewRecord_SetEnabled_SetMutateEnabled", r1, new SymbolProperties { CanMutate = false, CanSet = true, CanSetMutate = true }); + + // Set and Patch/Collect/Remove/etc deep mutation testing + engine.UpdateVariable("t1_SetMutateEnabled", t1, new SymbolProperties { CanMutate = false, CanSet = false, CanSetMutate = true }); + engine.UpdateVariable("t1_MutateEnabled_SetMutateEnabled", t1, new SymbolProperties { CanMutate = true, CanSet = false, CanSetMutate = true }); + engine.UpdateVariable("t1_MutateEnabled", t1, new SymbolProperties { CanMutate = true, CanSet = false, CanSetMutate = false }); + engine.UpdateVariable("r1_SetMutateEnabled", r1, new SymbolProperties { CanMutate = false, CanSet = false, CanSetMutate = true }); + engine.UpdateVariable("r1_MutateEnabled_SetMutateEnabled", r1, new SymbolProperties { CanMutate = true, CanSet = false, CanSetMutate = true }); + engine.UpdateVariable("r1_MutateEnabled", r1, new SymbolProperties { CanMutate = true, CanSet = false, CanSetMutate = false }); + engine.UpdateVariable("rwr1_SetMutateEnabled", recordWithRecord1, new SymbolProperties { CanMutate = false, CanSet = false, CanSetMutate = true }); + engine.UpdateVariable("rwr1_MutateEnabled_SetMutateEnabled", recordWithRecord1, new SymbolProperties { CanMutate = true, CanSet = false, CanSetMutate = true }); + engine.UpdateVariable("rwr1_MutateEnabled", recordWithRecord1, new SymbolProperties { CanMutate = true, CanSet = false, CanSetMutate = false }); + engine.UpdateVariable("rwt1_SetMutateEnabled", recordWithTable1, new SymbolProperties { CanMutate = false, CanSet = false, CanSetMutate = true }); + engine.UpdateVariable("rwt1_MutateEnabled_SetMutateEnabled", recordWithTable1, new SymbolProperties { CanMutate = true, CanSet = false, CanSetMutate = true }); + engine.UpdateVariable("rwt1_MutateEnabled", recordWithTable1, new SymbolProperties { CanMutate = true, CanSet = false, CanSetMutate = false }); + var valueTableType = TableType.Empty().Add("Value", numberType); var tEmpty = FormulaValue.NewTable(valueTableType.ToRecord()); var tEmpty2 = FormulaValue.NewTable(valueTableType.ToRecord()); diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/Scenarios/DataTableMarshallerProvider.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/Scenarios/DataTableMarshallerProvider.cs index 72dff283ed..d97ce9a0e2 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/Scenarios/DataTableMarshallerProvider.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/Scenarios/DataTableMarshallerProvider.cs @@ -62,7 +62,7 @@ public static RecordType ComputeType(DataTable dataTable) return recordType; } - protected override bool TryGetIndex(int index1, out DValue record) + protected override bool TryGetIndex(int index1, out DValue record, bool mutationCopy = false) { TryGetIndexNumberOfCalls++; diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/SetFunctionTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/SetFunctionTests.cs index 0ac9733d39..2d5ccb86d0 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/SetFunctionTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/SetFunctionTests.cs @@ -124,11 +124,32 @@ public void SetRecord() var r1 = engine.Eval("Set(obj, {X: 11, Y: 21}); obj.X", null, _opts); Assert.Equal(11m, r1.ToObject()); - // But SetField fails - var r2 = engine.Check("Set(obj.X, 31); obj.X", null, _opts); + // But SetField fails + var r2 = engine.Check("Set(obj.X, 31); obj.X", null, _opts); Assert.False(r2.IsSuccess); - } + } + + [Fact] + public void SetRecord_CanMutate() + { + var config = new PowerFxConfig(); + config.EnableSetFunction(); + var engine = new RecalcEngine(config); + var cache = new TypeMarshallerCache(); + var obj = cache.Marshal(new { X = 10m, Y = 20m }); + + engine.UpdateVariable("obj", obj, new SymbolProperties { CanSet = true, CanSetMutate = true }); + + // Can update record + var r1 = engine.Eval("Set(obj, {X: 11, Y: 21}); obj.X", null, _opts); + Assert.Equal(11m, r1.ToObject()); + + // Can deep mutate record + var r2 = engine.Eval("Set(obj.X, 31); obj.X", null, _opts); + Assert.Equal(31m, r2.ToObject()); + } + [Fact] public void SetRecordFloat() { @@ -145,9 +166,30 @@ public void SetRecordFloat() var r1 = engine.Eval("Set(obj, {X: Float(11), Y: Float(21)}); obj.X", null, _opts); Assert.Equal(11.0, r1.ToObject()); - // But SetField fails - var r2 = engine.Check("Set(obj.X, Float(31)); obj.X", null, _opts); + // But SetField fails + var r2 = engine.Check("Set(obj.X, 31); obj.X", null, _opts); Assert.False(r2.IsSuccess); + } + + [Fact] + public void SetRecordFloat_SetMutate() + { + var config = new PowerFxConfig(); + config.EnableSetFunction(); + var engine = new RecalcEngine(config); + + var cache = new TypeMarshallerCache(); + var obj = cache.Marshal(new { X = 10f, Y = 20f }); + + engine.UpdateVariable("obj", obj, new SymbolProperties { CanSet = true, CanSetMutate = true }); + + // Can update record + var r1 = engine.Eval("Set(obj, {X: Float(11), Y: Float(21)}); obj.X", null, _opts); + Assert.Equal(11.0, r1.ToObject()); + + // Can deep mutate record + var r2 = engine.Eval("Set(obj.X, Float(31)); obj.X", null, _opts); + Assert.Equal(31.0, r2.ToObject()); } // Test various failure cases diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/SymbolValueTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/SymbolValueTests.cs index 079481ab6b..88892ae47b 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/SymbolValueTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/SymbolValueTests.cs @@ -795,11 +795,15 @@ public void DeferredSymbolTest() } [Theory] - [InlineData(true, true)] - [InlineData(true, false)] - [InlineData(false, true)] - [InlineData(false, false)] - public void MutationTests(bool canMutate, bool canSet) + [InlineData(true, true, true)] + [InlineData(true, false, true)] + [InlineData(false, true, true)] + [InlineData(false, false, true)] + [InlineData(true, true, false)] + [InlineData(true, false, false)] + [InlineData(false, true, false)] + [InlineData(false, false, false)] + public void MutationTests(bool canMutate, bool canSet, bool canSetMutate) { var recordType = RecordType.Empty().Add("Field1", FormulaType.Number); var tableType = recordType.ToTable(); @@ -808,7 +812,8 @@ public void MutationTests(bool canMutate, bool canSet) symbols.AddVariable("var", tableType, new SymbolProperties { CanMutate = canMutate, - CanSet = canSet + CanSet = canSet, + CanSetMutate = canSetMutate }); var config = new PowerFxConfig(); @@ -823,9 +828,11 @@ public void MutationTests(bool canMutate, bool canSet) var checkSet = engine.Check("Set(var, Table({ Field1 : 123}))", opts, symbols); var checkMutate = engine.Check("Collect(var, { Field1 : 123})", opts, symbols); + var checkSetMutate = engine.Check("Set( First(var).Field1, 123 )", opts, symbols); Assert.Equal(canSet, checkSet.IsSuccess); Assert.Equal(canMutate, checkMutate.IsSuccess); + Assert.Equal(canSetMutate, checkSetMutate.IsSuccess); } // Get a convenient string representation of a SymbolValue diff --git a/src/tests/Microsoft.PowerFx.TexlFunctionExporter.Shared/YamlTexlFunction.cs b/src/tests/Microsoft.PowerFx.TexlFunctionExporter.Shared/YamlTexlFunction.cs index d8562da80b..14a5c547ff 100644 --- a/src/tests/Microsoft.PowerFx.TexlFunctionExporter.Shared/YamlTexlFunction.cs +++ b/src/tests/Microsoft.PowerFx.TexlFunctionExporter.Shared/YamlTexlFunction.cs @@ -104,7 +104,6 @@ internal YamlTexlFunction(TexlFunction texlFunction, bool isLibrary) IsVariadicFunction = texlFunction.IsVariadicFunction; ManipulatesCollections = texlFunction.ManipulatesCollections; ModifiesValues = texlFunction.ModifiesValues; - MutatesArg0 = texlFunction.MutatesArg0; PropagatesMutability = texlFunction.PropagatesMutability; RequireAllParamColumns = texlFunction.RequireAllParamColumns; RequiresDataSourceScope = texlFunction.RequiresDataSourceScope; @@ -167,7 +166,6 @@ internal YamlTexlFunction(TexlFunction texlFunction, bool isLibrary) public bool IsVariadicFunction; public bool ManipulatesCollections; public bool ModifiesValues; - public bool MutatesArg0; public bool PropagatesMutability; public bool RequireAllParamColumns; public bool RequiresDataSourceScope; From 9ef5901c0b5760e5ce947059d5956c8f183c8130 Mon Sep 17 00:00:00 2001 From: Mike Stall Date: Thu, 6 Jun 2024 04:33:24 -0700 Subject: [PATCH 26/29] Restore STJ nuget version to 6.0.8 (#2462) #2442 bumped System.Text.Json nuget from to 6.0.9. But not all consumers are updated. Restore back to 6.0.8 --- .../Microsoft.PowerFx.Connectors.csproj | 2 +- .../Microsoft.PowerFx.Json/Microsoft.PowerFx.Json.csproj | 2 +- .../Microsoft.PowerFx.LanguageServerProtocol.csproj | 2 +- .../Microsoft.PowerFx.Core.Tests.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Microsoft.PowerFx.Connectors.csproj b/src/libraries/Microsoft.PowerFx.Connectors/Microsoft.PowerFx.Connectors.csproj index c1d7118aa2..cf2eb8ebd4 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Microsoft.PowerFx.Connectors.csproj +++ b/src/libraries/Microsoft.PowerFx.Connectors/Microsoft.PowerFx.Connectors.csproj @@ -23,7 +23,7 @@ - + diff --git a/src/libraries/Microsoft.PowerFx.Json/Microsoft.PowerFx.Json.csproj b/src/libraries/Microsoft.PowerFx.Json/Microsoft.PowerFx.Json.csproj index dc7f1c6304..5fa461f201 100644 --- a/src/libraries/Microsoft.PowerFx.Json/Microsoft.PowerFx.Json.csproj +++ b/src/libraries/Microsoft.PowerFx.Json/Microsoft.PowerFx.Json.csproj @@ -19,7 +19,7 @@ - + diff --git a/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Microsoft.PowerFx.LanguageServerProtocol.csproj b/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Microsoft.PowerFx.LanguageServerProtocol.csproj index 027db61ac9..9405b205c5 100644 --- a/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Microsoft.PowerFx.LanguageServerProtocol.csproj +++ b/src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Microsoft.PowerFx.LanguageServerProtocol.csproj @@ -17,7 +17,7 @@ - + diff --git a/src/tests/.Net4.6.2/Microsoft.PowerFx.Core.Tests/Microsoft.PowerFx.Core.Tests.csproj b/src/tests/.Net4.6.2/Microsoft.PowerFx.Core.Tests/Microsoft.PowerFx.Core.Tests.csproj index b4b3b38313..e1acb9682d 100644 --- a/src/tests/.Net4.6.2/Microsoft.PowerFx.Core.Tests/Microsoft.PowerFx.Core.Tests.csproj +++ b/src/tests/.Net4.6.2/Microsoft.PowerFx.Core.Tests/Microsoft.PowerFx.Core.Tests.csproj @@ -11,7 +11,7 @@ - + From 1be2479bfa38912437195e816297e21ad48ce24b Mon Sep 17 00:00:00 2001 From: Mike Stall Date: Fri, 7 Jun 2024 10:53:38 -0700 Subject: [PATCH 27/29] Move IMutationCopyField to RecordValue so that other implementations can handle it. (#2467) --- .../Public/Values/InMemoryRecordValue.cs | 4 +-- .../Public/Values/RecordValue.cs | 28 ++++++++++--------- .../EvalVisitor.cs | 8 ++---- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Values/InMemoryRecordValue.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Values/InMemoryRecordValue.cs index 42d867f83f..076b97f54f 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Values/InMemoryRecordValue.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Values/InMemoryRecordValue.cs @@ -12,7 +12,7 @@ namespace Microsoft.PowerFx.Types { // Represent record backed by known list of values. - internal class InMemoryRecordValue : RecordValue, IMutationCopyField + internal class InMemoryRecordValue : RecordValue { protected readonly IReadOnlyDictionary _fields; private readonly IDictionary _mutableFields; @@ -46,7 +46,7 @@ public InMemoryRecordValue(InMemoryRecordValue orig) { } - void IMutationCopyField.ShallowCopyFieldInPlace(string fieldName) + public override void ShallowCopyFieldInPlace(string fieldName) { if (_fields.TryGetValue(fieldName, out FormulaValue result)) { diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/Values/RecordValue.cs b/src/libraries/Microsoft.PowerFx.Core/Public/Values/RecordValue.cs index 607fae3904..044ff0d433 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/Values/RecordValue.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/Values/RecordValue.cs @@ -364,22 +364,24 @@ public override bool TryShallowCopy(out FormulaValue copy) } public override bool CanShallowCopy => true; - } - /// - /// Copy a single record field and shallow copy contents, used during mutation copy-on-write. - /// For example: Set( aa, [[1,2,3], [4,5,6]] ); Set( ab, First(aa) ); Patch( ab.Value, {Value:2}, {Value:9}); - /// No copies are made until the mutation in Patch, and then copies are made as the first argument's - /// value is traversed through EvalVisitor: - /// 1. ab (record) shallow copies the root record and dictionary which references fields with IMutationCopy. - /// 2. .Value (field) is copied with IMutationCopyField, which shallow copies the inner table with IMutationCopy. - /// - internal interface IMutationCopyField - { /// + /// Copy a single record field and shallow copy contents, used during mutation copy-on-write. + /// For example: Set( aa, [[1,2,3], [4,5,6]] ); Set( ab, First(aa) ); Patch( ab.Value, {Value:2}, {Value:9}); + /// No copies are made until the mutation in Patch, and then copies are made as the first argument's + /// value is traversed through EvalVisitor: + /// 1. ab (record) shallow copies the root record and dictionary which references fields with IMutationCopy. + /// 2. .Value (field) is copied with IMutationCopyField, which shallow copies the inner table with IMutationCopy. + /// /// Makes a shallow copy of a field within a record, in place, and does not return the copy. - /// Earlier copies of the record will reference the original field. + /// Earlier copies of the record will reference the original field. /// - void ShallowCopyFieldInPlace(string fieldName); + public virtual void ShallowCopyFieldInPlace(string fieldName) + { + // Records that are not mutable should have been stopped by the compiler before we get here. + // But if we get here and the cast fails, the implementation of the record was not prepared for the mutation. + + throw new InvalidOperationException($"Record doesn't support copying: {this.GetType().FullName}"); + } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs b/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs index 5e256748e3..eaa832db94 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs @@ -171,9 +171,9 @@ private async Task TryHandleSet(CallNode node, EvalVisitorContext { var arg0value = await rfan.From.Accept(this, context).ConfigureAwait(false); - if (arg0value is RecordValue rv && arg0value is IMutationCopyField) + if (arg0value is RecordValue rv) { - ((IMutationCopyField)rv).ShallowCopyFieldInPlace(rfan.Field); + rv.ShallowCopyFieldInPlace(rfan.Field); rv.UpdateField(rfan.Field, newValue); return node.IRContext.ResultType._type.Kind == DKind.Boolean ? FormulaValue.New(true) : FormulaValue.NewVoid(); } @@ -702,9 +702,7 @@ public override async ValueTask Visit(RecordFieldAccessNode node, if (node.IRContext.IsMutation) { - // Records that are not mutable should have been stopped by the compiler before we get here. - // But if we get here and the cast fails, the implementation of the record was not prepared for the mutation. - ((IMutationCopyField)record).ShallowCopyFieldInPlace(node.Field.Value); + record.ShallowCopyFieldInPlace(node.Field.Value); } var val = await record.GetFieldAsync(node.IRContext.ResultType, node.Field.Value, _cancellationToken).ConfigureAwait(false); From 91881fbff21ed5347bfad0c608b21de2e06b44f9 Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Fri, 7 Jun 2024 14:52:55 -0500 Subject: [PATCH 28/29] Merging changes. --- .../Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 10ae39da61..6af52b6922 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -43,9 +43,11 @@ internal class CollectFunction : BuiltinFunction, ISuggestionAwareFunction public override bool CanSuggestInputColumns => true; - public override bool MutatesArg0 => true; + public override bool MutatesArg(int argIndex, TexlNode arg) => argIndex == 0; - public override RequiredDataSourcePermissions FunctionPermission => RequiredDataSourcePermissions.Create; + public override RequiredDataSourcePermissions FunctionPermission => RequiredDataSourcePermissions.Create; + + public override bool IsRestrictedUDFName => true; /// /// Since Arg1 and Arg2 depends on type of Arg1 return false for them. From fd98493a03fdab826f08ee65fb41fb01f3e1886e Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Thu, 13 Jun 2024 13:15:53 -0500 Subject: [PATCH 29/29] PR feedback. --- .../Texl/Builtins/Collect.cs | 58 ++++++++++++++++--- .../Microsoft.PowerFx.Core/Types/DType.cs | 2 - .../Utils/MutationUtils.cs | 39 ------------- .../Functions/LibraryMutation.cs | 8 +-- .../ParsedExpression.cs | 2 +- src/strings/PowerFxResources.en-US.resx | 10 ++-- .../TexlTests.cs | 17 ++++-- .../MutationFunctionsTests.cs | 26 --------- 8 files changed, 72 insertions(+), 90 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs index 6af52b6922..a404052247 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -17,7 +17,8 @@ using Microsoft.PowerFx.Core.Types; using Microsoft.PowerFx.Core.Utils; using Microsoft.PowerFx.Syntax; -using Microsoft.PowerFx.Types; +using Microsoft.PowerFx.Types; +using CallNode = Microsoft.PowerFx.Syntax.CallNode; using RecordNode = Microsoft.PowerFx.Core.IR.Nodes.RecordNode; namespace Microsoft.PowerFx.Core.Texl.Builtins @@ -116,7 +117,7 @@ public virtual DType GetCollectedType(Features features, DType argType) return argType; } - private bool TryGetUnifiedCollectedTypeCanvas(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType) + private bool TryGetUnifiedCollectedTypeAllowTypeExpansion(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType) { Contracts.AssertValue(args); Contracts.AssertAllValues(args); @@ -176,7 +177,7 @@ private bool TryGetUnifiedCollectedTypeCanvas(TexlNode[] args, DType[] argTypes, } // Attempt to get the unified schema of the items being collected by an invocation. - private bool TryGetUnifiedCollectedTypeV1(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType) + private bool TryGetUnifiedCollectedTypeDontAllowTypeExpansion(TexlNode[] args, DType[] argTypes, IErrorContainer errors, Features features, out DType collectedType) { Contracts.AssertValue(args); Contracts.AssertAllValues(args); @@ -265,13 +266,13 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp // Get the unified collected type on the RHS. This will generate appropriate // document errors for invalid arguments such as unsupported aggregate types. - if (context.Features.PowerFxV1CompatibilityRules && !context.AnalysisMode) + if (context.AnalysisMode) { - fValid &= TryGetUnifiedCollectedTypeV1(args, argTypes, errors, context.Features, out collectedType); + fValid &= TryGetUnifiedCollectedTypeAllowTypeExpansion(args, argTypes, errors, context.Features, out collectedType); } else { - fValid &= TryGetUnifiedCollectedTypeCanvas(args, argTypes, errors, context.Features, out collectedType); + fValid &= TryGetUnifiedCollectedTypeDontAllowTypeExpansion(args, argTypes, errors, context.Features, out collectedType); } Contracts.Assert(collectedType.IsTable); @@ -415,7 +416,46 @@ protected static string CreateInvariantFieldName(PowerFx.Features features, DKin { Contracts.Assert(dKind >= DKind._Min && dKind < DKind._Lim); - return MutationUtils.GetScalarSingleColumnNameForType(features, dKind); + return GetScalarSingleColumnNameForType(features, dKind); + } + + protected static string GetScalarSingleColumnNameForType(Features features, DKind kind) + { + return kind switch + { + DKind.Image or + DKind.Hyperlink or + DKind.Media or + DKind.Blob or + DKind.PenImage => features.ConsistentOneColumnTableResult ? TableValue.ValueName : "Url", + + _ => TableValue.ValueName + }; + } + + protected static List CreateIRCallNodeCollect(CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) + { + var newArgs = new List() { args[0] }; + + foreach (var arg in args.Skip(1)) + { + if (arg.IRContext.ResultType._type.IsPrimitive) + { + newArgs.Add( + new RecordNode( + new IRContext(arg.IRContext.SourceContext, RecordType.Empty().Add(TableValue.ValueName, arg.IRContext.ResultType)), + new Dictionary + { + { TableValue.ValueDName, arg } + })); + } + else + { + newArgs.Add(arg); + } + } + + return newArgs; } } @@ -440,7 +480,7 @@ public override DType GetCollectedType(Features features, DType argType) internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) { - return base.CreateIRCallNode(node, context, MutationUtils.CreateIRCallNodeCollect(node, context, args, scope), scope); + return base.CreateIRCallNode(node, context, CreateIRCallNodeCollect(node, context, args, scope), scope); } } @@ -486,7 +526,7 @@ public override DType GetCollectedType(PowerFx.Features features, DType argType) internal override IntermediateNode CreateIRCallNode(PowerFx.Syntax.CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) { - return base.CreateIRCallNode(node, context, MutationUtils.CreateIRCallNodeCollect(node, context, args, scope), scope); + return base.CreateIRCallNode(node, context, CreateIRCallNodeCollect(node, context, args, scope), scope); } } } diff --git a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs index 68cb925a65..37e4806f84 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs @@ -646,8 +646,6 @@ public bool IsActivityPointer public bool HasPolymorphicInfo => PolymorphicInfo != null; - public bool IsSingleColumnTable => IsTable && GetNames(DPath.Root).Count() == 1; - /// /// Whether this type is a subtype of all possible types, meaning that it can be placed in /// any location without coercion. diff --git a/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs index a476a41319..e0d2df8089 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs @@ -70,44 +70,5 @@ public static void CheckSemantics(TexlBinding binding, TexlFunction function, Te return; } } - - public static string GetScalarSingleColumnNameForType(Features features, DKind kind) - { - return kind switch - { - DKind.Image or - DKind.Hyperlink or - DKind.Media or - DKind.Blob or - DKind.PenImage => features.ConsistentOneColumnTableResult ? TableValue.ValueName : "Url", - - _ => TableValue.ValueName - }; - } - - public static List CreateIRCallNodeCollect(CallNode node, IRTranslator.IRTranslatorContext context, List args, ScopeSymbol scope) - { - var newArgs = new List() { args[0] }; - - foreach (var arg in args.Skip(1)) - { - if (arg.IRContext.ResultType._type.IsPrimitive) - { - newArgs.Add( - new RecordNode( - new IRContext(arg.IRContext.SourceContext, RecordType.Empty().Add(TableValue.ValueName, arg.IRContext.ResultType)), - new Dictionary - { - { TableValue.ValueDName, arg } - })); - } - else - { - newArgs.Add(arg); - } - } - - return newArgs; - } } } diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs index fc3242f536..c583acd946 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -236,7 +236,7 @@ internal class CollectImpl : CollectFunction, IAsyncTexlFunction3 { public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { - return await new CollectProcess().Process(irContext, args, cancellationToken).ConfigureAwait(false); + return await CollectProcess.Process(irContext, args, cancellationToken).ConfigureAwait(false); } } @@ -244,13 +244,13 @@ internal class CollectScalarImpl : CollectScalarFunction, IAsyncTexlFunction3 { public async Task InvokeAsync(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { - return await new CollectProcess().Process(irContext, args, cancellationToken).ConfigureAwait(false); + return await CollectProcess.Process(irContext, args, cancellationToken).ConfigureAwait(false); } } internal class CollectProcess { - internal async Task Process(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) + internal static async Task Process(FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken) { FormulaValue arg0; var argc = args.Length; @@ -323,7 +323,7 @@ internal async Task Process(FormulaType irContext, FormulaValue[] if (resultRows.Count == 0) { - return FormulaValue.NewBlank(arg0.Type); + return FormulaValue.NewBlank(irContext); } if (irContext._type.IsTable) diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs b/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs index 29dda48fd1..ce8ad19063 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs @@ -86,7 +86,7 @@ internal static IExpressionEvaluator GetEvaluator(this CheckResult result, Stack _globals = globals, _allSymbols = result.Symbols, _parameterSymbolTable = result.Parameters, - _additionalFunctions = result.Engine.Config.AdditionalFunctions, + _additionalFunctions = result.Engine.Config.AdditionalFunctions }; return expr; diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index eeb32789ed..9aaa0b5171 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -3146,8 +3146,8 @@ Display text representing the Contrast value of TeamsTheme enum (TeamsTheme_Contrast_Name). The possible values for this enumeration are: Default, Dark, Contrast. - Adds one or more items to the specified 'collection'. The items can be from a different table or collection (e.g. Collect(collection, source_collection)), or one or more records Collect(collection, {key1: val1, key2: val2, ...}, ...). - Description of 'Collect' function. + Adds one or more items to the specified collection. The items can be from a different table or collection (e.g. Collect(collection, source_collection)), or one or more records Collect(collection, {key1: val1, key2: val2, ...}, ...). + {Locked=Collect(collection, source_collection)} {Locked=Collect(collection, {key1: val1, key2: val2, ...}, ...)} Description of 'Collect' function. data_source @@ -3168,8 +3168,8 @@ A record or table to collect. A record will be appended to the collection. A table will have its rows appended to the collection. - Clears the collection first and adds one or more items to the specified 'collection'. The items can be from a different table or collection (e.g. ClearCollect(collection, source_collection)), or one or more records ClearCollect(collection, {key1: val1, key2: val2, ...}, ...). - Description of 'ClearCollect' function. + Clears the collection first and adds one or more items to the specified collection. The items can be from a different table or collection (e.g. ClearCollect(collection, source_collection)), or one or more records ClearCollect(collection, {key1: val1, key2: val2, ...}, ...). + {Locked=ClearCollect(collection, source_collection)} {Locked=ClearCollect(collection, {key1: val1, key2: val2, ...}, ...)} Description of 'ClearCollect' function. A new or existing collection to augment. @@ -4628,7 +4628,7 @@ It's used to determine how to combine multiple expressions with the same name and operator. - The data source that you want to add data to. + The datasource to add data to. A record or table to collect. A record will be appended to the datasource. A table will have its rows appended to the datasource. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs index ab22d08a83..dedb99e263 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs @@ -142,6 +142,16 @@ public void TexlDateTableFunctions_Float(string expression, string expectedType) [InlineData("Collect(t1, Table({a:1}))", "*[a:w, b:s]", false)] [InlineData("Collect(t2, 2)", "*[Value:w]", false)] [InlineData("Collect(t2, 2, 3, 4)", "*[Value:w]", false)] + + [InlineData("ClearCollect(t1, {a:1})", "![a:w, b:s]")] + [InlineData("ClearCollect(t1, {a:1},{a:2})", "*[a:w, b:s]")] + [InlineData("ClearCollect(t1, Table({a:1}))", "*[a:w, b:s]")] + [InlineData("ClearCollect(t2, 2, 3, 4)", "*[Value:w]")] + [InlineData("ClearCollect(t1, {a:1})", "*[a:w, b:s]", false)] + [InlineData("ClearCollect(t1, {a:1},{a:2})", "*[a:w, b:s]", false)] + [InlineData("ClearCollect(t1, Table({a:1}))", "*[a:w, b:s]", false)] + [InlineData("ClearCollect(t2, 2)", "*[Value:w]", false)] + [InlineData("ClearCollect(t2, 2, 3, 4)", "*[Value:w]", false)] public void TexlMutationFunctionsV1Tests(string expression, string expectedType, bool isPFxV1 = true) { var engine = new Engine(new PowerFxConfig(isPFxV1 ? Features.PowerFxV1 : Features.None)); @@ -152,10 +162,9 @@ public void TexlMutationFunctionsV1Tests(string expression, string expectedType, engine.Config.SymbolTable.AddFunction(new CollectFunction()); engine.Config.SymbolTable.AddFunction(new CollectScalarFunction()); - engine.Config.SymbolTable.AddFunction(new PatchFunction()); - engine.Config.SymbolTable.AddFunction(new PatchSingleRecordFunction()); - engine.Config.SymbolTable.AddFunction(new PatchAggregateFunction()); - engine.Config.SymbolTable.AddFunction(new PatchAggregateSingleTableFunction()); + engine.Config.SymbolTable.AddFunction(new ClearCollectFunction()); + engine.Config.SymbolTable.AddFunction(new ClearCollectScalarFunction()); + engine.Config.SymbolTable.AddVariable("t1", FormulaType.Build(expectedDType), mutable: true); engine.Config.SymbolTable.AddVariable("t2", FormulaType.Build(expectedDTypeScalar), mutable: true); diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationFunctionsTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationFunctionsTests.cs index 40d9f0969a..7727a8b9c4 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationFunctionsTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests.Shared/MutationFunctionsTests.cs @@ -389,32 +389,6 @@ public void SymbolTableEnableMutationFuntionsTest() Assert.Contains(checkEnabled.Symbols.Functions.FunctionNames, f => f == "Collect"); } - [Theory] - [InlineData("Collect(t, {Value:1})")] - [InlineData("Collect(t, 1)")] - [InlineData("ForAll(Sequence(5), Collect(t, {Value:ThisRecord.Value}))")] - [InlineData("Switch(2,1,Last(Collect(t,{Value:1})).Value,2,Last(Collect(t,{Value:2})).Value,3,Last(Collect(t,{Value:3})).Value)")] - [InlineData("IfError(1/0,Last(Collect(t,{Value:3})).Value)")] - public void CollectPFxV1Disabled(string expression) - { - var engine = new RecalcEngine(new PowerFxConfig(Features.None)); - var t = FormulaValue.NewTable(RecordType.Empty().Add(new NamedFormulaType("Value", FormulaType.Decimal))); - - engine.Config.SymbolTable.EnableMutationFunctions(); - engine.UpdateVariable("t", t); - - var check = engine.Check(expression, options: new ParserOptions() { AllowsSideEffects = true }); - - // Compilation will be successful, but the function will not be executed. - // This is because PA depends on the CheckType to determine if the function is valid. - Assert.True(check.IsSuccess); - - var evaluator = check.GetEvaluator(); - - // Runtime exception - Assert.ThrowsAsync(async () => await evaluator.EvalAsync(CancellationToken.None).ConfigureAwait(false)).ConfigureAwait(true); - } - [Theory] [InlineData("Patch(t, First(t), {Value:1})")] public async Task MutationPFxV1Disabled(string expression)