From 67607d83b6f4d9504bacab44269b5545364dee24 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Mon, 10 Feb 2025 23:24:42 +0000 Subject: [PATCH 1/9] Update compiled test_functions_py3.3.4 to match input file. Compiled with the 3.4.10-alpine container --- tests/compiled/test_functions_py3.3.4.pyc | Bin 1546 -> 2313 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/compiled/test_functions_py3.3.4.pyc b/tests/compiled/test_functions_py3.3.4.pyc index 6c0fe16ae2f2cfd256cb14efcf851f0f8ebd564e..1149970a68a1749ee9cdc9a77cd4821464498bbb 100644 GIT binary patch delta 547 zcmZXQ%SyvQ6o${V31UrKO)u#sX{){1w9+Dof(ud6r3G=9Wpki(<03wQ6uPlHIS=7d z&}R^Q48gs90YT49tqe3Ff6jdWnSU-b>%ne(S;@r7XHjR{b%t#oF8#u) zA9gjrxZ&MCdmT1x3%w?^(&9BsSJooqi1@m57*8l(j delta 209 zcmeAa>f%v($HU9D_28V?dyEVWj~S2v3y|#q#Kor=CaSb}@i3%_GNgzxv@kG6aWbTc zFr)xQ#2Hc~z#?2Q5lMy=DX<7POhlR?MFuPaR4>aAtSL7+j?tJ?PESuiGq0euWb+Kh zKTJ$T{F5iLgiN+(b)BrmDl~Ztn+s#&$-eB4B8jRXXD|T?2F4&~GgqT4P02{q3C;$Ke From 3400283f1252c475208ff4d06ca1be4791e73ac4 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Mon, 10 Feb 2025 23:26:22 +0000 Subject: [PATCH 2/9] Update MAKE_FUNCTION to pull args in correct order for py3.4 --- ASTree.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ASTree.cpp b/ASTree.cpp index 1f419d0c9..943872282 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -1565,12 +1565,21 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) ASTFunction::defarg_t defArgs, kwDefArgs; const int defCount = operand & 0xFF; const int kwDefCount = (operand >> 8) & 0xFF; - for (int i = 0; i < defCount; ++i) { - defArgs.push_front(stack.top()); - stack.pop(); + const int annotationCount = (operand >> 16) & 0x7FFF; + + if(annotationCount) { + stack.pop(); // Tuple of param names for annotations + for (int i = 0; i < annotationCount; ++i) { + stack.pop(); // Pop annotation objects and ignore + } } for (int i = 0; i < kwDefCount; ++i) { kwDefArgs.push_front(stack.top()); + stack.pop(); // KW Pair object + stack.pop(); // KW Pair name + } + for (int i = 0; i < defCount; ++i) { + defArgs.push_front(stack.top()); stack.pop(); } stack.push(new ASTFunction(fun_code, defArgs, kwDefArgs)); From 64a9b831511e8a32421bdd18bbea050df45f918f Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Mon, 10 Feb 2025 23:29:41 +0000 Subject: [PATCH 3/9] Update compiled test_functions_py3.3.0 to match input file. Compiled with 3.0.1 locally built container --- tests/compiled/test_functions_py3.3.0.pyc | Bin 2099 -> 3121 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/compiled/test_functions_py3.3.0.pyc b/tests/compiled/test_functions_py3.3.0.pyc index f29f671488bfbda506dbd65b4e3d02494fa5da8d..417eb709fbb3b8c3f3d60068533b0fb673a63213 100644 GIT binary patch literal 3121 zcmd5;O>5gg5FJ@|Vmo&1*lF5Ap-^Zs6q4GJENDw8_|QWkJ>=#~kam+GlG?$tXmTw0 zzw|%$)S1z)rMgm5RgidvcD{D@&A#0g`p{_BkAD2T5RhnP)IZ@H{K8HGEC9x+7_1l| zORSVYnyi>0%dC_^R#>Tk+yJ==vI>h5j8Tuw8k^{k%sQJ@kXxKp1KHqE6HQ}W*XA~x zTOeCJ497&n+U##~$_^|H7;keZn!ke{W84DS<*c^O+U2Yr&U(UGokZ3iXLUh7g_GVh zT>c-#80Z)gF~krObJ8QvaSuCK>B1N2JMbscpB>E4g8AurI1zJyItfm%Zx8pcZ^IG} zxOqnt1f+r?NT)`U-b^S8hE#()yXeQj|A^VAOpVFVXp_7XlLjm?wKB*JKG;lvqsmP$ z_%m9m+$I+s(T?$bT8&S03*v+tt(m-;tkZyyMxRfoEIoG?$tzc%fc&&rZWpK2w5%4Bxf^#D zQ-CGQ+e;~@fvEbXH)?0*bTf9oZ6UFpu_z}g@=w8Anu1Id$hwkgg!}h3LeH$Nm z-Qw-zVaxp#Y&uB9LFRFz0e@Zl0$(4;?XN`{p0yTb2xl$IkniI%j(fXksK2#H^dm9g zgMfb*N_t9)NLtEoDZi!sw({G`Z!7;$`G?9s)P5Pw=cu@&;*N?tGXCJOcK>f+6<74} e+f;^EXGe70gE!dK%=l@U^wgGp!|brtFn%lp($qYz<8OU}3;^M^<8 zkacn)6NYM;$xoRA(N)V#j$w|%6uiS+h$&dil8GtE#9D+YIE$5cvNW3~rUV}w@8s!h zK9~|+Y Date: Mon, 10 Feb 2025 23:49:26 +0000 Subject: [PATCH 4/9] Update tokenized test_functions_py3.txt to match input file. --- tests/tokenized/test_functions_py3.txt | 36 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/tests/tokenized/test_functions_py3.txt b/tests/tokenized/test_functions_py3.txt index 8308e0b70..b9d25db24 100644 --- a/tests/tokenized/test_functions_py3.txt +++ b/tests/tokenized/test_functions_py3.txt @@ -34,18 +34,46 @@ def x4c ( foo , bar = 1 , bla = 2 , * args , ** kwargs ) : pass -def x5a ( * , bar = 1 ) : +def x5a ( * , bar ) : pass -def x5b ( * , bar = 1 , ** kwargs ) : +def x5b ( * , bar = 1 ) : pass -def x6a ( foo , * , bar = 1 ) : +def x5c ( * , bar = 1 , ** kwargs ) : pass -def x7a ( foo , * , bar = 1 , ** kwargs ) : +def x6a ( foo , * , bar ) : + +pass + +def x6b ( foo , * , bar = 1 ) : + +pass + +def x6c ( foo = 1 , * , bar ) : + +pass + +def x6d ( foo = 1 , * , bar = 2 ) : + +pass + +def x7a ( foo , * , bar , ** kwargs ) : + +pass + +def x7b ( foo , * , bar = 1 , ** kwargs ) : + +pass + +def x7c ( foo = 1 , * , bar , ** kwargs ) : + +pass + +def x7d ( foo = 1 , * , bar = 2 , ** kwargs ) : pass From 040a766cc735b8ac5223af6da8d4d549d6289aa5 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Mon, 10 Feb 2025 23:51:49 +0000 Subject: [PATCH 5/9] Fix MAKE_FUNCTION for py3.0-3.3 --- ASTree.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ASTree.cpp b/ASTree.cpp index 943872282..6f0f1145b 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -1567,6 +1567,19 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) const int kwDefCount = (operand >> 8) & 0xFF; const int annotationCount = (operand >> 16) & 0x7FFF; + /* Docs for 3.3 say KW Pairs come first, but reality disagrees */ + if (mod->verCompare(3, 3) <= 0) { + for (int i = 0; i < defCount; ++i) { + defArgs.push_front(stack.top()); + stack.pop(); + } + /* KW Defaults not mentioned in docs, but they come after the positional args */ + for (int i = 0; i < kwDefCount; ++i) { + kwDefArgs.push_front(stack.top()); + stack.pop(); // KW Pair object + stack.pop(); // KW Pair name + } + } else { if(annotationCount) { stack.pop(); // Tuple of param names for annotations for (int i = 0; i < annotationCount; ++i) { @@ -1582,6 +1595,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) defArgs.push_front(stack.top()); stack.pop(); } + } stack.push(new ASTFunction(fun_code, defArgs, kwDefArgs)); } break; From d8d86e9dd9ce69a74dd0f47dd0d62ca8c2cd8aa1 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 11 Feb 2025 00:00:52 +0000 Subject: [PATCH 6/9] Fix MAKE_FUNCTION indent --- ASTree.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ASTree.cpp b/ASTree.cpp index 6f0f1145b..395771419 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -1580,21 +1580,21 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.pop(); // KW Pair name } } else { - if(annotationCount) { - stack.pop(); // Tuple of param names for annotations - for (int i = 0; i < annotationCount; ++i) { - stack.pop(); // Pop annotation objects and ignore + if(annotationCount) { + stack.pop(); // Tuple of param names for annotations + for (int i = 0; i < annotationCount; ++i) { + stack.pop(); // Pop annotation objects and ignore + } + } + for (int i = 0; i < kwDefCount; ++i) { + kwDefArgs.push_front(stack.top()); + stack.pop(); // KW Pair object + stack.pop(); // KW Pair name + } + for (int i = 0; i < defCount; ++i) { + defArgs.push_front(stack.top()); + stack.pop(); } - } - for (int i = 0; i < kwDefCount; ++i) { - kwDefArgs.push_front(stack.top()); - stack.pop(); // KW Pair object - stack.pop(); // KW Pair name - } - for (int i = 0; i < defCount; ++i) { - defArgs.push_front(stack.top()); - stack.pop(); - } } stack.push(new ASTFunction(fun_code, defArgs, kwDefArgs)); } From ad675e500341f617c7beabdd594b6b1e1e247082 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 11 Feb 2025 23:05:14 +0000 Subject: [PATCH 7/9] Add additional test case for multiple kwargs in Python3 function --- tests/compiled/test_functions_py3.3.0.pyc | Bin 3121 -> 3311 bytes tests/compiled/test_functions_py3.3.4.pyc | Bin 2313 -> 2453 bytes tests/input/test_functions_py3.py | 3 +++ tests/tokenized/test_functions_py3.txt | 4 ++++ 4 files changed, 7 insertions(+) diff --git a/tests/compiled/test_functions_py3.3.0.pyc b/tests/compiled/test_functions_py3.3.0.pyc index 417eb709fbb3b8c3f3d60068533b0fb673a63213..56b7c9f80d33c304ed0af97836d2ef0b7c47cf97 100644 GIT binary patch delta 166 zcmdle@m^BFnunLm>e}k`WCkR_24p(`aj_8NM1`NSJPax93@PFaDH04RYz!%q3@waI zQ6dZ}QVhWw(wpTNS-8YYnSsia600&nbTT85U;<*6$qU&OCd;x&F(p<_R$#UfFi*7w s3KxUq7#RIDL?^3qFW?31tuRm30Ete%$f7g(8@C{%#$*m2O(r2`06Co;mH+?% delta 69 zcmaDaxluyFnunLmu5o#KG6ND|0kR!{xcC>tM1`NM;tatW5}VB!S-6<}G(;xfe}jbEoKIW#|%h-9msY7;^M%K%4UoTJPavf3@N+}DcTGvIt(eg z3@LgHEsRW2A`B_|48fWPo8uV2vx(edPD-qL2{ei!nGr}Z0Wr&DKXzGOmLfhNyNDk| u$WK1UE-jL33uG6AR4_0WDFR79P0`5_9I1?olXr5cF=|YHz@f<`$_xP4GafMj delta 79 zcmbO#+$p5|j)#}au5o$#dnN{k#|%h-707k~;$pLn%4UqL+6=*(I-4sQzq2v>X^Kp~ X%aO{+IN6X*cycr+AEVmjEKW@TnxYW7 diff --git a/tests/input/test_functions_py3.py b/tests/input/test_functions_py3.py index f11c75c78..502c13f73 100644 --- a/tests/input/test_functions_py3.py +++ b/tests/input/test_functions_py3.py @@ -57,3 +57,6 @@ def x7c(foo = 1, *, bar, **kwargs): def x7d(foo = 1, *, bar = 2, **kwargs): pass + +def x7e(foo = 1, *, bar = 2, baz = 3, **kwargs): + pass diff --git a/tests/tokenized/test_functions_py3.txt b/tests/tokenized/test_functions_py3.txt index b9d25db24..0cfae7a5d 100644 --- a/tests/tokenized/test_functions_py3.txt +++ b/tests/tokenized/test_functions_py3.txt @@ -77,3 +77,7 @@ pass def x7d ( foo = 1 , * , bar = 2 , ** kwargs ) : pass + +def x7e ( foo = 1 , * , bar = 2 , baz = 3 , ** kwargs ) : + +pass From f89a1a54c2388573f078e05efb3bb6734ecc3564 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 11 Feb 2025 23:21:19 +0000 Subject: [PATCH 8/9] Update compiled test_functions_py3.3.7 to match input file & remove from xfail. --- .../test_functions_py3.3.7.pyc | Bin 2289 -> 2432 bytes 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{xfail => compiled}/test_functions_py3.3.7.pyc (63%) diff --git a/tests/xfail/test_functions_py3.3.7.pyc b/tests/compiled/test_functions_py3.3.7.pyc similarity index 63% rename from tests/xfail/test_functions_py3.3.7.pyc rename to tests/compiled/test_functions_py3.3.7.pyc index 763ba8e6b55cd317a4fde46ed52e2807e30412cf..b059fba904c90a0f499575f6212a9f31bde0dbe1 100644 GIT binary patch delta 237 zcmew;*dWa3#LLUY00dUoR;Oz*Pvnzed@)hoMARUKFNME_F^Vn4Fhw9m5XfRrF`8K5 z%x#<^pP~Td@}-ze+^1@7nxdYf0ptp$n5FQhh|OV4(M-{5VU7|`F$eOsQ*`DqrRb*U z0YyYo^n)2REs{Z&fx+bWtm2cE8HLzo^fU8PGK)6bG45xRe#s1!)MP5+2U53~lM<_v z8G$S&AZD2y$Sym14l@sq8-ejX`Bo)8Vn}? vXBD4p$|%Gsr>Cc%nO9I+ve}PuKilLw4kt#D$tO5cCmV4pF{(`t< Date: Tue, 11 Feb 2025 23:42:22 +0000 Subject: [PATCH 9/9] Update MAKE_FUNCTION to handle args based on the new flags for py3.6-12 --- ASTree.cpp | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/ASTree.cpp b/ASTree.cpp index 395771419..33efd8661 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -1579,7 +1579,9 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.pop(); // KW Pair object stack.pop(); // KW Pair name } - } else { + } else if (mod->verCompare(3, 5) <= 0) { + /* From Py 3.4 there can now be annotation params + The order has also switched so Kwargs come before Pos Args */ if(annotationCount) { stack.pop(); // Tuple of param names for annotations for (int i = 0; i < annotationCount; ++i) { @@ -1595,6 +1597,35 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) defArgs.push_front(stack.top()); stack.pop(); } + } else { + /* From Py 3.6 the operand stopped being an argument count + and changed to a flag that indicates what is represented by + preceding tuples on the stack. Docs for 3.7 are clearer, + docs for 3.6 may have not been correctly updated */ + if(operand & 0x08) { // Cells for free vars to create a closure + stack.pop(); // Ignore these for syntax generation + } + if(operand & 0x04) { // Annotation dict (3.6-9) or string (3.10+) + stack.pop(); // Ignore annotations + } + if(operand & 0x02) { // Kwarg Defaults + PycRef kw_tuple = stack.top(); + stack.pop(); + std::vector> kw_values = kw_tuple.cast()->values(); + + for(const PycRef& kw : kw_values) { + kwDefArgs.push_front(kw); + } + } + if(operand & 0x01) { // Positional Defaults (including positional-or-KW args) + PycRef pos_tuple = stack.top(); + stack.pop(); + std::vector> pos_values = pos_tuple.cast()->object().cast()->values(); + + for(const PycRef& pos : pos_values) { + defArgs.push_back(new ASTObject(pos)); + } + } } stack.push(new ASTFunction(fun_code, defArgs, kwDefArgs)); }