Merge branch 'j5'

Author: matthewfl

.github/workflows/run-tests.yml

@@ -6,7 +6,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: run tests
       run: |
-        timeout 15m make test
+        timeout 15m make github-test

Makefile

@@ -3,7 +3,7 @@ JAVA ?= java
 
 # this version should match the version in project.clj as that is what is going to be built by lein
 JVERSION:= 0.1.0
-VERSION= $(shell date '+%Y%m%d')
+VERSION:= $(shell git describe --always --long --abbrev=12)-$(shell date '+%Y%m%d')
 
 SOURCE=$(wildcard src/*/*/*.clj) $(wildcard src/*/*/*.java)
 JAR_TARGET=target/dyna-$(JVERSION)-SNAPSHOT-standalone.jar
@@ -22,11 +22,15 @@ clean:
 	rm -rf target/ dyna-standalone-* python_build/
 
 test:
-	_JAVA_OPTIONS='-Ddyna.debug=false -Ddyna.trace_rexpr_construction=false -Xss8m' $(LEIN) test
+	_JAVA_OPTIONS='-Ddyna.debug=false -Ddyna.trace_rexpr_construction=false -Ddyna.debug_repl=false -Xss8m -ea' $(LEIN) test
 
 test-debug:
 	$(LEIN) test
 
+# some of the tests are failing randomly or somehow failing depending on the order in which they run.  Trying to fix this, but annoying right now with github running of the tests failing
+github-test:
+	_JAVA_OPTIONS='-Ddyna.debug=false -Ddyna.trace_rexpr_construction=false -Ddyna.debug_repl=false -Xss8m -ea' $(LEIN) test || 	_JAVA_OPTIONS='-Ddyna.debug=false -Ddyna.trace_rexpr_construction=false -Ddyna.debug_repl=false -Xss8m -ea' $(LEIN) retest
+
 # start the repl for dyna code from the source directory
 repl: dyna-repl
 dyna-repl: $(PARSER_TARGET)

README.md

@@ -16,12 +16,12 @@ make
 
 A recent version of java is required.  Running `make` should automattically
 download all dependencies.  The dyna runtime will be compiled into
-`./dyna-standalone-*`.
+`./dyna-standalone-*.run`.
 
 ## Running dyna
 ```
 make
-./dyna-standalone-*
+./dyna-standalone-*.run
 ```
 
 ## Running tests

bugs.org

@@ -0,0 +1,94 @@
+* bugs
+1. run-iterator is calling simplify after it binds each variable.  This is
+   causing the R-exprs to perform a lot of rewriting, hence be slow.  This needs
+   a "fast fail" flag which will attempt to just to simplification operations
+   which can quickly get done without causing too much rewriting.  Even if we
+   keep going with an R-expr which is equivalent to 0 (without known) it will
+   still be correct, but we will just waste time on additional inner loops
+   - this is now setting a fast fail flag, which blocks some operations such as
+     expanding user calls.  This should at least prevent those from running and
+     getting the result which corresponds with an error
+
+2. I think that variable renaming is causing the quicksort program to fail atm
+   - when one of the disjuncts causes it to unify with an expression, it will
+     result in it finding that there is some result which handle that value.
+
+3. factorial program is not terminating atm, but somehow pressing Ctrl-C is able
+   to cause it to stop and return the right answer?  Now sure why it would get
+   interrupted and then somehow get the right answer.  Though I suppose this
+   could be something with backchaining going on for too long?  It would have
+   that there is something that might cause it to run for too long.
+   - pressing Ctrl-C will set the thread interrupted flag, which stops
+     simplification from continuing to run in a loop.
+   - now this is just not stopping anymore, so seems like it is doing too many
+     steps of simplification.
+
+** bugs fixed
+1. Iterator when a variable is already bound to a constant, need to skip ahread in the iterator sequence
+2. ~$free~ when there is some value used in ~$memo~, the value will be passed.
+   This should really just behave more like a no-op in this case.
+2. disjunct-op needs to allow for taking inner disjuncts which are returned and
+   merging it into its internal representation.
+   - this issue can be provoked by setting the simplify method of run-iterator
+     to the identity (disabling simplify during each step of the iteration)
+2. Macros are not subsuiting the variables in correctly it seems
+
+** bugs deferred
+1. make-memoization-controller-function is supposed to generate efficient
+   memoization controller functions which do not have to call into R-exprs every
+   time to evaluate the policy.  This seems to currently be buggy.
+   - going to just use the R-expr representation every time.  Will eventually
+     tie in the JIT such that it should get a good generated version of the
+     code, so it hopefully will not be too bad.
+
+* enhancements
+1. disjunct-op should know information about what is being stored.  There are
+   currently a lot of calls to perform rewriting/simplification, and if we knew
+   that there was nothing but multiplicies inside of the trie (meaning it only
+   contains ground values), then it would be able to short cut a lot of the operations.
+2. disjunct-op is being called with simplify many times where the variable
+   bindings are unlikely to change that much.  We should save the current
+   bindings into the variable-slots and then will avoid redoing a simplification
+   of the trie in the case that there are no additional variables which are
+   ground.
+
+3. the clojure ^:dynamic is very slower, as it indirects the reads through a
+   clojure hash map.  Ideally convert those to a java ThreadLocal which should
+   be able to use an array access and be efficient.  However, this will not play
+   nicely with the
+
+
+* further out enhancements
+1. JIT compiler
+   - integrate into the looping such that it will use it to perform the
+     simplification for a given value of an expression.
+
+2. memoization point wise update when a user-definition is changed.  This needs
+   to not rebuild the memo tables from scratch, as that is going to be slow.
+
+
+3. should track what the user call stack is for different operations.  Currently
+   it is too hard to figure out where something is going wrong.  if it tracked
+   the call stack, then it would be able to figure out which of the operations
+   will correspond to it getting some result.  But the call stack is going to
+   become more complicated when it has different things like memoization
+   involved, as those call stack sequence will not really correspond with
+   anything.
+   - I suppose that debugging the program could also be done by adding
+     additional rules to track stuff, but not sure if that will work out
+     correctly either?  I think that debugging the programs is just going to
+     difficult in general given the number of things which are getting combined
+     together to make a final result.  If everything is "simple" meaning that it
+     is all ground values, then there is not that much of an issue, but once
+     there are complex delayed constraints, those constraints are going to cause
+     it to get some result which can only happen in the exact case where all of
+     those delayed constraints combine together to make some inference.
+
+
+* working on a large relational system
+https://news.ycombinator.com/item?id=18442941
+- oracle database where new flags are added to handle the different use cases
+https://www.postgresql.org/message-id/flat/31cc6df9-53fe-3cd9-af5b-ac0d801163f4%40iki.fi
+- postgres has around 1666 variables which are "global" to a connection to tack
+  various aspects of the session state.  Conceptually similar to the case we
+  have here

dyna

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# this runs the dyna repl without having to generate the jar first
+# this runs the dyna repl
 
 make
 exec java -cp `make run-class-path` dyna.DynaMain "$@"

dyna_python_module/dyna/__init__.py

@@ -12,9 +12,18 @@
 class Dyna:
 
     def __init__(self):
+        self.__system_v = None
+
+    @property
+    def __system(self):
+        v = self.__system_v
+        if v is not None:
+            return v
         # this will reference the python class which wrap the java class.  This will block until the dyna runtime is loaded
         from .java_wrapper import DynaInstance
-        self.__system = DynaInstance()
+        v = DynaInstance()
+        self.__system_v = v
+        return v
 
     def define_function(self, name=None, arity=None):
         """

dyna_python_module/dyna/start_dyna_repl.py

@@ -5,6 +5,7 @@ def start_dyna_repl():
     jar = importlib.resources.path('dyna', 'dyna.jar')
     if not os.path.exists(jar):
         print('Unable to find dyna runtime')
+        sys.exit(1)
     os.execv(jar, sys.argv)
 
 if __name__ == '__main__':

project.clj

@@ -5,6 +5,7 @@
   :dependencies [[org.clojure/clojure "1.11.1" ;"1.12.0-mfl-SNAPSHOT"
                   ]
                  [org.clojure/tools.namespace "1.2.0"]
+                 ;[org.ow2.asm/asm "9.5"]
                  ;[org.clojure/tools.macro "0.1.5"]
                  [aprint "0.1.3"]  ;; for formatted printing
                  ;[clj-python/libpython-clj "2.00-beta-22"]
@@ -23,10 +24,6 @@
                  ;[reply "0.5.1"]
                  ]
   :repl-options {:init-ns dyna.core}
-  ;; :aot [dyna.interface
-  ;;       dyna.parser_interface]
-  ;;:aot :all
-  ;;:aot [dyna.rexpr]
   :source-paths ["src/clojure"]
   :java-source-paths ["target/gen-src" "src/java"]
   :resource-paths ["src/resources"]
@@ -38,15 +35,13 @@
   :antlr-src-dir "src/antlr"
   :antlr-dest-dir "target/gen-src"
 
-  :profiles {:uberjar {:main dyna.DynaMain
-                       ;; clojure only requires java 8, and we shouldn't need any of the "newer" features ourselves also
-                       ;; so this should make this work with any version released after 1.8 (march 2014)
-                       :javac-options ["-source" "1.8" "-target" "1.8"]
-
-                                        ;:aot [dyna.core] ;; this makes it start faster but run slower....
-                       }}
+  :profiles {:uberjar {:main dyna.DynaMain}}
   :main dyna.core
 
+  ;; clojure only requires java 8, and we shouldn't need any of the "newer" features ourselves also
+  ;; so this should make this work with any version released after 1.8 (march 2014)
+  :javac-options ["-source" "1.8" "-target" "1.8" "-XDignore.symbol.file" "-Xlint:-options"]
+
   ;; this is needed to generate the uberjar, but it makes it slower to run when working on stuff
   ;; so can comment out if running tests or something from the local directory
   :aliases {"compile" ["do" ["antlr"] ["javac"] "compile"]
@@ -55,7 +50,7 @@
 
   ;;:main dyna.DynaMain
   :global-vars {;*warn-on-reflection* true  ;; useful for identifying where it uses clojure's reflection which is slow...
-                                        ;*unchecked-math* :warn-on-boxed ;; boxed math is slow9
+                                        ;*unchecked-math* :warn-on-boxed ;; boxed math is slow
                 }
 
   ;; this will check the repo every time it runs...

src/antlr/dyna/dyna_grammar2.g4

@@ -84,7 +84,7 @@ atom returns[String t]
         $t = $m.getText();
         $t = $t.substring(1, $t.length() - 1);
         if($t.startsWith("\$_"))
-            throw new RuntimeException("the namespace \$_ is reserved for internal use");
+            throw new DynaSyntaxError("the namespace \$_ is reserved for internal use");
       }
     ;
 
@@ -101,7 +101,7 @@ Variable
 // going to change this to only match the aggregators which are actually defined
 MergedAggregator
     : [&*\-+:|?] '='
-    | [a-z][a-z&*\-+:|?]* '=' {
+    | [a-z][a-z&*\-+:|?_]* '=' {
     // this checks that the aggregator name is defined in rexpr_aggregators.clj and will conditionally enable this lexer rule
         aggregator_defined(getText())}?
     ;
@@ -277,12 +277,14 @@ term returns[DynaTerm rterm = null]
             String ttext = $t.ctx.start.getInputStream().getText(new Interval($t.ctx.start.getStartIndex(), $t.ctx.stop.getStopIndex()));
             $rterm = DynaTerm.create("\$print", $t.rterm, ttext, $t.ctx.getStart().getLine());
        }
-    | 'debug_repl'
+    | 'debug_repl_clojure'
       t=termBody["print="] EndTerm
       {
            String ttext = $t.ctx.start.getInputStream().getText(new Interval($t.ctx.start.getStartIndex(), $t.ctx.stop.getStopIndex()));
            $rterm = DynaTerm.create("\$_debug_repl", $t.rterm, ttext, $t.ctx.getStart().getLine());
       }
+    | 'debug' EndTerm
+      { $rterm = DynaTerm.create("\$debug"); }
 
 // there could be warnings if some library is used in a particular way.  This should somehow defer in the case that some dynabase has not been constructed, but this would want to have that the expression would later come into existence
     | 'warning' '(' we=expression ')' t=termBody["warning="] EndTerm
@@ -544,6 +546,9 @@ expressionRoot returns [DynaTerm rterm]
     | mp=assocativeMap { $rterm=$mp.rterm; }
     | db=dynabase { $rterm = $db.rterm; }
     | v=Variable '(' arguments ')' {
+            if($v.getText().equals("_")) {
+                throw new DynaSyntaxError("Calling an unscore function is not supported");
+            }
             // for doing an indirect call to some value
             $arguments.args.add(0, DynaTerm.create("\$variable", $v.getText()));
             $rterm = DynaTerm.create_arr("\$call", $arguments.args);
@@ -564,7 +569,7 @@ expressionDynabaseAccess returns[DynaTerm rterm]
     : a=expressionRoot {$rterm=$a.rterm;}
       ('.' m=methodCall {
                 if($rterm.name.equals("\$quote") || $rterm.name.equals("\$inline_function")) {
-                    throw new RuntimeException("syntax error");
+                    throw new DynaSyntaxError();//"syntax error");
                     //assert(false); /// should be a syntax error
                 }
                 if($rterm.name.equals("\$quote1")) {
@@ -579,8 +584,7 @@ expressionDynabaseAccess returns[DynaTerm rterm]
         })*
       ('.' bracketTerm {
         if($rterm.name.equals("\$quote") || $rterm.name.equals("\$quote1") || $rterm.name.equals("\$dynabase_quote1")) {
-            throw new RuntimeException("syntax error");
-            //assert(false); // should be a syntax error
+            throw new DynaSyntaxError("Structured terms can not be used as a dynabase");
         }
         $rterm = DynaTerm.create("\$dynbase_quote1", $rterm, DynaTerm.create_arr($bracketTerm.name, $bracketTerm.args)); })?
     ;
@@ -607,8 +611,7 @@ locals [DynaTerm add_arg=null]
                || $rterm.name.equals("\$dynabase_create") || $rterm.name.equals("\$map_empty") || $rterm.name.equals("\$map_element")
                || $rterm.name.equals("\$nil") || $rterm.name.equals("\$cons"))
                {
-               throw new RuntimeException("syntax error");
-               //assert(false); // this should return a syntax error rather than an assert(false)
+               throw new DynaSyntaxError("Implicit call using {} can only be done to a term");
                }
             if($rterm.name.equals("\$dynabase_call")) {
                 // have to put the argument onto the dynabase call element,
@@ -664,15 +667,20 @@ expressionAdditive returns [DynaTerm rterm]
             {$rterm = DynaTerm.create($op.getText(), $rterm, $b.rterm);})*
     ;
 
+expressionNot returns [DynaTerm rterm]
+    : a=expressionAdditive {$rterm=$a.rterm;}
+    | '!' a=expressionAdditive {$rterm = DynaTerm.create("!", $a.rterm);}
+    ;
+
 expressionRelationCompare returns [DynaTerm rterm]
 locals[ArrayList<DynaTerm> expressions, ArrayList<String> ops]
-    : a=expressionAdditive {$rterm = $a.rterm;}
-    | a=expressionAdditive {
+    : a=expressionNot {$rterm = $a.rterm;}
+    | a=expressionNot {
           $expressions = new ArrayList<>();
           $ops = new ArrayList<>();
           $expressions.add($a.rterm);
       }
-      (op=('>'|'<'|'<='|'>=') b=expressionAdditive
+      (op=('>'|'<'|'<='|'>=') b=expressionNot
           { $ops.add($op.getText());
             $expressions.add($b.rterm);
           })+
@@ -722,8 +730,8 @@ expression returns [DynaTerm rterm]
 compilerExpressionArgument returns [Object val]
     locals [ArrayList<Object> args=null]
     : p=primitive {$val=$p.v;}
-    | {$args=new ArrayList<>();} a=atom ('(' (ag=compilerExpressionArgument Comma {$args.add($ag.val);})*
-                                             (ag=compilerExpressionArgument Comma? {$args.add($ag.val);})?  ')' )?
+    | {$args=new ArrayList<>();} a=atom ('(' ((ag=compilerExpressionArgument Comma {$args.add($ag.val);})*
+                                              ag=compilerExpressionArgument Comma? {$args.add($ag.val);})?  ')' )?
       {$val = DynaTerm.create_arr($a.t, $args); }
     | '*' {$val = "*";}
     | '**' {$val = "**";}