From c37d2f3a6c88e1bf1f842e1b2cf55a95d4f80471 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Thu, 21 Sep 2023 14:53:36 -0400 Subject: [PATCH 01/12] Support `fn-fn` for capturing closure scope for sequences --- src/nodely/data.clj | 22 ++++++++++++++-------- src/nodely/engine/core.clj | 3 ++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/nodely/data.clj b/src/nodely/data.clj index 5b4b512..3ab3e42 100644 --- a/src/nodely/data.clj +++ b/src/nodely/data.clj @@ -24,10 +24,15 @@ :truthy (s/recursive #'Node) :falsey (s/recursive #'Node)}) -(s/defschema Sequence #::{:type (s/eq :sequence) - :input s/Keyword - :fn (s/pred ifn?) - :tags #{node-tag}}) +(s/defschema Sequence (s/either #::{:type (s/eq :sequence) + :input s/Keyword + :fn (s/pred ifn?) + :tags #{node-tag}} + #::{:type (s/eq :sequence) + :input s/Keyword + :closure-inputs #{s/Keyword} + :fn-fn (s/pred ifn?) + :tags #{node-tag}})) (s/defschema Node (s/conditional #(= (get % ::type) :value) Value @@ -69,14 +74,15 @@ (s/defn sequence ([input :- s/Keyword - fn] - (sequence input fn #{})) + f] + (sequence input f #{})) ([input :- s/Keyword - fn + f tags :- #{node-tag}] #::{:type :sequence :input input - :fn fn + :closure-inputs #{} + :fn-fn (fn [env-deps] f) :tags tags})) ;; diff --git a/src/nodely/engine/core.clj b/src/nodely/engine/core.clj index 2c6338a..70a8060 100644 --- a/src/nodely/engine/core.clj +++ b/src/nodely/engine/core.clj @@ -126,7 +126,8 @@ (defn- sequence->value [node env] (let [in-key (::data/input node) - f (::data/fn node) + f (or (::data/fn node) + ((::data/fn-fn node) (prepare-inputs (::data/closure-inputs node) env))) new-env (resolve-inputs [in-key] env) in (data/get-value new-env in-key)] [(data/value (mapv f in)) new-env])) From 3d06272b758940b03514be4d00f0859d02d55313 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Wed, 18 Oct 2023 16:06:55 -0400 Subject: [PATCH 02/12] Make all engines run against a closing function --- src/nodely/engine/applicative.clj | 10 ++++---- src/nodely/engine/core.clj | 10 ++++++-- .../core_async/iterative_scheduling.clj | 2 +- .../engine/core_async/lazy_scheduling.clj | 23 ++++++++++++++----- src/nodely/engine/manifold.clj | 2 +- test/nodely/syntax_test.clj | 2 +- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/nodely/engine/applicative.clj b/src/nodely/engine/applicative.clj index 51b7b0c..3212da7 100644 --- a/src/nodely/engine/applicative.clj +++ b/src/nodely/engine/applicative.clj @@ -26,11 +26,11 @@ (defn eval-sequence [node lazy-env opts] - (let [f (::data/fn node) - in-key (::data/input node)] - (m/alet [input-seq (get lazy-env in-key) - result (sequence (map (fn [x] (m/fmap f (m/pure x))) - (::data/value input-seq)))] + (let [in-key (::data/input node)] + (m/alet [f (m/pure (core/sequence-fn node lazy-env)) + input-seq (get lazy-env in-key) + result (sequence (map (fn [x] (m/fmap f (m/pure x))) + (::data/value input-seq)))] (data/value result)))) (defn eval-branch diff --git a/src/nodely/engine/core.clj b/src/nodely/engine/core.clj index 70a8060..972ecff 100644 --- a/src/nodely/engine/core.clj +++ b/src/nodely/engine/core.clj @@ -123,11 +123,17 @@ [k env] (resolve k (resolve-one-branch k env))) +(defn sequence-fn + ([node env] + (sequence-fn prepare-inputs node env)) + ([inputs-fn node env] + (or (::data/fn node) + ((::data/fn-fn node) (inputs-fn (::data/closure-inputs node) env))))) + (defn- sequence->value [node env] (let [in-key (::data/input node) - f (or (::data/fn node) - ((::data/fn-fn node) (prepare-inputs (::data/closure-inputs node) env))) + f (sequence-fn node env) new-env (resolve-inputs [in-key] env) in (data/get-value new-env in-key)] [(data/value (mapv f in)) new-env])) diff --git a/src/nodely/engine/core_async/iterative_scheduling.clj b/src/nodely/engine/core_async/iterative_scheduling.clj index 18102a9..651e8ab 100644 --- a/src/nodely/engine/core_async/iterative_scheduling.clj +++ b/src/nodely/engine/core_async/iterative_scheduling.clj @@ -15,7 +15,7 @@ [node resolved-env {::keys [max-sequence-parallelism] :or {max-sequence-parallelism 4}}] (let [in-key (::data/input node) - f (::data/fn node) + f (core/sequence-fn node resolved-env) sequence (map data/value (get (core/prepare-inputs [in-key] resolved-env) in-key)) in-chan (async/to-chan! sequence) pipeline-result (async/chan)] diff --git a/src/nodely/engine/core_async/lazy_scheduling.clj b/src/nodely/engine/core_async/lazy_scheduling.clj index dd578ca..bd6fc35 100644 --- a/src/nodely/engine/core_async/lazy_scheduling.clj +++ b/src/nodely/engine/core_async/lazy_scheduling.clj @@ -11,14 +11,27 @@ (declare eval-async) +(defn async-sequence-fn + [node lazy-env] + (async/go + (or (:data/fn node) + ((::data/fn-fn node) + (loop [[cur & pending] (::data/closure-inputs node) + res {}] + (if cur + (recur pending (assoc res cur (async/> (get in in-key) (mapv #(deferred/future (f %))) diff --git a/test/nodely/syntax_test.clj b/test/nodely/syntax_test.clj index 7b0b4bb..c634888 100644 --- a/test/nodely/syntax_test.clj +++ b/test/nodely/syntax_test.clj @@ -164,7 +164,7 @@ :fn fn?} :b #::data {:type :sequence :input :a - :fn fn?}} + :fn-fn fn?}} {:a (>leaf [1 2 3]) :b (>sequence inc ?a)})))) From 3dc93c1b929aaa3cc19789cc05054eb34a0857b5 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Fri, 8 Dec 2023 12:41:02 -0500 Subject: [PATCH 03/12] Closure sequences tested and working with the lazy sync engine --- src/nodely/data.clj | 16 ++++++++++++---- src/nodely/engine/core_async/lazy_scheduling.clj | 2 +- src/nodely/syntax.clj | 8 ++++++-- test/nodely/engine/core_test.clj | 13 +++++++++++-- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/nodely/data.clj b/src/nodely/data.clj index 3ab3e42..9999ff8 100644 --- a/src/nodely/data.clj +++ b/src/nodely/data.clj @@ -30,7 +30,7 @@ :tags #{node-tag}} #::{:type (s/eq :sequence) :input s/Keyword - :closure-inputs #{s/Keyword} + :closure-inputs [s/Keyword] :fn-fn (s/pred ifn?) :tags #{node-tag}})) @@ -81,9 +81,17 @@ tags :- #{node-tag}] #::{:type :sequence :input input - :closure-inputs #{} - :fn-fn (fn [env-deps] f) - :tags tags})) + :fn f + :tags tags}) + ([input :- s/Keyword + closure-inputs + f + tags :- #{node-tag}] + #::{:type :sequence + :input input + :closure-inputs closure-inputs + :fn-fn f + :tags tags})) ;; ;; Node Utilities diff --git a/src/nodely/engine/core_async/lazy_scheduling.clj b/src/nodely/engine/core_async/lazy_scheduling.clj index bd6fc35..26054f5 100644 --- a/src/nodely/engine/core_async/lazy_scheduling.clj +++ b/src/nodely/engine/core_async/lazy_scheduling.clj @@ -14,7 +14,7 @@ (defn async-sequence-fn [node lazy-env] (async/go - (or (:data/fn node) + (or (::data/fn node) ((::data/fn-fn node) (loop [[cur & pending] (::data/closure-inputs node) res {}] diff --git a/src/nodely/syntax.clj b/src/nodely/syntax.clj index 7f3f0d7..095d658 100644 --- a/src/nodely/syntax.clj +++ b/src/nodely/syntax.clj @@ -36,8 +36,12 @@ (data/value n))) (defmacro >sequence - [fn input] - (list `data/sequence (question-mark->keyword input) fn)) + [f input] + (let [symbols-to-be-replaced (question-mark-symbols f) + closure-inputs (mapv question-mark->keyword symbols-to-be-replaced) + fn-fn (fn-with-arg-map symbols-to-be-replaced f)] + (assert-not-shadowing! symbols-to-be-replaced) + `(data/sequence ~(question-mark->keyword input) ~closure-inputs ~fn-fn #{}))) (defmacro >leaf [expr] diff --git a/test/nodely/engine/core_test.clj b/test/nodely/engine/core_test.clj index 6b96394..9f27158 100644 --- a/test/nodely/engine/core_test.clj +++ b/test/nodely/engine/core_test.clj @@ -7,7 +7,7 @@ [nodely.data :as data] [nodely.engine.core :as core] [nodely.fixtures :as fixtures] - [nodely.syntax :as syntax :refer [>cond >if >leaf]] + [nodely.syntax :as syntax :refer [>cond >if >leaf >sequence]] [schema.test])) (use-fixtures :once schema.test/validate-schemas) @@ -69,6 +69,10 @@ (def env-with-sequence {:x (data/value [1 2 3]) :y (data/sequence :x inc)}) +(def env-with-closure-sequence {:x (data/value [1 2 3]) + :z (data/value 2) + :y (>sequence (fn [e] (* e ?z)) ?x)}) + (def branch-with-sequence {:x (data/value [1 2 3]) :y (data/value [4 5 6]) :a (data/value true) @@ -165,6 +169,11 @@ (is (= {:x (data/value [1 2 3]) :y (data/value [2 3 4])} (core/resolve :y env-with-sequence)))) + (testing "resolve closure sequence" + (is (= {:x (data/value [1 2 3]) + :z (data/value 2) + :y (data/value [2 4 6])} + (core/resolve :y env-with-closure-sequence)))) (testing "resolve with nested branches" (is (= {:x (data/value 1) :y (data/value 2) @@ -245,4 +254,4 @@ (testing "there is a cycle in the env and checked-env detects it" (is (thrown? clojure.lang.ExceptionInfo (core/checked-env env-with-cycle)))) (testing "there is no cycle but checked-env reports there is (corner case mutually exclusive)" - (is (thrown? clojure.lang.ExceptionInfo (core/checked-env mutually-exclusive-env-without-cycles))))) \ No newline at end of file + (is (thrown? clojure.lang.ExceptionInfo (core/checked-env mutually-exclusive-env-without-cycles))))) From eadaf697d85587e28a4601582aaa234095e23ab0 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Fri, 8 Dec 2023 13:10:30 -0500 Subject: [PATCH 04/12] Closure sequences in core.async... --- src/nodely/engine/core_async/lazy_scheduling.clj | 2 +- .../engine/core_async/lazy_scheduling_test.clj | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/nodely/engine/core_async/lazy_scheduling.clj b/src/nodely/engine/core_async/lazy_scheduling.clj index 26054f5..30ffb9b 100644 --- a/src/nodely/engine/core_async/lazy_scheduling.clj +++ b/src/nodely/engine/core_async/lazy_scheduling.clj @@ -19,7 +19,7 @@ (loop [[cur & pending] (::data/closure-inputs node) res {}] (if cur - (recur pending (assoc res cur (async/cond >leaf >value blocking]])) + [nodely.syntax :as syntax :refer [>cond >leaf >value >sequence blocking]])) (def test-env {:a (>leaf (+ 1 2)) :b (>leaf (* ?a 2)) @@ -57,7 +57,11 @@ :c ?c})}) (def env-with-sequence {:a (>leaf [1 2 3]) - :b (syntax/>sequence inc ?a)}) + :b (>sequence inc ?a)}) + +(def env-with-closure-sequence {:a (>leaf [1 2 3]) + :c (>value 2) + :b (>sequence #(* % ?c) ?a)}) (def env+sequence-with-nil-values {:a (>leaf [1 2 nil 4]) @@ -142,7 +146,9 @@ (is (match? (matchers/within-delta 8000000 2000000000) (- nanosec-sync nanosec-async))))) (testing "Actually computes the correct answers" - (is (= [2 3 4] (nasync/eval-key env-with-sequence+delay :b))))) + (is (= [2 3 4] (nasync/eval-key env-with-sequence+delay :b)))) + (testing "When there's a closure in the sequence expr" + (is (= [2 4 6] (nasync/eval-key env-with-closure-sequence :b))))) (deftest eval-test (testing "eval node async" From 659f53e6d6948a5d2ad086fc5cb9b609c14cb9cf Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Thu, 25 Jan 2024 14:36:19 -0500 Subject: [PATCH 05/12] Make applicative closure sequences work - With Joao Luigi - Have to redefine prepare-inputs because LazyEnvs are not clojure.lang.Associative or java.util.Maps - Do not like the impl of `eval-sequence` or `applicative-sequence-fn` but they work and tests pass. Time to make a checkpoint. --- src/nodely/engine/applicative.clj | 12 +++++++++++- src/nodely/engine/core.clj | 4 ++-- test/nodely/engine/applicative_test.clj | 11 +++++++++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/nodely/engine/applicative.clj b/src/nodely/engine/applicative.clj index 3212da7..bcc65f1 100644 --- a/src/nodely/engine/applicative.clj +++ b/src/nodely/engine/applicative.clj @@ -24,10 +24,20 @@ (declare eval-node) +(defn applicative-sequence-fn + ;; monadic context of node + ;; map of keyword -> monadic context of lazy-env + + ;; monadic context of a function of one argument out + [node lazy-env] + (m/pure (or (::data/fn node) + ((::data/fn-fn node) + (into {} (map (juxt identity (comp ::data/value m/extract (partial get lazy-env))) (::data/closure-inputs node))))))) + (defn eval-sequence [node lazy-env opts] (let [in-key (::data/input node)] - (m/alet [f (m/pure (core/sequence-fn node lazy-env)) + (m/alet [f (applicative-sequence-fn node lazy-env) ;;(m/pure (core/sequence-fn node lazy-env)) input-seq (get lazy-env in-key) result (sequence (map (fn [x] (m/fmap f (m/pure x))) (::data/value input-seq)))] diff --git a/src/nodely/engine/core.clj b/src/nodely/engine/core.clj index 972ecff..b8e33f6 100644 --- a/src/nodely/engine/core.clj +++ b/src/nodely/engine/core.clj @@ -76,8 +76,8 @@ (defn prepare-inputs [input-keys env] - (->> (select-keys env input-keys) - (map (juxt key (comp ::data/value val))) + (->> input-keys + (map (juxt identity (comp ::data/value (partial get env)))) (into {}))) (defn eval-leaf diff --git a/test/nodely/engine/applicative_test.clj b/test/nodely/engine/applicative_test.clj index 83139c7..eb134d8 100644 --- a/test/nodely/engine/applicative_test.clj +++ b/test/nodely/engine/applicative_test.clj @@ -16,7 +16,7 @@ [nodely.engine.core-async.core :as nodely.async] [nodely.engine.schema :as schema] [nodely.fixtures :as fixtures] - [nodely.syntax :as syntax :refer [>leaf >value]] + [nodely.syntax :as syntax :refer [>leaf >value >sequence]] [nodely.syntax.schema :refer [yielding-schema]] [promesa.core :as p] [schema.core :as s])) @@ -74,6 +74,10 @@ (def env-with-sequence {:a (>leaf [1 2 3]) :b (syntax/>sequence inc ?a)}) +(def env-with-closure-sequence {:x (data/value [1 2 3]) + :z (data/value 2) + :y (>sequence (fn [e] (* e ?z)) ?x)}) + (def env+sequence-with-nil-values {:a (>leaf [1 2 nil 4]) :b (syntax/>sequence #(when % (inc %)) ?a)}) @@ -175,7 +179,10 @@ (is (match? (matchers/within-delta 8000000 2000000000) (- nanosec-sync nanosec-async))))) (testing "Actually computes the correct answers" - (is (match? [2 3 4] (applicative/eval-key env-with-sequence+delay :c))))) + (is (match? [2 3 4] (applicative/eval-key env-with-sequence+delay :c)))) + (testing "resolve closure sequence" + (is (= [2 4 6] + (applicative/eval-key env-with-closure-sequence :y))))) (deftest schema-test (let [env-with-schema {:a (>value 2) From d0009a9c94f2b11f1338d04847e9ac41bddac213 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Fri, 2 Feb 2024 12:04:03 -0500 Subject: [PATCH 06/12] Make applicative impls "more monadic" Looks cleaner to me at least! --- src/nodely/engine/applicative.clj | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/nodely/engine/applicative.clj b/src/nodely/engine/applicative.clj index bcc65f1..dc372fb 100644 --- a/src/nodely/engine/applicative.clj +++ b/src/nodely/engine/applicative.clj @@ -25,23 +25,26 @@ (declare eval-node) (defn applicative-sequence-fn - ;; monadic context of node - ;; map of keyword -> monadic context of lazy-env - - ;; monadic context of a function of one argument out [node lazy-env] - (m/pure (or (::data/fn node) - ((::data/fn-fn node) - (into {} (map (juxt identity (comp ::data/value m/extract (partial get lazy-env))) (::data/closure-inputs node))))))) + (or (when-let [f (::data/fn node)] + (m/pure f)) + (let [ks (::data/closure-inputs node)] + (m/fmap #((::data/fn-fn node) (zipmap ks (map ::data/value %))) + (m/sequence (map (partial get lazy-env) ks)))))) (defn eval-sequence [node lazy-env opts] - (let [in-key (::data/input node)] - (m/alet [f (applicative-sequence-fn node lazy-env) ;;(m/pure (core/sequence-fn node lazy-env)) - input-seq (get lazy-env in-key) - result (sequence (map (fn [x] (m/fmap f (m/pure x))) - (::data/value input-seq)))] - (data/value result)))) + (let [in-key (::data/input node) + mf (applicative-sequence-fn node lazy-env) + mseq (get lazy-env in-key)] + (->> mseq + (m/fmap (comp m/sequence + (partial map + (comp (partial m/fapply mf) + m/pure)) + ::data/value)) + m/join + (m/fmap data/value)))) (defn eval-branch [{::data/keys [condition truthy falsey]} From 35170a04c3770781936812e6723385203e927378 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Thu, 8 Feb 2024 15:16:14 -0500 Subject: [PATCH 07/12] Change from `:fn-fn` special case (and lots of leaf redundancy) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just use a leaf. With João and Sophia We want to continue evaluating how we model the sequence node's data to minimize changes, and, minimize scope of what needs to be handled in engines bespokely. --- src/nodely/data.clj | 22 +++++++++---------- src/nodely/engine/applicative.clj | 13 +++++------ src/nodely/engine/core.clj | 8 +++---- .../engine/core_async/lazy_scheduling.clj | 22 +++++++++---------- src/nodely/engine/manifold.clj | 9 +++++++- src/nodely/syntax.clj | 4 +++- test/nodely/engine/applicative_test.clj | 2 +- .../core_async/lazy_scheduling_test.clj | 8 +++---- test/nodely/syntax_test.clj | 2 +- 9 files changed, 49 insertions(+), 41 deletions(-) diff --git a/src/nodely/data.clj b/src/nodely/data.clj index 9999ff8..95cd564 100644 --- a/src/nodely/data.clj +++ b/src/nodely/data.clj @@ -24,15 +24,12 @@ :truthy (s/recursive #'Node) :falsey (s/recursive #'Node)}) -(s/defschema Sequence (s/either #::{:type (s/eq :sequence) - :input s/Keyword - :fn (s/pred ifn?) - :tags #{node-tag}} - #::{:type (s/eq :sequence) - :input s/Keyword - :closure-inputs [s/Keyword] - :fn-fn (s/pred ifn?) - :tags #{node-tag}})) +(s/defschema Sequence #::{:type (s/eq :sequence) + :input s/Keyword + :fn (s/conditional + #(= (get % ::type) :leaf) Leaf + ifn? (s/pred ifn?)) + :tags #{node-tag}}) (s/defschema Node (s/conditional #(= (get % ::type) :value) Value @@ -89,8 +86,7 @@ tags :- #{node-tag}] #::{:type :sequence :input input - :closure-inputs closure-inputs - :fn-fn f + :fn (leaf closure-inputs f) :tags tags})) ;; @@ -104,6 +100,10 @@ [node] (= :value (::type node))) +(defn leaf? + [node] + (= :leaf (::type node))) + (defn node-inputs [node] (case (::type node) diff --git a/src/nodely/engine/applicative.clj b/src/nodely/engine/applicative.clj index dc372fb..5012587 100644 --- a/src/nodely/engine/applicative.clj +++ b/src/nodely/engine/applicative.clj @@ -25,17 +25,16 @@ (declare eval-node) (defn applicative-sequence-fn - [node lazy-env] - (or (when-let [f (::data/fn node)] - (m/pure f)) - (let [ks (::data/closure-inputs node)] - (m/fmap #((::data/fn-fn node) (zipmap ks (map ::data/value %))) - (m/sequence (map (partial get lazy-env) ks)))))) + [node lazy-env opts] + (let [f (::data/fn node)] + (if (data/leaf? f) + (m/fmap ::data/value (eval-node f lazy-env opts)) + (m/pure f)))) (defn eval-sequence [node lazy-env opts] (let [in-key (::data/input node) - mf (applicative-sequence-fn node lazy-env) + mf (applicative-sequence-fn node lazy-env opts) mseq (get lazy-env in-key)] (->> mseq (m/fmap (comp m/sequence diff --git a/src/nodely/engine/core.clj b/src/nodely/engine/core.clj index b8e33f6..97450a0 100644 --- a/src/nodely/engine/core.clj +++ b/src/nodely/engine/core.clj @@ -125,10 +125,10 @@ (defn sequence-fn ([node env] - (sequence-fn prepare-inputs node env)) - ([inputs-fn node env] - (or (::data/fn node) - ((::data/fn-fn node) (inputs-fn (::data/closure-inputs node) env))))) + (let [f (::data/fn node)] + (if (data/leaf? f) + (::data/value (first (leaf->value f env))) + f)))) (defn- sequence->value [node env] diff --git a/src/nodely/engine/core_async/lazy_scheduling.clj b/src/nodely/engine/core_async/lazy_scheduling.clj index 30ffb9b..83fc657 100644 --- a/src/nodely/engine/core_async/lazy_scheduling.clj +++ b/src/nodely/engine/core_async/lazy_scheduling.clj @@ -12,25 +12,25 @@ (declare eval-async) (defn async-sequence-fn - [node lazy-env] - (async/go - (or (::data/fn node) - ((::data/fn-fn node) - (loop [[cur & pending] (::data/closure-inputs node) - res {}] - (if cur - (recur pending (assoc res cur (::data/value (async/> (get in in-key) (mapv #(deferred/future (f %))) diff --git a/src/nodely/syntax.clj b/src/nodely/syntax.clj index 095d658..fb9e1f4 100644 --- a/src/nodely/syntax.clj +++ b/src/nodely/syntax.clj @@ -41,7 +41,9 @@ closure-inputs (mapv question-mark->keyword symbols-to-be-replaced) fn-fn (fn-with-arg-map symbols-to-be-replaced f)] (assert-not-shadowing! symbols-to-be-replaced) - `(data/sequence ~(question-mark->keyword input) ~closure-inputs ~fn-fn #{}))) + (if (seq symbols-to-be-replaced) + `(data/sequence ~(question-mark->keyword input) ~closure-inputs ~fn-fn #{}) + `(data/sequence ~(question-mark->keyword input) ~f #{})))) (defmacro >leaf [expr] diff --git a/test/nodely/engine/applicative_test.clj b/test/nodely/engine/applicative_test.clj index eb134d8..8fb9de6 100644 --- a/test/nodely/engine/applicative_test.clj +++ b/test/nodely/engine/applicative_test.clj @@ -76,7 +76,7 @@ (def env-with-closure-sequence {:x (data/value [1 2 3]) :z (data/value 2) - :y (>sequence (fn [e] (* e ?z)) ?x)}) + :y (>sequence #(* % ?z) ?x)}) (def env+sequence-with-nil-values {:a (>leaf [1 2 nil 4]) diff --git a/test/nodely/engine/core_async/lazy_scheduling_test.clj b/test/nodely/engine/core_async/lazy_scheduling_test.clj index 6037f54..638282a 100644 --- a/test/nodely/engine/core_async/lazy_scheduling_test.clj +++ b/test/nodely/engine/core_async/lazy_scheduling_test.clj @@ -59,9 +59,9 @@ (def env-with-sequence {:a (>leaf [1 2 3]) :b (>sequence inc ?a)}) -(def env-with-closure-sequence {:a (>leaf [1 2 3]) +(def env-with-closure-sequence {:a (>value [1 2 3]) :c (>value 2) - :b (>sequence #(* % ?c) ?a)}) + :b (>sequence (fn [e] (* e ?c)) ?a)}) (def env+sequence-with-nil-values {:a (>leaf [1 2 nil 4]) @@ -146,9 +146,9 @@ (is (match? (matchers/within-delta 8000000 2000000000) (- nanosec-sync nanosec-async))))) (testing "Actually computes the correct answers" - (is (= [2 3 4] (nasync/eval-key env-with-sequence+delay :b)))) + (is (match? [2 3 4] (nasync/eval-key env-with-sequence+delay :b)))) (testing "When there's a closure in the sequence expr" - (is (= [2 4 6] (nasync/eval-key env-with-closure-sequence :b))))) + (is (match? [2 4 6] (nasync/eval-key env-with-closure-sequence :b))))) (deftest eval-test (testing "eval node async" diff --git a/test/nodely/syntax_test.clj b/test/nodely/syntax_test.clj index c634888..8a6a026 100644 --- a/test/nodely/syntax_test.clj +++ b/test/nodely/syntax_test.clj @@ -164,7 +164,7 @@ :fn fn?} :b #::data {:type :sequence :input :a - :fn-fn fn?}} + :fn fn?}} {:a (>leaf [1 2 3]) :b (>sequence inc ?a)})))) From b5912c9f27ef58a6b5dc9b0d88d29fbcb4ced357 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Fri, 9 Feb 2024 13:48:27 -0500 Subject: [PATCH 08/12] Make sequences get their mapped function off a node - Fixed fns are a value node. - Fns that need to close over the environment are leaf nodes. - Maybe we figure out when branch nodes are appropriate? - ONLY needed to update impl and syntax test. Strong validation of the abstraction that all other tests pass without updates! --- src/nodely/data.clj | 26 +++++++++---------- src/nodely/engine/applicative.clj | 10 ++----- src/nodely/engine/core.clj | 9 +------ .../core_async/iterative_scheduling.clj | 2 +- .../engine/core_async/lazy_scheduling.clj | 16 ++++-------- src/nodely/engine/manifold.clj | 11 +++----- test/nodely/syntax_test.clj | 3 ++- 7 files changed, 26 insertions(+), 51 deletions(-) diff --git a/src/nodely/data.clj b/src/nodely/data.clj index 95cd564..6026261 100644 --- a/src/nodely/data.clj +++ b/src/nodely/data.clj @@ -24,12 +24,10 @@ :truthy (s/recursive #'Node) :falsey (s/recursive #'Node)}) -(s/defschema Sequence #::{:type (s/eq :sequence) - :input s/Keyword - :fn (s/conditional - #(= (get % ::type) :leaf) Leaf - ifn? (s/pred ifn?)) - :tags #{node-tag}}) +(s/defschema Sequence #::{:type (s/eq :sequence) + :input s/Keyword + :process-node (s/recursive #'Node) + :tags #{node-tag}}) (s/defschema Node (s/conditional #(= (get % ::type) :value) Value @@ -76,18 +74,18 @@ ([input :- s/Keyword f tags :- #{node-tag}] - #::{:type :sequence - :input input - :fn f - :tags tags}) + #::{:type :sequence + :input input + :process-node (value f) + :tags tags}) ([input :- s/Keyword closure-inputs f tags :- #{node-tag}] - #::{:type :sequence - :input input - :fn (leaf closure-inputs f) - :tags tags})) + #::{:type :sequence + :input input + :process-node (leaf closure-inputs f) + :tags tags})) ;; ;; Node Utilities diff --git a/src/nodely/engine/applicative.clj b/src/nodely/engine/applicative.clj index 5012587..bec54b1 100644 --- a/src/nodely/engine/applicative.clj +++ b/src/nodely/engine/applicative.clj @@ -24,17 +24,11 @@ (declare eval-node) -(defn applicative-sequence-fn - [node lazy-env opts] - (let [f (::data/fn node)] - (if (data/leaf? f) - (m/fmap ::data/value (eval-node f lazy-env opts)) - (m/pure f)))) - (defn eval-sequence [node lazy-env opts] (let [in-key (::data/input node) - mf (applicative-sequence-fn node lazy-env opts) + mf (m/fmap ::data/value + (eval-node (::data/process-node node) lazy-env opts)) mseq (get lazy-env in-key)] (->> mseq (m/fmap (comp m/sequence diff --git a/src/nodely/engine/core.clj b/src/nodely/engine/core.clj index 97450a0..7afaf79 100644 --- a/src/nodely/engine/core.clj +++ b/src/nodely/engine/core.clj @@ -123,17 +123,10 @@ [k env] (resolve k (resolve-one-branch k env))) -(defn sequence-fn - ([node env] - (let [f (::data/fn node)] - (if (data/leaf? f) - (::data/value (first (leaf->value f env))) - f)))) - (defn- sequence->value [node env] (let [in-key (::data/input node) - f (sequence-fn node env) + f (::data/value (first (node->value (::data/process-node node) env))) new-env (resolve-inputs [in-key] env) in (data/get-value new-env in-key)] [(data/value (mapv f in)) new-env])) diff --git a/src/nodely/engine/core_async/iterative_scheduling.clj b/src/nodely/engine/core_async/iterative_scheduling.clj index 651e8ab..d68a109 100644 --- a/src/nodely/engine/core_async/iterative_scheduling.clj +++ b/src/nodely/engine/core_async/iterative_scheduling.clj @@ -15,7 +15,7 @@ [node resolved-env {::keys [max-sequence-parallelism] :or {max-sequence-parallelism 4}}] (let [in-key (::data/input node) - f (core/sequence-fn node resolved-env) + f (::data/value (first (core/node->value (::data/process-node node) resolved-env))) sequence (map data/value (get (core/prepare-inputs [in-key] resolved-env) in-key)) in-chan (async/to-chan! sequence) pipeline-result (async/chan)] diff --git a/src/nodely/engine/core_async/lazy_scheduling.clj b/src/nodely/engine/core_async/lazy_scheduling.clj index 83fc657..6476bdd 100644 --- a/src/nodely/engine/core_async/lazy_scheduling.clj +++ b/src/nodely/engine/core_async/lazy_scheduling.clj @@ -11,16 +11,6 @@ (declare eval-async) -(defn async-sequence-fn - [node lazy-env opts] - (let [f (::data/fn node)] - (if (data/leaf? f) - (async/go - (let [f-ch (async/chan 1)] - (eval-async f lazy-env (assoc opts ::out-ch f-ch)) - (::data/value (async/> (select-keys future-env input-keys) @@ -17,17 +19,10 @@ (let [in (prepare-inputs (::data/inputs node) future-env)] (core/eval-leaf node in))) -(defn sequence-fn - [node future-env] - (let [f (::data/fn node)] - (if (data/leaf? f) - (::data/value (eval-leaf f future-env)) - f))) - (defn eval-sequence [node future-env] (let [in-key (::data/input node) - f (sequence-fn node future-env) + f (::data/value (eval-async (::data/process-node node) future-env)) in (prepare-inputs [in-key] future-env)] (data/value (->> (get in in-key) (mapv #(deferred/future (f %))) diff --git a/test/nodely/syntax_test.clj b/test/nodely/syntax_test.clj index 8a6a026..169a9df 100644 --- a/test/nodely/syntax_test.clj +++ b/test/nodely/syntax_test.clj @@ -164,7 +164,8 @@ :fn fn?} :b #::data {:type :sequence :input :a - :fn fn?}} + :process-node #::data {:type :value + :value fn?}}} {:a (>leaf [1 2 3]) :b (>sequence inc ?a)})))) From 24d9f690ef3016dc72ede6e9e91b1522918fa5f4 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Fri, 16 Feb 2024 11:30:02 -0500 Subject: [PATCH 09/12] Closure sequences in manifold --- src/nodely/data.clj | 18 +++++++++++------- test/nodely/engine/manifold_test.clj | 9 ++++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/nodely/data.clj b/src/nodely/data.clj index 6026261..a44e936 100644 --- a/src/nodely/data.clj +++ b/src/nodely/data.clj @@ -1,7 +1,8 @@ (ns nodely.data (:refer-clojure :exclude [map sequence]) (:require - [schema.core :as s])) + [schema.core :as s] + [clojure.set :as set])) ;; ;; Node Definitions @@ -103,12 +104,15 @@ (= :leaf (::type node))) (defn node-inputs - [node] - (case (::type node) - :value #{} - :leaf (::inputs node) - :branch (recur (::condition node)) - :sequence #{(::input node)})) + ([node] + (node-inputs node #{})) + ([node inputs] + (case (::type node) + :value inputs + :leaf (set/union inputs (::inputs node)) + :branch (recur (::condition node) inputs) + :sequence (recur (::process-node node) + (conj inputs (::input node)))))) ;; ;; Env Utils diff --git a/test/nodely/engine/manifold_test.clj b/test/nodely/engine/manifold_test.clj index c0a7644..0515d9b 100644 --- a/test/nodely/engine/manifold_test.clj +++ b/test/nodely/engine/manifold_test.clj @@ -27,6 +27,11 @@ (def env-with-sequence {:a (>leaf [1 2 3]) :b (syntax/>sequence inc ?a)}) +(def env-with-sequence-with-closure + {:a (syntax/>value [1 2 3]) + :c (syntax/>value 2) + :b (syntax/>sequence #(* % ?c) ?a)}) + (def env-with-sequence+delay {:a (>leaf [1 2 3]) :b (syntax/>sequence #(do (Thread/sleep 1000) (inc %)) @@ -54,7 +59,9 @@ (is (match? (matchers/within-delta 8000000 2000000000) (- nanosec-sync nanosec-async))))) (testing "Actually computes the correct answers" - (is (= [2 3 4] (manifold/eval-key env-with-sequence+delay :b))))) + (is (= [2 3 4] (manifold/eval-key env-with-sequence+delay :b)))) + (testing "Supports closure of nodes in the iterated fn" + (is (= [2 4 6] (manifold/eval-key env-with-sequence-with-closure :b))))) (deftest eval-test (testing "eval node async" From d3049c094103516031b6c9025e2da32de3e29dc4 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Fri, 16 Feb 2024 11:38:32 -0500 Subject: [PATCH 10/12] Version and Changelog --- CHANGELOG.md | 4 ++-- project.clj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b243b53..8321c66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog -## Unreleased -- Better error message for sync lazy engine when env is missing a key +## 1.16.0 / 2024-02-16 +- Allow sequences to refer to any node in the environment for the mapped fn ## 1.15.0 / 2023-12-07 - Allow running the lazy synchronous engine with a channel return. diff --git a/project.clj b/project.clj index b580f86..357955d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject dev.nu/nodely "1.15.0" +(defproject dev.nu/nodely "1.16.0" :description "Decoupling data fetching from data dependency declaration" :url "https://github.com/nubank/nodely" :license {:name "MIT"} From ae24d8a5015a1d166b3017590b2049d7ff9d73fc Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Fri, 16 Feb 2024 11:54:09 -0500 Subject: [PATCH 11/12] lint fix --- src/nodely/data.clj | 4 ++-- test/nodely/engine/applicative_test.clj | 2 +- test/nodely/engine/core_async/lazy_scheduling_test.clj | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nodely/data.clj b/src/nodely/data.clj index a44e936..041a59e 100644 --- a/src/nodely/data.clj +++ b/src/nodely/data.clj @@ -1,8 +1,8 @@ (ns nodely.data (:refer-clojure :exclude [map sequence]) (:require - [schema.core :as s] - [clojure.set :as set])) + [clojure.set :as set] + [schema.core :as s])) ;; ;; Node Definitions diff --git a/test/nodely/engine/applicative_test.clj b/test/nodely/engine/applicative_test.clj index 8fb9de6..9848e32 100644 --- a/test/nodely/engine/applicative_test.clj +++ b/test/nodely/engine/applicative_test.clj @@ -16,7 +16,7 @@ [nodely.engine.core-async.core :as nodely.async] [nodely.engine.schema :as schema] [nodely.fixtures :as fixtures] - [nodely.syntax :as syntax :refer [>leaf >value >sequence]] + [nodely.syntax :as syntax :refer [>leaf >sequence >value]] [nodely.syntax.schema :refer [yielding-schema]] [promesa.core :as p] [schema.core :as s])) diff --git a/test/nodely/engine/core_async/lazy_scheduling_test.clj b/test/nodely/engine/core_async/lazy_scheduling_test.clj index 638282a..7eef078 100644 --- a/test/nodely/engine/core_async/lazy_scheduling_test.clj +++ b/test/nodely/engine/core_async/lazy_scheduling_test.clj @@ -14,7 +14,7 @@ [nodely.engine.core-async.lazy-scheduling :as nasync] [nodely.engine.lazy :as engine.lazy] [nodely.fixtures :as fixtures] - [nodely.syntax :as syntax :refer [>cond >leaf >value >sequence blocking]])) + [nodely.syntax :as syntax :refer [>cond >leaf >sequence >value blocking]])) (def test-env {:a (>leaf (+ 1 2)) :b (>leaf (* ?a 2)) From cf417711fff769844c3f45893b80c6246ac5100b Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Fri, 16 Feb 2024 12:27:14 -0500 Subject: [PATCH 12/12] Lock down process-node to only leaf and value --- src/nodely/data.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nodely/data.clj b/src/nodely/data.clj index 041a59e..126fd79 100644 --- a/src/nodely/data.clj +++ b/src/nodely/data.clj @@ -27,7 +27,9 @@ (s/defschema Sequence #::{:type (s/eq :sequence) :input s/Keyword - :process-node (s/recursive #'Node) + :process-node (s/conditional + #(= (get % ::type) :value) Value + #(= (get % ::type) :leaf) Leaf) :tags #{node-tag}}) (s/defschema Node (s/conditional