Tuples on read path #364 #427

Author: tonsky

.gitignore

@@ -28,4 +28,5 @@ web/out
 web/target-cljs
 .rebl
 .shadow-cljs
-.clj-kondo
+.clj-kondo
+.lsp

CHANGELOG.md

@@ -1,6 +1,6 @@
 # WIP
 
-- Support heterogeneous (`:db/tupleTypes`) and homogeneous (`:db/tupleType`) tuples #364
+- Support heterogeneous (`:db/tupleTypes`) and homogeneous (`:db/tupleType`) tuples #364 #427
 
 # 1.7.8 - Oct 11, 2025
 

docs/tuples.md

@@ -152,7 +152,18 @@ Slot types other than `:db.type/ref` are not enforced — DataScript is dynamica
   [[:db/add 2 :ref+long [[:name "Ivan"] 8]]]) ; lookup ref works too
 ```
 
-Refs inside tuples are resolved on write, but are not full-fledged references otherwise: they don’t show up in reverse index, don’t get retracted when the referenced entity is retracted (the eid stays in the tuple, dangling), can’t be `:db/isComponent`, and entity API returns them as plain numbers, not entities. Same is true in Datomic.
+Ref slots are resolved on read, too: lookup refs and idents can be used anywhere a tuple value is expected, and are resolved against the current db.
+
+```
+(d/datoms db :avet :ref+long [[:name "Ivan"] 8])
+(d/index-range db :ref+long [[:name "Ivan"] 0] [[:name "Ivan"] 100])
+(d/q '[:find ?e :where [?e :ref+long [[:name "Ivan"] 8]]] db)
+(d/q '[:find ?e :in $ [?t ...] :where [?e :ref+long ?t]] db [[[:name "Ivan"] 8]])
+```
+
+This works for composite tuples with ref attributes as well. A lookup ref that resolves to nothing raises, same as it does for plain `:db.type/ref` attributes.
+
+Refs inside tuples are not full-fledged references otherwise: they don’t show up in reverse index, don’t get retracted when the referenced entity is retracted (the eid stays in the tuple, dangling), can’t be `:db/isComponent`, and entity API returns them as plain numbers, not entities. Same is true in Datomic.
 
 Unlike composite tuples, heterogeneous and homogeneous tuples are not indexed by default. Add `:db/index true` or `:db/unique` if you want AVET access. Unique declared tuples work in upserts and lookup refs, same as composite ones:
 

src/datascript/db.cljc

@@ -1163,15 +1163,23 @@
 #?(:clj  (declare ref?)
    :cljs (defn ^boolean ref? [db attr]))
 
+#?(:clj  (declare tuple?)
+   :cljs (defn ^boolean tuple? [db attr]))
+
+#?(:clj  (declare resolve-tuple-refs)
+   :cljs (defn resolve-tuple-refs [db a vs context]))
+
 (defn+ resolve-datom [db e a v t default-e default-tx]
   (when (some? a)
     (validate-attr a (list 'resolve-datom 'db e a v t)))
   (datom
     (if (some? e) (entid-strict db e) default-e)
     a
-    (if (and (some? v) (ref? db a))
-      (entid-strict db v)
-      v)
+    (cond
+      (nil? v)      v
+      (ref? db a)   (entid-strict db v)
+      (tuple? db a) (resolve-tuple-refs db a v (list 'resolve-datom 'db e a v t))
+      :else         v)
     (if (some? t) (entid-strict db t) default-tx)))
 
 (defn+ components->pattern [db index c0 c1 c2 c3 default-e default-tx]
@@ -1271,38 +1279,38 @@
          (util/raise "Attribute " a " expected a vector, got: " v " in " context
            {:error :transact/syntax, :attribute a, :value v, :context context}))))))
 
-(defn resolve-tuple-refs [db a vs context]
-  (let [attr (-> db -schema (get a))]
-    (validate-tuple db a attr vs context)
-    (cond
-      (not (sequential? vs)) ;; e.g. nil ov in :db.fn/cas
-      vs
+(defn- #?@(:clj  [^Boolean tuple-ref-slot?]
+           :cljs [^boolean tuple-ref-slot?])
+  [db schema-a idx]
+  (cond
+    (:db/tupleAttrs schema-a) (ref? db (nth (:db/tupleAttrs schema-a) idx nil))
+    (:db/tupleTypes schema-a) (= :db.type/ref (nth (:db/tupleTypes schema-a) idx nil))
+    :else                     (= :db.type/ref (:db/tupleType schema-a))))
 
-      (:db/tupleAttrs attr)
-      (mapv
-        (fn [el-a v]
-          (if (and (ref? db el-a) (sequential? v)) ;; lookup-ref
-            (entid-strict db v)
-            v))
-        (:db/tupleAttrs attr) vs)
-
-      (:db/tupleTypes attr)
-      (mapv
-        (fn [el-type v]
-          (if (and (= :db.type/ref el-type) (sequential? v)) ;; lookup-ref
-            (entid-strict db v)
-            v))
-        (:db/tupleTypes attr) vs)
-
-      (= :db.type/ref (:db/tupleType attr))
-      (mapv (fn [v]
-              (if (sequential? v) ;; lookup-ref
-                (entid-strict db v)
-                v))
-        vs)
+(defn- #?@(:clj  [^Boolean resolved-eid?]
+           :cljs [^boolean resolved-eid?])
+  [x]
+  (or
+    (nil? x)
+    (and (number? x) (not (neg? x)))))
 
-      :else
-      vs)))
+(defn+ resolve-tuple-refs
+  "Resolves lookup refs and idents in ref slots of a tuple value.
+   Read path: no tempids, unresolved refs raise"
+  [db a vs context]
+  (let [schema-a (-> db -schema (get a))]
+    (validate-tuple db a schema-a vs context)
+    (if-not (sequential? vs) ;; e.g. nil ov in :db.fn/cas
+      vs
+      (into []
+        (map-indexed
+          (fn [idx v]
+            (if (and
+                  (tuple-ref-slot? db schema-a idx)
+                  (not (resolved-eid? v)))
+              (entid-strict db v)
+              v)))
+        vs))))
 
 (defn+ ^number entid [db eid]
   {:pre [(db? db)]}
@@ -1323,13 +1331,13 @@
         (util/raise "Lookup ref attribute should be marked as :db/unique: " eid
           {:error :lookup-ref/unique, :entity-id eid})
 
+        (nil? value)
+        nil
+
         (tuple? db attr)
         (let [value' (resolve-tuple-refs db attr value eid)]
           (-> (-datoms db :avet attr value' nil nil) first :e))
-        
-        (nil? value)
-        nil
-        
+
         :else
         (-> (-datoms db :avet attr value nil nil) first :e)))
 
@@ -1468,7 +1476,8 @@
     (auto-tempid? x)))
 
 (defn- new-eid? [db eid]
-  (and (> eid (:max-eid db))
+  (and
+    (> eid (:max-eid db))
     (< eid tx0))) ;; tx0 is max eid
 
 (defn- advance-max-eid [db eid]
@@ -1497,83 +1506,43 @@
      true
      (update :db-after advance-max-eid eid))))
 
-(defn- #?@(:clj  [^Boolean tuple-slot-resolved?]
-           :cljs [^boolean tuple-slot-resolved?])
-  [x]
-  (or
-    (nil? x)
-    (and (number? x) (not (neg? x)))))
-
-(defn- resolve-tuple-slot [report v]
-  (let [db (:db-after report)]
-    (cond
-      (tuple-slot-resolved? v)
-      [report v]
-
-      (tx-id? v)
-      (let [id (current-tx report)]
-        [(allocate-eid report v id) id])
-
-      (tempid? v)
-      (if-some [resolved (get (:tempids report) v)]
-        [(update report ::value-tempids assoc resolved v) resolved]
-        (let [resolved (next-eid db)]
-          [(-> report
-             (allocate-eid v resolved)
-             (update ::value-tempids assoc resolved v))
-           resolved]))
-
-      :else ;; lookup ref or ident
-      [report (entid-strict db v)])))
-
 (defn- resolve-tuple-value
   "Resoleves refs, tempids and :db/current-tx in ref slots. Returns [report' vs']"
   [report a vs context]
-  (let [db   (:db-after report)
-        attr (-> db -schema (get a))
-        _    (validate-tuple db a attr vs context)]
-    (cond
-      (not (vector? vs))
+  (let [db       (:db-after report)
+        schema-a (-> db -schema (get a))
+        _        (validate-tuple db a schema-a vs context)]
+    (if-not (vector? vs)
       [report vs]
-
-      (:db/tupleAttrs attr)
-      (let [el-attrs (:db/tupleAttrs attr)]
-        (loop [idx     0
-               report' report
-               vs'     (transient [])]
-          (if (>= idx (count vs))
-            [report' (persistent! vs')]
-            (let [v (nth vs idx)]
-              (if (ref? db (nth el-attrs idx nil))
-                (let [[report'' v'] (resolve-tuple-slot report' v)]
-                  (recur (inc idx) report'' (conj! vs' v')))
-                (recur (inc idx) report' (conj! vs' v)))))))
-
-      (:db/tupleTypes attr)
-      (let [types (:db/tupleTypes attr)]
-        (loop [idx     0
-               report' report
-               vs'     (transient [])]
-          (if (>= idx (count vs))
-            [report' (persistent! vs')]
-            (let [v (nth vs idx)]
-              (if (= :db.type/ref (nth types idx nil))
-                (let [[report'' v'] (resolve-tuple-slot report' v)]
-                  (recur (inc idx) report'' (conj! vs' v')))
-                (recur (inc idx) report' (conj! vs' v)))))))
-
-      (= :db.type/ref (:db/tupleType attr))
       (loop [idx     0
              report' report
              vs'     (transient [])]
         (if (>= idx (count vs))
           [report' (persistent! vs')]
-          (let [v             (nth vs idx)
-                [report'' v'] (resolve-tuple-slot report' v)]
-            (recur (inc idx) report'' (conj! vs' v')))))
-
-      :else
-      [report vs])))
+          (let [v (nth vs idx)]
+            (if (tuple-ref-slot? db schema-a idx)
+              (let [[report'' v'] (cond
+                                    (resolved-eid? v)
+                                    [report' v]
+
+                                    (tx-id? v)
+                                    (let [id (current-tx report')]
+                                      [(allocate-eid report' v id) id])
+
+                                    (tempid? v)
+                                    (if-some [resolved (get (:tempids report') v)]
+                                      [(update report' ::value-tempids assoc resolved v) resolved]
+                                      (let [resolved (next-eid db)]
+                                        [(-> report'
+                                             (allocate-eid v resolved)
+                                             (update ::value-tempids assoc resolved v))
+                                         resolved]))
+
+                                    :else ;; lookup ref or ident
+                                    [report' (entid-strict db v)])]
+
+                (recur (inc idx) report'' (conj! vs' v')))
+              (recur (inc idx) report' (conj! vs' v)))))))))
 
 ;; In context of `with-datom` we can use faster comparators which
 ;; do not check for nil (~10-15% performance gain in `transact`)

src/datascript/query.cljc

@@ -263,7 +263,8 @@
 ;;
 
 (def ^{:dynamic true
-       :doc "List of symbols in current pattern that might potentiall be resolved to refs"}
+       :doc "Map of symbol in current pattern that might potentially be resolved to refs
+             => fn that resolves its values"}
   *lookup-attrs* nil)
 
 (def ^{:dynamic true
@@ -272,7 +273,7 @@
 
 (defn getter-fn [attrs attr]
   (let [idx (attrs attr)]
-    (if (contains? *lookup-attrs* attr)
+    (if-some [resolve-fn (get *lookup-attrs* attr)]
       (if (int? idx)
         (let [idx (int idx)]
           (fn contained-int-getter-fn [tuple]
@@ -282,17 +283,17 @@
                                 (nth tuple idx)))]
               (cond
                 (number? eid)     eid ;; quick path to avoid fn call
-                (sequential? eid) (db/entid *implicit-source* eid)
-                (da/array? eid)   (db/entid *implicit-source* eid)
+                (sequential? eid) (resolve-fn eid)
+                (da/array? eid)   (resolve-fn eid)
                 :else             eid))))
         ;; If the index is not an int?, the target can never be an array
         (fn contained-getter-fn [tuple]
           (let [eid #?(:cljs (da/aget tuple idx)
                        :clj (.valAt ^ILookup tuple idx))]
             (cond
               (number? eid)     eid ;; quick path to avoid fn call
-              (sequential? eid) (db/entid *implicit-source* eid)
-              (da/array? eid)   (db/entid *implicit-source* eid)
+              (sequential? eid) (resolve-fn eid)
+              (da/array? eid)   (resolve-fn eid)
               :else             eid))))
       (if (int? idx)
         (let [idx (int idx)]
@@ -405,8 +406,19 @@
           e'         (if (or (lookup-ref? e) (attr? e))
                        (db/entid-strict source e)
                        e)
-          v'         (if (and v (attr? a) (db/ref? source a) (or (lookup-ref? v) (attr? v)))
-                       (db/entid-strict source v)
+          v'         (cond
+                       (or (nil? v) (not (attr? a)))
+                       v
+
+                       (db/ref? source a)
+                       (if (or (lookup-ref? v) (attr? v))
+                         (db/entid-strict source v)
+                         v)
+
+                       (and (db/tuple? source a) (sequential? v))
+                       (db/resolve-tuple-refs source a v pattern)
+
+                       :else
                        v)
           tx'        (if (lookup-ref? tx)
                        (db/entid-strict source tx)
@@ -686,15 +698,18 @@
                         rel))))))))
         rel))))
 
+(defn- resolve-eid [eid]
+  (db/entid *implicit-source* eid))
+
 (defn dynamic-lookup-attrs [source pattern]
-  (let [[e a v tx] pattern]
-    (cond-> #{}
-      (free-var? e) (conj e)
-      (free-var? tx) (conj tx)
-      (and
-        (free-var? v)
-        (not (free-var? a))
-        (db/ref? source a)) (conj v))))
+  (let [[e a v tx] pattern
+        v-ref?     (and (free-var? v) (attr? a) (db/ref? source a))
+        v-tuple?   (and (free-var? v) (attr? a) (db/tuple? source a))]
+    (cond-> {}
+      (free-var? e)  (assoc e  resolve-eid)
+      (free-var? tx) (assoc tx resolve-eid)
+      v-ref?         (assoc v  resolve-eid)
+      v-tuple?       (assoc v  #(db/resolve-tuple-refs *implicit-source* a %1 pattern)))))
 
 (defn limit-rel [rel vars]
   (when-some [attrs' (not-empty (select-keys (:attrs rel) vars))]