From e9887911515ff96a55a46e17d3aa194de280630c Mon Sep 17 00:00:00 2001
From: lkct <liangkct@yahoo.com>
Date: Sat, 9 Dec 2023 16:11:32 +0000
Subject: [PATCH 1/6] repurpose mixing layer and be specific on symb layer

---
 cirkit/new/layers/inner/sum/mixing.py   |  7 ++++++-
 cirkit/new/symbolic/symbolic_circuit.py | 26 +++++++++++++++----------
 cirkit/new/symbolic/symbolic_layer.py   | 17 ++++++++++------
 3 files changed, 33 insertions(+), 17 deletions(-)

diff --git a/cirkit/new/layers/inner/sum/mixing.py b/cirkit/new/layers/inner/sum/mixing.py
index f2eb5281..337dc2e8 100644
--- a/cirkit/new/layers/inner/sum/mixing.py
+++ b/cirkit/new/layers/inner/sum/mixing.py
@@ -6,7 +6,12 @@
 
 
 class MixingLayer(SumLayer):
-    """The sum layer for mixture among layers."""
+    """The sum layer for mixture among layers.
+
+    It can also be used as a sparse sum within a layer when arity=1.
+    """
+
+    # TODO: do we use another name for another purpose?
 
     def __init__(
         self,
diff --git a/cirkit/new/symbolic/symbolic_circuit.py b/cirkit/new/symbolic/symbolic_circuit.py
index 9d7adac3..5c62f582 100644
--- a/cirkit/new/symbolic/symbolic_circuit.py
+++ b/cirkit/new/symbolic/symbolic_circuit.py
@@ -1,6 +1,6 @@
-from typing import Any, Dict, Iterable, Iterator, Optional, Type
+from typing import Any, Dict, Iterable, Iterator, Optional, Type, Union
 
-from cirkit.new.layers import InnerLayer, InputLayer
+from cirkit.new.layers import InputLayer, ProductLayer, SumLayer, SumProductLayer
 from cirkit.new.region_graph import PartitionNode, RegionGraph, RegionNode, RGNode
 from cirkit.new.reparams import Reparameterization
 from cirkit.new.symbolic.symbolic_layer import (
@@ -31,31 +31,37 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
         input_layer_cls: Type[InputLayer],
         input_layer_kwargs: Optional[Dict[str, Any]] = None,
         input_reparam: Optional[Reparameterization] = None,
-        sum_layer_cls: Type[InnerLayer],  # TODO: more specific?
+        sum_layer_cls: Type[Union[SumLayer, SumProductLayer]],
         sum_layer_kwargs: Optional[Dict[str, Any]] = None,
         sum_reparam: Reparameterization,
-        prod_layer_cls: Type[InnerLayer],  # TODO: more specific?
+        prod_layer_cls: Type[Union[ProductLayer, SumProductLayer]],
         prod_layer_kwargs: Optional[Dict[str, Any]] = None,
     ):
         """Construct symbolic circuit from a region graph.
 
         Args:
             region_graph (RegionGraph): The region graph to convert.
-            num_input_units (int): _description_
-            num_sum_units (int): _description_
-            num_classes (int, optional): _description_. Defaults to 1.
+            num_input_units (int): The number of units in the input layer.
+            num_sum_units (int): The number of units in the sum layer. Will also be used to infer \
+                the number of product units.
+            num_classes (int, optional): The number of classes of the circuit output, i.e., the \
+                number of units in the output layer. Defaults to 1.
             input_layer_cls (Type[InputLayer]): The layer class for input layers.
             input_layer_kwargs (Optional[Dict[str, Any]], optional): The additional kwargs for \
                 input layer class. Defaults to None.
             input_reparam (Optional[Reparameterization], optional): The reparameterization for \
                 input layer parameters, can be None if it has no params. Defaults to None.
-            sum_layer_cls (Type[InnerLayer]): The layer class for sum layers.
+            sum_layer_cls (Type[Union[SumLayer, SumProductLayer]]): The layer class for sum \
+                layers, can be either just a class of SumLayer, or a class of SumProductLayer to \
+                indicate layer fusion..
             sum_layer_kwargs (Optional[Dict[str, Any]], optional): The additional kwargs for sum \
                 layer class. Defaults to None.
             sum_reparam (Reparameterization): The reparameterization for sum layer parameters.
-            prod_layer_cls (Type[InnerLayer]): The layer class for product layers.
+            prod_layer_cls (Type[Union[ProductLayer, SumProductLayer]]): The layer class for \
+                product layers, can be either just a class of ProductLayer, or a class of \
+                SumProductLayer to indicate layer fusion.
             prod_layer_kwargs (Optional[Dict[str, Any]], optional): The additional kwargs for \
-                product layer class. Defaults to None.
+                product layer class, will be ignored if SumProductLayer is used. Defaults to None.
         """
         self.region_graph = region_graph
         self.scope = region_graph.scope
diff --git a/cirkit/new/symbolic/symbolic_layer.py b/cirkit/new/symbolic/symbolic_layer.py
index 21f27e57..98841da8 100644
--- a/cirkit/new/symbolic/symbolic_layer.py
+++ b/cirkit/new/symbolic/symbolic_layer.py
@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Iterable, Optional, Type
+from typing import Any, Dict, Iterable, Optional, Type, Union
 
-from cirkit.new.layers import InnerLayer, InputLayer, Layer
+from cirkit.new.layers import InputLayer, Layer, ProductLayer, SumLayer, SumProductLayer
 from cirkit.new.region_graph import PartitionNode, RegionNode, RGNode
 from cirkit.new.reparams import Reparameterization
 from cirkit.new.utils import OrderedSet
@@ -53,6 +53,7 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
             rg_node.inputs
         ), "The number of inputs to this layer does not match the RG."
 
+        self.arity = len(self.inputs)
         self.num_units = num_units
         self.layer_cls = layer_cls
         # Ignore: Unavoidable for kwargs.
@@ -103,7 +104,7 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
         layers_in: Iterable[SymbolicLayer],
         *,
         num_units: int,
-        layer_cls: Type[InnerLayer],  # TODO: more specific?
+        layer_cls: Type[Union[SumLayer, SumProductLayer]],
         layer_kwargs: Optional[Dict[str, Any]] = None,
         reparam: Reparameterization,
     ) -> None:
@@ -113,7 +114,9 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
             rg_node (RegionNode): The region node corresponding to this layer.
             layers_in (Iterable[SymbolicLayer]): The input to this layer.
             num_units (int): The number of units in this layer.
-            layer_cls (Type[InnerLayer]): The concrete layer class to become.
+            layer_cls (Type[Union[SumLayer, SumProductLayer]]): The concrete layer class to \
+                become, can be either just a class of SumLayer, or a class of SumProductLayer to \
+                indicate layer fusion.
             layer_kwargs (Optional[Dict[str, Any]], optional): The additional kwargs to initialize \
                 layer_cls. Defaults to None.
             reparam (Reparameterization): The reparameterization for layer parameters.
@@ -160,7 +163,7 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
         layers_in: Iterable[SymbolicLayer],
         *,
         num_units: int,
-        layer_cls: Type[InnerLayer],  # TODO: more specific?
+        layer_cls: Type[Union[ProductLayer, SumProductLayer]],
         layer_kwargs: Optional[Dict[str, Any]] = None,
         reparam: Optional[Reparameterization] = None,
     ) -> None:
@@ -170,7 +173,9 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
             rg_node (PartitionNode): The partition node corresponding to this layer.
             layers_in (Iterable[SymbolicLayer]): The input to this layer.
             num_units (int): The number of units in this layer.
-            layer_cls (Type[InnerLayer]): The concrete layer class to become.
+            layer_cls (Type[Union[ProductLayer, SumProductLayer]]): The concrete layer class to \
+                become, can be either just a class of ProductLayer, or a class of SumProductLayer \
+                to indicate layer fusion.
             layer_kwargs (Optional[Dict[str, Any]], optional): The additional kwargs to initialize \
                 layer_cls. Defaults to None.
             reparam (Optional[Reparameterization], optional): Ignored. This layer has no params. \

From 3fd3bdf2d863837dd3aa1ce2e54a8b412b9a1c5c Mon Sep 17 00:00:00 2001
From: lkct <liangkct@yahoo.com>
Date: Sat, 9 Dec 2023 16:27:59 +0000
Subject: [PATCH 2/6] update ordered set

---
 cirkit/new/region_graph/region_graph.py |  5 +++--
 cirkit/new/utils/ordered_set.py         | 22 +++++++++++++++++-----
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/cirkit/new/region_graph/region_graph.py b/cirkit/new/region_graph/region_graph.py
index a60ca2ee..9fb1d2a9 100644
--- a/cirkit/new/region_graph/region_graph.py
+++ b/cirkit/new/region_graph/region_graph.py
@@ -77,8 +77,9 @@ def add_edge(self, tail: RGNode, head: RGNode) -> None:
         # add_node will check for _is_frozen.
         self.add_node(tail)
         self.add_node(head)
-        tail.outputs.append(head)  # TODO: this insertion order may be different from add_node order
-        head.inputs.append(tail)
+        # TODO: this insertion order may be different from add_node order
+        assert tail.outputs.append(head), "The edges in RG should not be repeated."
+        head.inputs.append(tail)  # Only need to check duplicate in one direction.
 
     def add_partitioning(self, region: RegionNode, sub_regions: Iterable[RegionNode]) -> None:
         """Add a partitioning structure to the graph, with a PartitionNode constructed internally.
diff --git a/cirkit/new/utils/ordered_set.py b/cirkit/new/utils/ordered_set.py
index 3fe4e2f1..73c74635 100644
--- a/cirkit/new/utils/ordered_set.py
+++ b/cirkit/new/utils/ordered_set.py
@@ -1,4 +1,4 @@
-from typing import Any, Collection, Dict, Iterable, Iterator, Literal, Protocol, TypeVar
+from typing import Any, Collection, Dict, Iterable, Iterator, Protocol, TypeVar
 from typing_extensions import Self  # TODO: in typing from 3.11
 
 
@@ -40,8 +40,8 @@ def __init__(self, *iterables: Iterable[ComparableT]) -> None:
         """
         super().__init__()
         # The dict values are unused and always set to True.
-        self._container: Dict[ComparableT, Literal[True]] = {
-            element: True for iterable in iterables for element in iterable
+        self._container: Dict[ComparableT, ComparableT] = {
+            element: element for iterable in iterables for element in iterable
         }
 
     # Ignore: We should only test the element type.
@@ -87,9 +87,21 @@ def append(self, element: ComparableT) -> bool:
         if element in self:
             return False  # Meaning it's a no-op.
 
-        self._container[element] = True
+        self._container[element] = element
         return True  # Meaning a successful append.
 
+    def extend(self, elements: Iterable[ComparableT]) -> None:
+        """Add elements to the end of the set, for each one in order that does not exist yet; \
+        duplicates are skipped.
+
+        The input iterable should have a deterministic order and the order is preserved.
+
+        Args:
+            elements (Iterable[ComparableT]): The elements to extend.
+        """
+        for element in elements:
+            self.append(element)
+
     def sort(self) -> Self:
         """Sort the set inplace and return self.
 
@@ -101,5 +113,5 @@ def sort(self) -> Self:
             Self: The self object.
         """
         # This relies on that sorted() is stable.
-        self._container = {element: True for element in sorted(self._container)}
+        self._container = {element: element for element in sorted(self._container)}
         return self

From 13058980b814d46290165ebbcc161f6d743fa244 Mon Sep 17 00:00:00 2001
From: lkct <liangkct@yahoo.com>
Date: Sat, 9 Dec 2023 16:50:35 +0000
Subject: [PATCH 3/6] rearrange SymbC structure, closer to concrete C

---
 cirkit/new/symbolic/symbolic_circuit.py     | 78 ++++++++++++++++-----
 cirkit/new/symbolic/symbolic_layer.py       | 11 ++-
 tests/new/symbolic/test_symbolic_circuit.py |  4 +-
 3 files changed, 66 insertions(+), 27 deletions(-)

diff --git a/cirkit/new/symbolic/symbolic_circuit.py b/cirkit/new/symbolic/symbolic_circuit.py
index 5c62f582..25ce6a64 100644
--- a/cirkit/new/symbolic/symbolic_circuit.py
+++ b/cirkit/new/symbolic/symbolic_circuit.py
@@ -1,6 +1,6 @@
 from typing import Any, Dict, Iterable, Iterator, Optional, Type, Union
 
-from cirkit.new.layers import InputLayer, ProductLayer, SumLayer, SumProductLayer
+from cirkit.new.layers import InputLayer, MixingLayer, ProductLayer, SumLayer, SumProductLayer
 from cirkit.new.region_graph import PartitionNode, RegionGraph, RegionNode, RGNode
 from cirkit.new.reparams import Reparameterization
 from cirkit.new.symbolic.symbolic_layer import (
@@ -9,7 +9,7 @@
     SymbolicProductLayer,
     SymbolicSumLayer,
 )
-from cirkit.new.utils import Scope
+from cirkit.new.utils import OrderedSet, Scope
 
 # TODO: double check docs and __repr__
 
@@ -71,23 +71,41 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
         self.is_structured_decomposable = region_graph.is_structured_decomposable
         self.is_omni_compatible = region_graph.is_omni_compatible
 
-        node_layer: Dict[RGNode, SymbolicLayer] = {}
+        self._layers: OrderedSet[SymbolicLayer] = OrderedSet()
+        # The RGNode and SymbolicLayer does not map 1-to-1 but 1-to-many. This still leads to a
+        # deterministic order: SymbolicLayer of the same RGNode are adjcent, and ordered based on
+        # the order of edges in the RG.
+
+        node_layer: Dict[RGNode, SymbolicLayer] = {}  # Map RGNode to its "output" SymbolicLayer.
 
         for rg_node in region_graph.nodes:
-            layers_in = (node_layer[node_in] for node_in in rg_node.inputs)
-            layer: SymbolicLayer
+            # Cannot use a generator as layers_in, because it's used twice.
+            layers_in = [node_layer[node_in] for node_in in rg_node.inputs]
+            layer_out: SymbolicLayer
             # Ignore: Unavoidable for kwargs.
-            if isinstance(rg_node, RegionNode) and not rg_node.inputs:  # Input node.
-                layer = SymbolicInputLayer(
+            if isinstance(rg_node, RegionNode) and not rg_node.inputs:  # Input region.
+                layers_in = [
+                    SymbolicInputLayer(
+                        rg_node,
+                        (),  # Old layers_in should be empty.
+                        num_units=num_input_units,
+                        layer_cls=input_layer_cls,
+                        layer_kwargs=input_layer_kwargs,  # type: ignore[misc]
+                        reparam=input_reparam,
+                    )
+                ]
+                # This also works when the input is also output, in which case num_classes is used.
+                layer_out = SymbolicSumLayer(
                     rg_node,
                     layers_in,
-                    num_units=num_input_units,
-                    layer_cls=input_layer_cls,
-                    layer_kwargs=input_layer_kwargs,  # type: ignore[misc]
-                    reparam=input_reparam,
+                    num_units=num_sum_units if rg_node.outputs else num_classes,
+                    layer_cls=sum_layer_cls,
+                    layer_kwargs=sum_layer_kwargs,  # type: ignore[misc]
+                    reparam=sum_reparam,
                 )
-            elif isinstance(rg_node, RegionNode) and rg_node.inputs:  # Inner region node.
-                layer = SymbolicSumLayer(
+            elif isinstance(rg_node, RegionNode) and len(rg_node.inputs) == 1:  # Simple inner.
+                # layers_in keeps the same.
+                layer_out = SymbolicSumLayer(
                     rg_node,
                     layers_in,
                     num_units=num_sum_units if rg_node.outputs else num_classes,
@@ -95,8 +113,30 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
                     layer_kwargs=sum_layer_kwargs,  # type: ignore[misc]
                     reparam=sum_reparam,
                 )
-            elif isinstance(rg_node, PartitionNode):  # Partition node.
-                layer = SymbolicProductLayer(
+            elif isinstance(rg_node, RegionNode) and len(rg_node.inputs) > 1:  # Inner with mixture.
+                # MixingLayer cannot change number of units, so must project early.
+                layers_in = [
+                    SymbolicSumLayer(
+                        rg_node,
+                        (layer_in,),
+                        num_units=num_sum_units if rg_node.outputs else num_classes,
+                        layer_cls=sum_layer_cls,
+                        layer_kwargs=sum_layer_kwargs,  # type: ignore[misc]
+                        reparam=sum_reparam,
+                    )
+                    for layer_in in layers_in
+                ]
+                layer_out = SymbolicSumLayer(
+                    rg_node,
+                    layers_in,
+                    num_units=num_sum_units if rg_node.outputs else num_classes,
+                    layer_cls=MixingLayer,
+                    layer_kwargs={},  # type: ignore[misc]
+                    reparam=sum_reparam,  # TODO: use a constant reparam here?
+                )
+            elif isinstance(rg_node, PartitionNode):
+                # layers_in keeps the same.
+                layer_out = SymbolicProductLayer(
                     rg_node,
                     layers_in,
                     num_units=prod_layer_cls._infer_num_prod_units(
@@ -108,9 +148,9 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
                 )
             else:
                 assert False, "This should not happen."
-            node_layer[rg_node] = layer
-
-        self._node_layer = node_layer  # Insertion order is preserved by dict@py3.7+.
+            self._layers.extend(layers_in)  # May be existing layers from node_layer, or new layers.
+            self._layers.append(layer_out)
+            node_layer[rg_node] = layer_out
 
     #######################################    Properties    #######################################
     # Here are the basic properties and some structural properties of the SymbC. Some of them are
@@ -159,7 +199,7 @@ def is_compatible(
     @property
     def layers(self) -> Iterator[SymbolicLayer]:
         """All layers in the circuit."""
-        return iter(self._node_layer.values())
+        return iter(self._layers)
 
     @property
     def sum_layers(self) -> Iterator[SymbolicSumLayer]:
diff --git a/cirkit/new/symbolic/symbolic_layer.py b/cirkit/new/symbolic/symbolic_layer.py
index 98841da8..8dabdbe6 100644
--- a/cirkit/new/symbolic/symbolic_layer.py
+++ b/cirkit/new/symbolic/symbolic_layer.py
@@ -49,9 +49,6 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
         for layer_in in layers_in:
             self.inputs.append(layer_in)
             layer_in.outputs.append(self)
-        assert len(self.inputs) == len(
-            rg_node.inputs
-        ), "The number of inputs to this layer does not match the RG."
 
         self.arity = len(self.inputs)
         self.num_units = num_units
@@ -85,7 +82,9 @@ def __lt__(self, other: "SymbolicLayer") -> bool:
         Returns:
             bool: Whether self < other.
         """
-        return self.rg_node < other.rg_node
+        return (
+            self.rg_node < other.rg_node or self.rg_node == other.rg_node and self in other.outputs
+        )  # Either the corresponding rg_node precedes, or for same rg_node, self directly precedes.
 
 
 # Disable: It's intended for SymbolicSumLayer to have only these methods.
@@ -121,7 +120,6 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
                 layer_cls. Defaults to None.
             reparam (Reparameterization): The reparameterization for layer parameters.
         """
-        assert rg_node.inputs, "SymbolicSumLayer must be based on an inner RegionNode."
         super().__init__(
             rg_node,
             layers_in,
@@ -130,6 +128,7 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
             layer_kwargs=layer_kwargs,  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
             reparam=reparam,
         )
+        assert self.inputs, "SymbolicSumLayer must be an inner layer of the SymbC."
 
     def __repr__(self) -> str:
         """Generate the repr string of the layer.
@@ -235,7 +234,6 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
             reparam (Optional[Reparameterization], optional): The reparameterization for layer \
                 parameters, can be None if layer_cls has no params. Defaults to None.
         """
-        assert not rg_node.inputs, "SymbolicInputLayer must be based on an input RegionNode."
         super().__init__(
             rg_node,
             layers_in,  # Should be empty, will be tested in super().__init__ by its length.
@@ -244,6 +242,7 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
             layer_kwargs=layer_kwargs,  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
             reparam=reparam,
         )
+        assert not self.inputs, "SymbolicInputLayer must be an input layer of the SymbC."
 
     def __repr__(self) -> str:
         """Generate the repr string of the layer.
diff --git a/tests/new/symbolic/test_symbolic_circuit.py b/tests/new/symbolic/test_symbolic_circuit.py
index 59d89a6a..270f9599 100644
--- a/tests/new/symbolic/test_symbolic_circuit.py
+++ b/tests/new/symbolic/test_symbolic_circuit.py
@@ -11,7 +11,7 @@ def test_symbolic_circuit_simple() -> None:
 
     circuit = get_symbolic_circuit_on_rg(rg)
 
-    assert len(list(circuit.layers)) == 4
+    assert len(list(circuit.layers)) == 6
     # Ignore: SymbolicInputLayer contains Any.
     assert all(
         isinstance(layer, SymbolicInputLayer)  # type: ignore[misc]
@@ -28,7 +28,7 @@ def test_symbolic_circuit_qt() -> None:
 
     circuit = get_symbolic_circuit_on_rg(rg)
 
-    assert len(list(circuit.layers)) == 46
+    assert len(list(circuit.layers)) == 62
     assert len(list(circuit.input_layers)) == 16
     assert len(list(circuit.output_layers)) == 1
     assert circuit.scope == Scope(range(16))

From e2e4c7a7fba51a011ee5a9e3ee4319fb0912f833 Mon Sep 17 00:00:00 2001
From: lkct <liangkct@yahoo.com>
Date: Sat, 9 Dec 2023 21:24:23 +0000
Subject: [PATCH 4/6] renaming

---
 cirkit/new/symbolic/__init__.py         |  2 +-
 cirkit/new/symbolic/symbolic_circuit.py | 10 +++++-----
 tests/new/symbolic/test_utils.py        |  6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/cirkit/new/symbolic/__init__.py b/cirkit/new/symbolic/__init__.py
index 1c699ea4..627ca669 100644
--- a/cirkit/new/symbolic/__init__.py
+++ b/cirkit/new/symbolic/__init__.py
@@ -1,4 +1,4 @@
-from .symbolic_circuit import SymbolicCircuit as SymbolicCircuit
+from .symbolic_circuit import SymbolicTensorizedCircuit as SymbolicTensorizedCircuit
 from .symbolic_layer import SymbolicInputLayer as SymbolicInputLayer
 from .symbolic_layer import SymbolicLayer as SymbolicLayer
 from .symbolic_layer import SymbolicProductLayer as SymbolicProductLayer
diff --git a/cirkit/new/symbolic/symbolic_circuit.py b/cirkit/new/symbolic/symbolic_circuit.py
index 25ce6a64..8eed551f 100644
--- a/cirkit/new/symbolic/symbolic_circuit.py
+++ b/cirkit/new/symbolic/symbolic_circuit.py
@@ -15,7 +15,7 @@
 
 
 # Disable: It's designed to have these many attributes.
-class SymbolicCircuit:  # pylint: disable=too-many-instance-attributes
+class SymbolicTensorizedCircuit:  # pylint: disable=too-many-instance-attributes
     """The symbolic representation of a tensorized circuit."""
 
     # TODO: how to design interface? require kwargs only?
@@ -76,11 +76,11 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
         # deterministic order: SymbolicLayer of the same RGNode are adjcent, and ordered based on
         # the order of edges in the RG.
 
-        node_layer: Dict[RGNode, SymbolicLayer] = {}  # Map RGNode to its "output" SymbolicLayer.
+        node_to_layer: Dict[RGNode, SymbolicLayer] = {}  # Map RGNode to its "output" SymbolicLayer.
 
         for rg_node in region_graph.nodes:
             # Cannot use a generator as layers_in, because it's used twice.
-            layers_in = [node_layer[node_in] for node_in in rg_node.inputs]
+            layers_in = [node_to_layer[node_in] for node_in in rg_node.inputs]
             layer_out: SymbolicLayer
             # Ignore: Unavoidable for kwargs.
             if isinstance(rg_node, RegionNode) and not rg_node.inputs:  # Input region.
@@ -150,7 +150,7 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
                 assert False, "This should not happen."
             self._layers.extend(layers_in)  # May be existing layers from node_layer, or new layers.
             self._layers.append(layer_out)
-            node_layer[rg_node] = layer_out
+            node_to_layer[rg_node] = layer_out
 
     #######################################    Properties    #######################################
     # Here are the basic properties and some structural properties of the SymbC. Some of them are
@@ -176,7 +176,7 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
     """Whether the SymbC is omni-compatible, i.e., compatible to all circuits of the same scope."""
 
     def is_compatible(
-        self, other: "SymbolicCircuit", *, scope: Optional[Iterable[int]] = None
+        self, other: "SymbolicTensorizedCircuit", *, scope: Optional[Iterable[int]] = None
     ) -> bool:
         """Test compatibility with another symbolic circuit over the given scope.
 
diff --git a/tests/new/symbolic/test_utils.py b/tests/new/symbolic/test_utils.py
index 16c7fa0e..50f892af 100644
--- a/tests/new/symbolic/test_utils.py
+++ b/tests/new/symbolic/test_utils.py
@@ -4,7 +4,7 @@
 from cirkit.new.layers import CategoricalLayer, CPLayer
 from cirkit.new.region_graph import RegionGraph, RegionNode
 from cirkit.new.reparams import ExpReparam
-from cirkit.new.symbolic import SymbolicCircuit
+from cirkit.new.symbolic import SymbolicTensorizedCircuit
 
 
 def get_simple_rg() -> RegionGraph:
@@ -16,7 +16,7 @@ def get_simple_rg() -> RegionGraph:
     return rg.freeze()
 
 
-def get_symbolic_circuit_on_rg(rg: RegionGraph) -> SymbolicCircuit:
+def get_symbolic_circuit_on_rg(rg: RegionGraph) -> SymbolicTensorizedCircuit:
     num_units = 4
     input_cls = CategoricalLayer
     input_kwargs = {"num_categories": 256}
@@ -24,7 +24,7 @@ def get_symbolic_circuit_on_rg(rg: RegionGraph) -> SymbolicCircuit:
     inner_kwargs: Dict[str, None] = {}  # Avoid Any.
     reparam = ExpReparam()
 
-    return SymbolicCircuit(
+    return SymbolicTensorizedCircuit(
         rg,
         num_input_units=num_units,
         num_sum_units=num_units,

From 69dcf473f2280a362e04b160b4662b29a3e8fb7f Mon Sep 17 00:00:00 2001
From: lkct <liangkct@yahoo.com>
Date: Sat, 9 Dec 2023 21:35:14 +0000
Subject: [PATCH 5/6] prefer list over ordered set if no dup

---
 cirkit/new/symbolic/symbolic_circuit.py | 5 ++++-
 cirkit/new/symbolic/symbolic_layer.py   | 9 ++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/cirkit/new/symbolic/symbolic_circuit.py b/cirkit/new/symbolic/symbolic_circuit.py
index 8eed551f..6ef688bb 100644
--- a/cirkit/new/symbolic/symbolic_circuit.py
+++ b/cirkit/new/symbolic/symbolic_circuit.py
@@ -148,7 +148,10 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
                 )
             else:
                 assert False, "This should not happen."
-            self._layers.extend(layers_in)  # May be existing layers from node_layer, or new layers.
+            # layers_in may be existing layers (from node_layer) which will be de-duplicated by
+            # OrderedSet, or newly constructed layers to be added.
+            self._layers.extend(layers_in)
+            # layer_out is what will be connected to the output of rg_node.
             self._layers.append(layer_out)
             node_to_layer[rg_node] = layer_out
 
diff --git a/cirkit/new/symbolic/symbolic_layer.py b/cirkit/new/symbolic/symbolic_layer.py
index 8dabdbe6..c9ab7eed 100644
--- a/cirkit/new/symbolic/symbolic_layer.py
+++ b/cirkit/new/symbolic/symbolic_layer.py
@@ -1,10 +1,9 @@
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Iterable, Optional, Type, Union
+from typing import Any, Dict, Iterable, List, Optional, Type, Union
 
 from cirkit.new.layers import InputLayer, Layer, ProductLayer, SumLayer, SumProductLayer
 from cirkit.new.region_graph import PartitionNode, RegionNode, RGNode
 from cirkit.new.reparams import Reparameterization
-from cirkit.new.utils import OrderedSet
 
 # TODO: double check __repr__
 
@@ -43,9 +42,9 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
         self.scope = rg_node.scope
 
         # self.inputs is filled using layers_in, while self.outputs is empty until self appears in
-        # another layer's layers_in.
-        self.inputs: OrderedSet[SymbolicLayer] = OrderedSet()
-        self.outputs: OrderedSet[SymbolicLayer] = OrderedSet()
+        # another layer's layers_in. No need to de-duplicate, so prefer list over OrderedSet.
+        self.inputs: List[SymbolicLayer] = []
+        self.outputs: List[SymbolicLayer] = []
         for layer_in in layers_in:
             self.inputs.append(layer_in)
             layer_in.outputs.append(self)

From c8aa97f0b9471da4e492bd8cb3a6ab7936d00da0 Mon Sep 17 00:00:00 2001
From: lkct <liangkct@yahoo.com>
Date: Sat, 9 Dec 2023 23:19:52 +0000
Subject: [PATCH 6/6] build concrete circuit

---
 cirkit/new/model/__init__.py            |   1 +
 cirkit/new/model/tensorized_circuit.py  | 142 ++++++++++++++++++++++++
 cirkit/new/region_graph/region_graph.py |   6 +-
 cirkit/new/symbolic/symbolic_circuit.py |   5 +
 cirkit/new/symbolic/symbolic_layer.py   |   2 +-
 5 files changed, 154 insertions(+), 2 deletions(-)
 create mode 100644 cirkit/new/model/__init__.py
 create mode 100644 cirkit/new/model/tensorized_circuit.py

diff --git a/cirkit/new/model/__init__.py b/cirkit/new/model/__init__.py
new file mode 100644
index 00000000..86f5fa61
--- /dev/null
+++ b/cirkit/new/model/__init__.py
@@ -0,0 +1 @@
+from .tensorized_circuit import TensorizedCircuit as TensorizedCircuit
diff --git a/cirkit/new/model/tensorized_circuit.py b/cirkit/new/model/tensorized_circuit.py
new file mode 100644
index 00000000..ed2d93ad
--- /dev/null
+++ b/cirkit/new/model/tensorized_circuit.py
@@ -0,0 +1,142 @@
+from typing import Dict, Optional
+
+import torch
+from torch import Tensor, nn
+
+from cirkit.new.layers import InputLayer, Layer, SumProductLayer
+from cirkit.new.symbolic import (
+    SymbolicLayer,
+    SymbolicProductLayer,
+    SymbolicSumLayer,
+    SymbolicTensorizedCircuit,
+)
+
+
+class TensorizedCircuit(nn.Module):
+    """The tensorized circuit with concrete computational graph in PyTorch.
+    
+    This class is aimed for computation, and therefore does not include excessive strutural \
+    properties. If those are really needed, use the properties of TensorizedCircuit.symb_circuit.
+    """
+
+    # TODO: do we also move num_channels to SymbolicTensorizedCircuit?
+    def __init__(self, symb_circuit: SymbolicTensorizedCircuit, *, num_channels: int) -> None:
+        """Init class.
+
+        All the other config other than num_channels should be provided to the symbolic form.
+
+        Args:
+            symb_circuit (SymbolicTensorizedCircuit): The symbolic version of the circuit.
+            num_channels (int): The number of channels in the input.
+        """
+        super().__init__()
+        self.symb_circuit = symb_circuit
+        self.scope = symb_circuit.scope
+        self.num_vars = symb_circuit.num_vars
+
+        self.layers = nn.ModuleList()  # Automatic layer registry, also publically available.
+
+        # TODO: or do we store edges in Layer?
+        # The actual internal container for forward.
+        self._symb_to_layers: Dict[SymbolicLayer, Optional[Layer]] = {}
+
+        for symb_layer in symb_circuit.layers:
+            layer: Optional[Layer]
+            # Ignore: all SymbolicLayer contains Any.
+            # Ignore: Unavoidable for kwargs.
+            if issubclass(symb_layer.layer_cls, SumProductLayer) and isinstance(
+                symb_layer, SymbolicProductLayer  # type: ignore[misc]
+            ):  # Sum-product fusion at prod: build the actual layer with arity of prod.
+                # len(symb_layer.outputs) == 1 should be guaranteed by PartitionNode.
+                next_layer = symb_layer.outputs[0]  # There should be exactly one SymbSum output.
+                assert (
+                    isinstance(next_layer, SymbolicSumLayer)  # type: ignore[misc]
+                    and next_layer.layer_cls == symb_layer.layer_cls
+                ), "Sum-product fusion inconsistent."
+                layer = symb_layer.layer_cls(
+                    # TODO: is it good to use only [0]?
+                    num_input_units=symb_layer.inputs[0].num_units,
+                    num_output_units=next_layer.num_units,
+                    arity=symb_layer.arity,
+                    reparam=next_layer.reparam,
+                    **next_layer.layer_kwargs,  # type: ignore[misc]
+                )
+            elif issubclass(symb_layer.layer_cls, SumProductLayer) and isinstance(
+                symb_layer, SymbolicSumLayer  # type: ignore[misc]
+            ):  # Sum-product fusion at sum: just run checks and fill a placeholder.
+                prev_layer = symb_layer.inputs[0]  # There should be at exactly SymbProd input.
+                assert (
+                    len(symb_layer.inputs) == 1  # I.e., symb_layer.arity == 1.
+                    and isinstance(prev_layer, SymbolicProductLayer)  # type: ignore[misc]
+                    and prev_layer.layer_cls == symb_layer.layer_cls
+                ), "Sum-product fusion inconsistent."
+                layer = None
+            elif not issubclass(symb_layer.layer_cls, SumProductLayer):  # Normal layers.
+                layer = symb_layer.layer_cls(
+                    # TODO: is it good to use only [0]?
+                    num_input_units=(  # num_channels for InputLayers or num_units of prev layer.
+                        symb_layer.inputs[0].num_units if symb_layer.inputs else num_channels
+                    ),
+                    num_output_units=symb_layer.num_units,
+                    arity=symb_layer.arity,
+                    reparam=symb_layer.reparam,
+                    **symb_layer.layer_kwargs,  # type: ignore[misc]
+                )
+            else:
+                # NOTE: In the above if/elif, we made all conditions explicit to make it more
+                #       readable and also easier for static analysis inside the blocks. Yet the
+                #       completeness cannot be inferred and is only guaranteed by larger picture.
+                #       Also, should anything really go wrong, we will hit this guard statement
+                #       instead of going into a wrong branch.
+                assert False, "This should not happen."
+            if layer is not None:  # Only register actual layers.
+                self.layers.append(layer)
+            self._symb_to_layers[symb_layer] = layer  # But keep a complete mapping.
+
+    def __call__(self, x: Tensor) -> Tensor:
+        """Invoke the forward function.
+
+        Args:
+            x (Tensor): The input of the circuit, shape (*B, D, C).
+
+        Returns:
+            Tensor: The output of the circuit, shape (*B, num_out, num_cls).
+        """  # TODO: single letter name?
+        # Ignore: Idiom for nn.Module.__call__.
+        return super().__call__(x)  # type: ignore[no-any-return,misc]
+
+    # TODO: do we accept each variable separately?
+    def forward(self, x: Tensor) -> Tensor:
+        """Invoke the forward function.
+
+        Args:
+            x (Tensor): The input of the circuit, shape (*B, D, C).
+
+        Returns:
+            Tensor: The output of the circuit, shape (*B, num_out, num_cls).
+        """
+        layer_outputs: Dict[SymbolicLayer, Tensor] = {}  # shape (*B, K).
+
+        for symb_layer, layer in self._symb_to_layers.items():
+            if layer is None:
+                assert (
+                    len(symb_layer.inputs) == 1
+                ), "Only symbolic layers with arity=1 can be implemented by a place-holder."
+                layer_outputs[symb_layer] = layer_outputs[symb_layer.inputs[0]]
+                continue
+
+            # Disable: Ternary will be too long for readability.
+            if isinstance(layer, InputLayer):  # pylint: disable=consider-ternary-expression
+                # TODO: mypy bug? tuple(symb_layer.scope) is inferred to Any
+                layer_input = x[..., tuple(symb_layer.scope), :].movedim(  # type: ignore[misc]
+                    -2, 0
+                )  # shape (*B, D, C) -> (H=D, *B, K=C).
+            else:
+                layer_input = torch.stack(
+                    [layer_outputs[layer_in] for layer_in in symb_layer.inputs], dim=0
+                )  # shape H * (*B, K) -> (H, *B, K).
+            layer_outputs[symb_layer] = layer(layer_input)
+
+        return torch.stack(
+            [layer_outputs[layer_out] for layer_out in self.symb_circuit.output_layers], dim=-2
+        )  # shape num_out * (*B, K) -> (*B, num_out, num_cls=K).
diff --git a/cirkit/new/region_graph/region_graph.py b/cirkit/new/region_graph/region_graph.py
index 9fb1d2a9..d90fedf0 100644
--- a/cirkit/new/region_graph/region_graph.py
+++ b/cirkit/new/region_graph/region_graph.py
@@ -121,7 +121,8 @@ def _sort_nodes(self) -> None:
             node.inputs.sort()
             node.outputs.sort()
 
-    def _validate(self) -> str:
+    # TODO: do we need these return? or just assert?
+    def _validate(self) -> str:  # pylint: disable=too-many-return-statements
         """Validate the RG structure to make sure it's a legal computational graph.
 
         Returns:
@@ -133,6 +134,9 @@ def _validate(self) -> str:
         if next(self.output_nodes, None) is None:
             return "RG must have at least one output node"
 
+        # Also guarantees the input/output nodes are all regions.
+        if not all(partition.inputs for partition in self.partition_nodes):
+            return "PartitionNode must have at least one input"
         if any(len(partition.outputs) != 1 for partition in self.partition_nodes):
             return "PartitionNode can only have one output RegionNode"
 
diff --git a/cirkit/new/symbolic/symbolic_circuit.py b/cirkit/new/symbolic/symbolic_circuit.py
index 6ef688bb..9fd2de2a 100644
--- a/cirkit/new/symbolic/symbolic_circuit.py
+++ b/cirkit/new/symbolic/symbolic_circuit.py
@@ -147,6 +147,11 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
                     reparam=None,
                 )
             else:
+                # NOTE: In the above if/elif, we made all conditions explicit to make it more
+                #       readable and also easier for static analysis inside the blocks. Yet the
+                #       completeness cannot be inferred and is only guaranteed by larger picture.
+                #       Also, should anything really go wrong, we will hit this guard statement
+                #       instead of going into a wrong branch.
                 assert False, "This should not happen."
             # layers_in may be existing layers (from node_layer) which will be de-duplicated by
             # OrderedSet, or newly constructed layers to be added.
diff --git a/cirkit/new/symbolic/symbolic_layer.py b/cirkit/new/symbolic/symbolic_layer.py
index c9ab7eed..294e6f3a 100644
--- a/cirkit/new/symbolic/symbolic_layer.py
+++ b/cirkit/new/symbolic/symbolic_layer.py
@@ -49,7 +49,7 @@ def __init__(  # type: ignore[misc]  # Ignore: Unavoidable for kwargs.
             self.inputs.append(layer_in)
             layer_in.outputs.append(self)
 
-        self.arity = len(self.inputs)
+        self.arity = len(self.inputs) if self.inputs else 1  # InputLayer is defined with artiy=1.
         self.num_units = num_units
         self.layer_cls = layer_cls
         # Ignore: Unavoidable for kwargs.