Smaller updates. Push to CP-SAT 9.10. Implementing #3

_app/input_log.py

@@ -1,6 +1,7 @@
 import streamlit as st
 import os
 
+
 def get_data_from_url(url):
     import urllib.request
     import urllib.parse

_app/overview.py

@@ -25,7 +25,7 @@ def show_overview(parser):
         response_block = parser.get_block_of_type(ResponseBlock)
         col1, col2 = st.columns(2)
         major, minor, patch = solver_block.get_parsed_version()
-        if major < 9 or (major == 9 and minor < 9):
+        if major < 9 or (major == 9 and minor < 10):
             col1.metric(
                 label="CP-SAT Version",
                 value=solver_block.get_version(),

app.py

@@ -56,7 +56,7 @@
                     if fig_3:
                         st.plotly_chart(fig_3, use_container_width=True)
                         st.info(
-                            "This plot shows you how the gap between the objective and the bound changes over time. If it quickly reaches a small value but then does not improve for a long time, you could set the `relative_gap_limit` parameter to allow to stop the search as soon as a specific solution quality is reached."
+                            "This plot shows you how the gap between the objective and the bound changes over time. If it quickly reaches a small value but then does not improve for a long time, you could set the `relative_gap_limit` parameter to allow to stop the search as soon as a specific solution quality is reached.\n\n The Final Gap is comparing the objective to the final bound, only known at the end. If it falls significantly faster than the Relative Gap, you could think about stopping the search earlier via a time limit, as the solution quality seems to improve quickly but proving the quality is slow."
                         )
                     fig_2 = block.model_changes_as_plotly()
                     if fig_2:

cpsat_log_parser/blocks/__init__.py

@@ -2,14 +2,14 @@
 This folder contains the logic for parsing individual blocks from the log.
 """
 
+
 from .search_progress import SearchProgressBlock
 from .search_stats import SearchStatsBlock
 from .lns_stats import LnsStatsBlock
 from .solution_repositories import SolutionRepositoriesBlock
 from .solutions import SolutionsBlock
 from .objective_bounds import ObjectiveBoundsBlock
 from .log_block import LogBlock
-from .tables import TableBlock
 from .solver import SolverBlock
 from .solver_response import ResponseBlock
 from .presolve_log import PresolveLogBlock
@@ -32,7 +32,7 @@
 # The first matching block is used.
 ALL_BLOCKS = [
     SearchProgressBlock,
-        SequentialSearchProgressBlock,
+    SequentialSearchProgressBlock,
     SearchStatsBlock,
     LnsStatsBlock,
     SolutionRepositoriesBlock,
@@ -54,8 +54,33 @@
     TaskTimingBlock,
     PreloadingModelBlock,
     PresolveSummaryBlock,
-    #TableBlock,  # Seems to be problematic
+    # TableBlock,  # Seems to be problematic
     LogBlock,
 ]
 
-__all__ = list(ALL_BLOCKS)
+__all__ = [
+    "SearchProgressBlock",
+    "SearchStatsBlock",
+    "LnsStatsBlock",
+    "SolutionRepositoriesBlock",
+    "SolutionsBlock",
+    "ObjectiveBoundsBlock",
+    "SolverBlock",
+    "ResponseBlock",
+    "PresolveLogBlock",
+    "InitialModelBlock",
+    "PresolvedModelBlock",
+    "LpStatsBlock",
+    "LpDebugBlock",
+    "LpDimensionBlock",
+    "LpPoolBlock",
+    "LpCutBlock",
+    "LsStatsBlock",
+    "ImprovingBoundsSharedBlock",
+    "ClausesSharedBlock",
+    "TaskTimingBlock",
+    "PreloadingModelBlock",
+    "PresolveSummaryBlock",
+    "SequentialSearchProgressBlock",
+    "LogBlock",
+]

cpsat_log_parser/blocks/presolved_model.py

@@ -39,14 +39,11 @@ def get_num_variables(self) -> int:
         )
 
     def get_num_constraints(self) -> int:
-        # "#kNoOverlap2D: 1 (#rectangles: 24)"
-        # "#kInterval: 48"
-        n = 0
-        for line in self.lines:
-            if line.startswith("#k"):
-
-                n += int(line.split(":")[1].strip().split(" ")[0].replace("'", ""))
-        return n
+        return sum(
+            int(line.split(":")[1].strip().split(" ")[0].replace("'", ""))
+            for line in self.lines
+            if line.startswith("#k")
+        )
 
     def get_help(self) -> typing.Optional[str]:
         return """
@@ -55,6 +52,8 @@ def get_help(self) -> typing.Optional[str]:
 
         `- 200 in [0,199]` will indicate that there are 200 variables with domain `[0,199]`, i.e., values between 0 and 199.
 
+        `- 6 in [0,1][34][67][100]` will indicate that there are 6 variables with domain `[0,1][34][67][100]`, i.e., values 0, 1, 34, 67, and 100.
+
         `#kLinearN: 3'000 (#terms: 980'948)` indicates that there are 3000 linear constraints with 980'948 coefficients.
 
         It is useful to compare this to the initial model, to see if your

cpsat_log_parser/blocks/search_progress.py

@@ -20,6 +20,7 @@
 import math
 import re
 import typing
+from typing import Optional
 import plotly.graph_objects as go
 from .log_block import LogBlock
 
@@ -43,17 +44,20 @@ def _get_bound(match: re.Match) -> float:
     Needs to differ between upper and lower bound.
     """
     if "next_lb" not in match.groupdict():
-        return float(match.group("obj"))
-    next_lb = match.group("next_lb")
-    next_ub = match.group("next_ub")
+        return float(match["obj"])
+    next_lb = match["next_lb"]
+    next_ub = match["next_ub"]
     if next_lb is None or next_ub is None:
-        return float(match.group("obj"))
+        return float(match["obj"])
     bound_lb, bound_ub = float(next_lb), float(next_ub)
-    obj = float(match.group("obj"))
-    if obj < bound_lb:
-        return bound_ub  # upper bound
-    else:
-        return bound_lb  # lower bound
+    obj = float(match["obj"])
+    return bound_ub if obj < bound_lb else bound_lb
+
+
+def calculate_gap(obj: Optional[float], bound: Optional[float]) -> Optional[float]:
+    if obj is None or bound is None:
+        return None
+    return 100 * (abs(obj - bound) / max(1, abs(obj)))
 
 
 class BoundEvent:
@@ -72,9 +76,7 @@ def is_lower_bound(self):
         return None if self.obj is None else self.bound < self.obj
 
     def get_gap(self):
-        if self.obj is None:
-            return None
-        return 100 * (abs(self.obj - self.bound) / max(1, abs(self.obj)))
+        return calculate_gap(self.obj, self.bound)
 
     @staticmethod
     def parse(line: str) -> typing.Optional["BoundEvent"]:
@@ -162,7 +164,7 @@ def parse(line: str) -> typing.Optional["ModelEvent"]:
 
 
 class SearchProgressBlock(LogBlock):
-    def __init__(self, lines: typing.List[str], check: bool=True) -> None:
+    def __init__(self, lines: typing.List[str], check: bool = True) -> None:
         lines = [line.strip() for line in lines if line.strip()]
         if not lines:
             raise ValueError("No lines to parse")
@@ -227,33 +229,43 @@ def get_help(self) -> typing.Optional[str]:
 
     def gap_as_plotly(self) -> typing.Optional[go.Figure]:
         gap_events = [
-            e
-            for e in self._parse_events()
-            if isinstance(e, BoundEvent) or isinstance(e, ObjEvent)
+            e for e in self._parse_events() if isinstance(e, (BoundEvent, ObjEvent))
         ]
 
         def is_valid_gap(gap):
-            if gap is None:
-                return False
-            if not math.isfinite(gap):
-                return False
-            return True
+            return False if gap is None else bool(math.isfinite(gap))
 
         gaps = [(e.time, e.get_gap()) for e in gap_events if is_valid_gap(e.get_gap())]
+
         fig = go.Figure()
         if not gap_events:
             return None
+
         # add gaps
         fig.add_trace(
             go.Scatter(
                 x=[t for t, _ in gaps],
                 y=[gap for _, gap in gaps],
                 mode="lines+markers",
                 line=dict(color="purple"),
-                name="Gap",
+                name="Relative Gap",
                 hovertext=[e.msg for e in gap_events],
             )
         )
+
+        last_bound = max(gap_events, key=lambda e: e.time).bound
+        final_gaps = [calculate_gap(e.obj, last_bound) for e in gap_events]
+        fig.add_trace(
+            go.Scatter(
+                x=[e.time for e in gap_events],
+                y=final_gaps,
+                mode="lines+markers",
+                line=dict(color="orange"),
+                name="Final Gap",
+                hovertext=[e.msg for e in gap_events],
+            )
+        )
+
         # make the x-axis start at 0
         fig.update_xaxes(range=[0, 1.01 * gaps[-1][0]])
         max_gap = max(gap for _, gap in gaps if gap is not None)
@@ -321,7 +333,7 @@ def as_plotly(self) -> typing.Optional[go.Figure]:
         fig = go.Figure()
         if not obj_events and not bound_events:
             return None
-        max_time = max([e.time for e in bound_events + obj_events])
+        max_time = max(e.time for e in bound_events + obj_events)
 
         # make sure that both bounds and objs have a value at max_time
         if obj_events and obj_events[-1].time < max_time:

cpsat_log_parser/blocks/sequential_search_progress.py

@@ -12,6 +12,7 @@
 import typing
 from .search_progress import SearchProgressBlock, parse_time
 
+
 class SequentialSearchProgressBlock(SearchProgressBlock):
     def __init__(self, lines: List[str]) -> None:
         super().__init__(lines, check=False)
@@ -26,11 +27,10 @@ def get_presolve_time(self) -> float:
         ):
             return parse_time(m["time"])
         raise ValueError(f"Could not parse presolve time from '{self.lines[0]}'")
-    
+
     def get_help(self) -> typing.Optional[str]:
         return """
         This block indicates the start of the search phase. It is only present in satisfaction models and unfortunately gives not much information.
 
         You will get significantly more information in optimization models, in which case, this sections will actually contain some plots.
         """
-

example_logs/99_01.txt

@@ -10,60 +10,60 @@ Initial satisfaction model '': (model_fingerprint: 0xbc34aec982cbd687)
   - 468 Booleans in [0,1]
   - 39 in [0,20]
   - 39 in [0,40]
-  - 14 constants in {2,3,4,5,6,7,8,9,10,11,12,14,15,16} 
+  - 14 constants in {2,3,4,5,6,7,8,9,10,11,12,14,15,16}
 #kExactlyOne: 154 (#literals: 462)
 #kInterval: 38 (#enforced: 38)
 #kLinear2: 40
 #kLinear3: 500 (#enforced: 500)
 #kNoOverlap2D: 4 (#rectangles: 19, #optional: 19)
 
 Starting presolve at 0.00s
-  8.55e-04s  0.00e+00d  [PresolveToFixPoint] #num_loops=1 
-  7.44e-06s  0.00e+00d  [ExtractEncodingFromLinear] #potential_supersets=154 
-  1.64e-03s  4.12e-04d  [Probe] #probed=944 #new_binary_clauses=525 
-  3.14e-06s  0.00e+00d  [MaxClique] 
-  3.74e-04s  0.00e+00d  [PresolveToFixPoint] 
-  8.94e-05s  0.00e+00d  [ProcessAtMostOneAndLinear] 
-  2.16e-04s  0.00e+00d  [DetectDuplicateConstraints] 
-  2.30e-06s  0.00e+00d  [DetectDominatedLinearConstraints] 
-  6.61e-05s  0.00e+00d  [DetectDifferentVariables] 
-  3.61e-05s  1.39e-06d  [ProcessSetPPC] #relevant_constraints=154 
-  2.34e-06s  0.00e+00d  [FindAlmostIdenticalLinearConstraints] 
-  4.61e-05s  5.62e-05d  [FindBigAtMostOneAndLinearOverlap] 
-  6.27e-05s  4.39e-05d  [FindBigVerticalLinearOverlap] 
-  2.18e-06s  0.00e+00d  [FindBigHorizontalLinearOverlap] 
-  3.89e-06s  0.00e+00d  [MergeClauses] 
-  3.59e-04s  0.00e+00d  [PresolveToFixPoint] 
-  3.50e-04s  0.00e+00d  [PresolveToFixPoint] 
-  1.55e-03s  4.18e-04d  [Probe] #probed=944 #new_binary_clauses=525 
-  1.34e-04s  0.00e+00d  [MaxClique] Merged 153(306 literals) into 153(459 literals) at_most_ones. 
-  3.68e-04s  0.00e+00d  [PresolveToFixPoint] 
-  1.02e-04s  0.00e+00d  [ProcessAtMostOneAndLinear] 
-  2.61e-04s  0.00e+00d  [DetectDuplicateConstraints] 
-  3.48e-06s  0.00e+00d  [DetectDominatedLinearConstraints] 
-  7.73e-05s  0.00e+00d  [DetectDifferentVariables] 
-  1.12e-04s  4.14e-06d  [ProcessSetPPC] #relevant_constraints=307 #num_inclusions=153 
-  3.88e-06s  0.00e+00d  [FindAlmostIdenticalLinearConstraints] 
-  5.22e-05s  5.62e-05d  [FindBigAtMostOneAndLinearOverlap] 
-  6.03e-05s  4.39e-05d  [FindBigVerticalLinearOverlap] 
-  4.47e-06s  0.00e+00d  [FindBigHorizontalLinearOverlap] 
-  4.68e-06s  0.00e+00d  [MergeClauses] 
-  3.63e-04s  0.00e+00d  [PresolveToFixPoint] 
-  3.39e-04s  0.00e+00d  [PresolveToFixPoint] 
-  1.55e-03s  4.18e-04d  [Probe] #probed=944 #new_binary_clauses=525 
-  1.33e-04s  0.00e+00d  [MaxClique] Merged 153(306 literals) into 153(459 literals) at_most_ones. 
-  3.48e-04s  0.00e+00d  [PresolveToFixPoint] 
-  1.01e-04s  0.00e+00d  [ProcessAtMostOneAndLinear] 
-  2.58e-04s  0.00e+00d  [DetectDuplicateConstraints] 
-  3.94e-06s  0.00e+00d  [DetectDominatedLinearConstraints] 
-  6.95e-05s  0.00e+00d  [DetectDifferentVariables] 
-  1.06e-04s  4.14e-06d  [ProcessSetPPC] #relevant_constraints=307 #num_inclusions=153 
-  6.18e-06s  0.00e+00d  [FindAlmostIdenticalLinearConstraints] 
-  5.50e-05s  5.62e-05d  [FindBigAtMostOneAndLinearOverlap] 
-  6.04e-05s  4.39e-05d  [FindBigVerticalLinearOverlap] 
-  4.62e-06s  0.00e+00d  [FindBigHorizontalLinearOverlap] 
-  7.36e-06s  0.00e+00d  [MergeClauses] 
-  3.65e-04s  0.00e+00d  [PresolveToFixPoint] 
+  8.55e-04s  0.00e+00d  [PresolveToFixPoint] #num_loops=1
+  7.44e-06s  0.00e+00d  [ExtractEncodingFromLinear] #potential_supersets=154
+  1.64e-03s  4.12e-04d  [Probe] #probed=944 #new_binary_clauses=525
+  3.14e-06s  0.00e+00d  [MaxClique]
+  3.74e-04s  0.00e+00d  [PresolveToFixPoint]
+  8.94e-05s  0.00e+00d  [ProcessAtMostOneAndLinear]
+  2.16e-04s  0.00e+00d  [DetectDuplicateConstraints]
+  2.30e-06s  0.00e+00d  [DetectDominatedLinearConstraints]
+  6.61e-05s  0.00e+00d  [DetectDifferentVariables]
+  3.61e-05s  1.39e-06d  [ProcessSetPPC] #relevant_constraints=154
+  2.34e-06s  0.00e+00d  [FindAlmostIdenticalLinearConstraints]
+  4.61e-05s  5.62e-05d  [FindBigAtMostOneAndLinearOverlap]
+  6.27e-05s  4.39e-05d  [FindBigVerticalLinearOverlap]
+  2.18e-06s  0.00e+00d  [FindBigHorizontalLinearOverlap]
+  3.89e-06s  0.00e+00d  [MergeClauses]
+  3.59e-04s  0.00e+00d  [PresolveToFixPoint]
+  3.50e-04s  0.00e+00d  [PresolveToFixPoint]
+  1.55e-03s  4.18e-04d  [Probe] #probed=944 #new_binary_clauses=525
+  1.34e-04s  0.00e+00d  [MaxClique] Merged 153(306 literals) into 153(459 literals) at_most_ones.
+  3.68e-04s  0.00e+00d  [PresolveToFixPoint]
+  1.02e-04s  0.00e+00d  [ProcessAtMostOneAndLinear]
+  2.61e-04s  0.00e+00d  [DetectDuplicateConstraints]
+  3.48e-06s  0.00e+00d  [DetectDominatedLinearConstraints]
+  7.73e-05s  0.00e+00d  [DetectDifferentVariables]
+  1.12e-04s  4.14e-06d  [ProcessSetPPC] #relevant_constraints=307 #num_inclusions=153
+  3.88e-06s  0.00e+00d  [FindAlmostIdenticalLinearConstraints]
+  5.22e-05s  5.62e-05d  [FindBigAtMostOneAndLinearOverlap]
+  6.03e-05s  4.39e-05d  [FindBigVerticalLinearOverlap]
+  4.47e-06s  0.00e+00d  [FindBigHorizontalLinearOverlap]
+  4.68e-06s  0.00e+00d  [MergeClauses]
+  3.63e-04s  0.00e+00d  [PresolveToFixPoint]
+  3.39e-04s  0.00e+00d  [PresolveToFixPoint]
+  1.55e-03s  4.18e-04d  [Probe] #probed=944 #new_binary_clauses=525
+  1.33e-04s  0.00e+00d  [MaxClique] Merged 153(306 literals) into 153(459 literals) at_most_ones.
+  3.48e-04s  0.00e+00d  [PresolveToFixPoint]
+  1.01e-04s  0.00e+00d  [ProcessAtMostOneAndLinear]
+  2.58e-04s  0.00e+00d  [DetectDuplicateConstraints]
+  3.94e-06s  0.00e+00d  [DetectDominatedLinearConstraints]
+  6.95e-05s  0.00e+00d  [DetectDifferentVariables]
+  1.06e-04s  4.14e-06d  [ProcessSetPPC] #relevant_constraints=307 #num_inclusions=153
+  6.18e-06s  0.00e+00d  [FindAlmostIdenticalLinearConstraints]
+  5.50e-05s  5.62e-05d  [FindBigAtMostOneAndLinearOverlap]
+  6.04e-05s  4.39e-05d  [FindBigVerticalLinearOverlap]
+  4.62e-06s  0.00e+00d  [FindBigHorizontalLinearOverlap]
+  7.36e-06s  0.00e+00d  [MergeClauses]
+  3.65e-04s  0.00e+00d  [PresolveToFixPoint]
 
 Presolve summary:
   - 0 affine relations were detected.