Refactor/streamline testplan report (#1163)

- remove unused entries from generated report
- merge utc_time & machine_time into timestamp field
- upgrade pnpm & node in ci; move pytest cfg to pyproject.toml
- use unix timestamps & store tz info in test-level report
- update frontend test w @testing-library/react
- delta encode level info of all dict/fix entries
- avoid match status expansion for dict/fix match entries
- omit fixed indices of tablelog entry
- add timezone field to TestReport, have default values to timezone fields

Author: zhenyu-ms

.github/workflows/test_pr.yml

@@ -76,15 +76,15 @@ jobs:
         with:
           python-version: 3.11
       - name: Set up Node
-        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
         with:
-          node-version: "16.x"
+          node-version: 20
       - name: Restore pip cache
         uses: ./.github/actions/pip-cache
       - name: Set up PNPM
         uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # v4.0.0
         with:
-          version: 8.10.4
+          version: 9
       - name: Setup
         run: |
           pip install -r requirements-build.txt -U
@@ -133,7 +133,7 @@ jobs:
       - name: Set up Kafka for tests
         if: ${{ matrix.os == 'ubuntu-22.04' }}
         run: |
-          wget https://archive.apache.org/dist/kafka/3.5.2/kafka_2.13-3.5.2.tgz -O kafka.tgz
+          wget https://dlcdn.apache.org/kafka/3.9.0/kafka_2.13-3.9.0.tgz -O kafka.tgz
           sudo mkdir /opt/kafka
           sudo chown -R $USER:$USER /opt/kafka
           tar zxf kafka.tgz -C /opt/kafka --strip-components 1

doc/newsfragments/2900_changed.streamline_json_report.rst

@@ -0,0 +1,16 @@
+Change Testplan exported JSON report structure to reduce report size.
+
+* Remove unused report entry fields.
+    * ``fix_spec_path``.
+    * ``status_override`` and ``status_reason`` in case they are empty.
+    * ``line_no``, ``code_context`` and ``file_path`` if ``--code`` is not enabled.
+    * ``env_status``, ``part``, ``strict_order`` and ``host`` depending on report category.
+* Remove unused assertion entry fields ``category`` and ``flag`` if they are ``DEFAULT``.
+* Merge assertion entry fields ``utc_time`` and ``machine_time`` into a unix timestamp field ``timestamp``, and store timezone info in parent Test-level report under key ``timezone``.
+* Replace ISO 8601 time string with unix timestamp in all ``timer`` fields, and add a ``timezone`` field to Testplan-level report as well.
+* Update data structure of several serialized assertion entries.
+    * Delta encode level info of ``flattened_dict`` fields of ``DictLog`` and ``FixLog`` entries.
+    * Delta encode level info of ``comparison`` fields of ``DictMatch`` and ``FixMatch`` entries.
+    * Delta encode level info of nested ``comparison`` fields of ``DictMatchAll`` and ``FixMatchAll`` entries, remove extra nesting of ``matches`` as well.
+    * Preserve abbreviations of match status of ``DictMatch``, ``FixMatch``, ``DictMatchAll`` and ``FixMatchAll`` entries, i.e. ``p`` instead of ``Passed``, ``f`` instead of ``Failed``, ``i`` instead of ``Ignored``.
+    * Remove ``indices`` field of ``TableLog`` entries.

pyproject.toml

@@ -34,6 +34,7 @@
         "pytest-mock",
         "psutil",
         "schema",
+        "tzlocal",
         "lxml",
         "reportlab",
         "marshmallow",
@@ -110,7 +111,6 @@
     ]
 
 [tool.releaseherald]
-
     news_fragments_directory = 'doc/newsfragments'
     unreleased = true
     news_file = 'doc/news_template.rst'
@@ -121,3 +121,16 @@
 [tool.releaseherald.filename_metadata_extractor]
     type="re"
     pattern='''^(((?P<id>\d+)_?)?((?P<type>changed|new|deprecated|removed))?\.)?.*$'''
+
+[tool.pytest.ini_options]
+    filterwarnings = [
+        "ignore::pytest.PytestWarning",
+        "ignore:.*flask_restx.*:DeprecationWarning",
+        # jsonschema warning from flask_restx
+        "ignore:.*jsonschema.*:DeprecationWarning",
+        "ignore:.*load_module.*:DeprecationWarning",
+        "ignore:.*LogMatcher.*:UserWarning",
+        # under most cases, included files are not hit
+        "ignore:No data was collected:coverage.exceptions.CoverageWarning",
+    ]
+    norecursedirs = "tests/helpers"

pytest.ini

@@ -48,6 +48,7 @@ def merge(self, reports: Iterable[TestReport]) -> TestReport:
                 description.append(f"{report.description}\n")
 
             # TODO: what to do with meta, tags_index
+            # TODO: timer & timezone
         result.description = "\n".join(description)
 
         return result

testplan/cli/merger/mergers.py

@@ -8,7 +8,7 @@
 from testplan.common.config import Config, Configurable
 from testplan.common.utils import strings
 from testplan.common.utils.comparison import is_regex
-from testplan.common.utils.timing import utcnow
+from testplan.common.utils.timing import now
 from testplan.report import TestReport
 
 
@@ -18,7 +18,7 @@ class ExporterResult:
     result: Dict = None
     traceback: str = None
     uid: str = strings.uuid4()
-    start_time: datetime = utcnow()
+    start_time: datetime = now()
     end_time: datetime = None
 
     @property
@@ -154,7 +154,7 @@ def run_exporter(
     except Exception:
         exp_result.traceback = traceback.format_exc()
     finally:
-        exp_result.end_time = utcnow()
+        exp_result.end_time = now()
         if not exp_result.success:
             exporter.logger.error(exp_result.traceback)
         if result:

testplan/common/exporters/__init__.py

@@ -246,6 +246,23 @@ class ReportCategories:
     # use for before/after_start/stop, setup, teardown, etc
     SYNTHESIZED = "synthesized"
 
+    @classmethod
+    def is_test_level(cls, cat):
+        return cat in (
+            cls.MULTITEST,
+            cls.TASK_RERUN,
+            cls.GTEST,
+            cls.CPPUNIT,
+            cls.BOOST_TEST,
+            cls.HOBBESTEST,
+            cls.PYTEST,
+            cls.PYUNIT,
+            cls.UNITTEST,
+            cls.QUNIT,
+            cls.JUNIT,
+            cls.ERROR,
+        )
+
 
 class Report:
     """
@@ -510,7 +527,6 @@ def __init__(self, name, **kwargs):
         super(BaseReportGroup, self).__init__(name=name, **kwargs)
 
         self._index: Dict = {}
-        self.host: Optional[str] = None
         self.children = []
 
         self.build_index()

testplan/common/report/base.py

@@ -28,9 +28,9 @@ def emit(self, record):
         if hasattr(record, "report_obj_id"):
             report = REPORT_MAP.get(record.report_obj_id)
             if report is not None:
-                created = datetime.datetime.utcfromtimestamp(
+                created = datetime.datetime.fromtimestamp(
                     record.created
-                ).replace(tzinfo=timezone.utc)
+                ).astimezone()
                 report.logs.append(
                     {
                         "message": self.format(record),

testplan/common/report/log.py

@@ -1,7 +1,9 @@
 """
   Base schemas for report serialization.
 """
-from marshmallow import Schema, fields, post_load
+import datetime
+
+from marshmallow import Schema, fields, post_dump, post_load, pre_load
 from marshmallow.utils import EXCLUDE
 
 from testplan.common.report.base import (
@@ -22,8 +24,24 @@
 class IntervalSchema(Schema):
     """Schema for ``timer.Interval``"""
 
-    start = custom_fields.UTCDateTime()
-    end = custom_fields.UTCDateTime(allow_none=True)
+    start = fields.DateTime("timestamp")
+    end = fields.DateTime("timestamp", allow_none=True)
+
+    @pre_load
+    def accept_old_isoformat(self, data, **kwargs):
+        try:
+            if data.get("start", None) and isinstance(data["start"], str):
+                data["start"] = datetime.datetime.fromisoformat(
+                    data["start"]
+                ).timestamp()
+            if data.get("end", None) and isinstance(data["end"], str):
+                data["end"] = datetime.datetime.fromisoformat(
+                    data["end"]
+                ).timestamp()
+            return data
+        except ValueError as e:
+            # no need to defer
+            raise ValueError("Invalid value when loading Interval.") from e
 
     @post_load
     def make_interval(self, data, **kwargs):
@@ -68,7 +86,7 @@ class ReportLogSchema(Schema):
     message = fields.String()
     levelname = fields.String()
     levelno = fields.Integer()
-    created = custom_fields.UTCDateTime()
+    created = fields.DateTime("timestamp")
     funcName = fields.String()
     lineno = fields.Integer()
     uid = fields.UUID()
@@ -126,6 +144,14 @@ def make_report(self, data, **kwargs):
         rep.timer = timer
         return rep
 
+    @post_dump
+    def strip_none(self, data, **kwargs):
+        if data["status_override"] is None:
+            del data["status_override"]
+        if data["status_reason"] is None:
+            del data["status_reason"]
+        return data
+
 
 class BaseReportGroupSchema(ReportSchema):
     """Schema for ``base.BaseReportGroup``."""

testplan/common/report/schemas.py

@@ -1,10 +1,9 @@
 """Conversion utilities."""
 import itertools
-from typing import Union, Tuple, Iterable, Callable, List, Sequence
+from typing import Callable, Iterable, List, Optional, Sequence, Tuple, Union
 
 from .reporting import Absent
 
-
 RecursiveListTuple = List[Union[Tuple, Tuple["RecursiveListTuple"]]]
 
 
@@ -77,27 +76,11 @@ def make_iterables(values: Iterable) -> List[Union[List, Tuple]]:
     return iterables
 
 
-def full_status(status: str) -> str:
-    """
-    Human readable status label.
-
-    :param status: status label
-    :return: human-readable status label
-    """
-    if status == "p":
-        return "Passed"
-    elif status == "f":
-        return "Failed"
-    elif status == "i":
-        return "Ignored"
-    return ""
-
-
 def expand_values(
     rows: List[Tuple],
     level: int = 0,
     ignore_key: bool = False,
-    key_path: List = None,
+    key_path: Optional[List] = None,
     match: str = "",
 ):
     """
@@ -145,24 +128,6 @@ def expand_values(
             key_path.pop()
 
 
-# TODO: position parameter is misleading and it allows extracting
-#             the key or match information as value
-#             "left" or "right" choices would be enough for clarity and
-#             would fail earlier upon any change to structure
-def extract_values(comparison: List[Tuple], position: int) -> List:
-    """
-    Extracts one-side of a comparison result based on value position.
-
-    :param comparison: list of key, match, and value pair quadruples
-    :param position: index pointing to particular value
-    :return: list of key, match, and value triples
-    """
-    result = []
-    for item in comparison:
-        result.append((item[0], item[1], item[position]))
-    return result
-
-
 def flatten_formatted_object(formatted_obj):
     """
     Flatten the formatted object which is the result of function
@@ -232,10 +197,10 @@ def flatten_dict_comparison(comparison: List[Tuple]) -> List[List]:
     :param comparison: list of comparison results
     :return: result table to be used in display
     """
-    result_table = []  # level, key, left, right, result
+    result_table = []  # level, key, result, left, right
 
-    left = list(expand_values(extract_values(comparison, 2)))
-    right = list(expand_values(extract_values(comparison, 3)))
+    left = list(expand_values(map(lambda x: (x[0], x[1], x[2]), comparison)))
+    right = list(expand_values(map(lambda x: (x[0], x[1], x[3]), comparison)))
 
     while left or right:
         lpart, rpart = None, None
@@ -271,7 +236,7 @@ def flatten_dict_comparison(comparison: List[Tuple]) -> List[List]:
             level -= 1
             # key = '(group)'
 
-        status = full_status(lpart[3] if lpart else rpart[3])
+        status = lpart[3] if lpart else rpart[3]
         lval = lpart[4] if lpart else None
         rval = rpart[4] if rpart else None
         result_table.append(
@@ -295,3 +260,31 @@ def flatten_dict_comparison(comparison: List[Tuple]) -> List[List]:
             break
 
     return result_table
+
+
+def delta_encode_level(homo):
+    prev = 0
+    hetero = []
+    for r in homo:
+        level = r[0]
+        res = r[1:]
+        diff = level - prev
+        if diff != 0:
+            hetero.append(diff)
+        prev = level
+        hetero.append(res)
+
+    return hetero
+
+
+def delta_decode_level(hetero):
+    level = 0
+    homo = []
+    for r in hetero:
+        if isinstance(r, int):
+            level += r
+            continue
+        else:
+            homo.append([level, *r])
+
+    return homo

testplan/common/utils/convert.py

@@ -78,8 +78,6 @@ def render(obj, key=None):
             ret = (0, None, str(obj))
         elif obj is None:
             ret = (0, None, None)
-        elif issubclass(obj_t, (int,)):
-            ret = (0, obj_t.__name__, str(obj))
         elif issubclass(obj_t, NATIVE_TYPES):
             ret = (0, obj_t.__name__, obj)
         elif isinstance(obj, ContextValue):