diff --git a/src/datasets/packaged_modules/json/json.py b/src/datasets/packaged_modules/json/json.py index 9845dd676d4..b49aecf197f 100644 --- a/src/datasets/packaged_modules/json/json.py +++ b/src/datasets/packaged_modules/json/json.py @@ -181,9 +181,9 @@ def _generate_tables(self, base_files, files_iterables, original_files, allow_fu for i, training_example in enumerate(training_examples): if training_example["metadata"]["trace_type"] == "hermes": timestamp = ujson_loads(lines[i])["started_at"] - milliseconds = timestamp if timestamp <= 10_000_000_000 else timestamp * 1_000 + seconds = timestamp if timestamp <= 10_000_000_000 else timestamp / 1_000 sent_at = ( - datetime.fromtimestamp(milliseconds, tz=timezone.utc) + datetime.fromtimestamp(seconds, tz=timezone.utc) .isoformat(timespec="milliseconds") .replace("+00:00", "Z") ) diff --git a/tests/packaged_modules/test_json.py b/tests/packaged_modules/test_json.py index 2d6fd5b3849..556c192817d 100644 --- a/tests/packaged_modules/test_json.py +++ b/tests/packaged_modules/test_json.py @@ -447,6 +447,10 @@ def assert_agent_traces_output(tmp_path, filename, rows, expected, num_sessions= ], } +# Same session, but `started_at` expressed in milliseconds (e.g. JS `Date.now()`-style) +# instead of seconds -- the same real-world instant as HERMES_SESSION. +HERMES_SESSION_MS = {**HERMES_SESSION, "started_at": 1_780_665_768_307} + DROID_SESSION = [ { @@ -721,6 +725,14 @@ def test_json_generate_tables_with_sorted_columns(file_fixture, config_kwargs, r [HERMES_SESSION] * 2, ("hermes", "20260605_092247_d018ec", "Run pwd and date.", "2026-06-05T13:22:48.307Z", 1, 1), ), + pytest.param( + "hermes_ms.jsonl", + [HERMES_SESSION_MS], + # Same instant as the seconds-form "hermes" case above -- `started_at` is + # just expressed in milliseconds here. + ("hermes", "20260605_092247_d018ec", "Run pwd and date.", "2026-06-05T13:22:48.307Z", 1, 1), + id="hermes-ms", + ), pytest.param( "droid.jsonl", DROID_SESSION,