From 8b2af513fa5476ec625b8d33388e1b3fbb1180bd Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Wed, 2 Sep 2026 00:57:07 +0200 Subject: [PATCH 01/18] test(e2e): gate product models on output vs cached HF references Compare compiled Spyre output for the product decoder and embedding models against cached CPU HF references, under a new model_quality marker with its own CI job. gemma-4-31B is gated on short prompts only: it drifts from HF as the prompt grows because torch-spyre normalises in fp16. Signed-off-by: Yannick Schnider --- .../cache_config/hf_models_and_datasets.yaml | 2 + .github/workflows/_test_matrix.yaml | 10 + .pre-commit-config.yaml | 3 + Makefile | 14 +- docs/contributing/README.md | 25 + pyproject.toml | 1 + tests/data/decoder_output_refs.json | 542 ++++++++++++++++++ tests/data/generate_decoder_output_refs.py | 131 +++++ tests/e2e/test_encoder_models.py | 14 +- tests/e2e/test_model_quality.py | 121 ++++ 10 files changed, 857 insertions(+), 6 deletions(-) create mode 100644 tests/data/decoder_output_refs.json create mode 100644 tests/data/generate_decoder_output_refs.py create mode 100644 tests/e2e/test_model_quality.py diff --git a/.github/cache_config/hf_models_and_datasets.yaml b/.github/cache_config/hf_models_and_datasets.yaml index e71827768..57afbfa6a 100644 --- a/.github/cache_config/hf_models_and_datasets.yaml +++ b/.github/cache_config/hf_models_and_datasets.yaml @@ -22,5 +22,7 @@ public_models: - BAAI/bge-reranker-v2-m3 - ibm-ai-platform/micro-g3.3-8b-instruct-1b - ibm-granite/granite-4.1-3b + - ibm-granite/granite-3.3-8b-instruct + - ibm-granite/granite-4.1-8b - google/gemma-4-31B - qwrt/Swedish0.1M diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml index 21938bae6..75a06cb3b 100644 --- a/.github/workflows/_test_matrix.yaml +++ b/.github/workflows/_test_matrix.yaml @@ -201,6 +201,16 @@ jobs: - linux image_label: image_torch_spyre test_target: test-compile + - cfg: Spyre model quality tests + # Each case loads and compiles an 8B-31B model, too slow for the + # `integration` tier the multi-repo hook runs. + test_types: regression trunk + runs_on: + - x86_64 + - spyre_pf_x1 + - linux + image_label: image_torch_spyre + test_target: test-model-quality # Decoder attention, sharded across parallel jobs. The compiled # (STOCK on device) cases dominate runtime and grow HBM within a # process, so each shard is its own job on its own card: wall-clock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 71a7fcacd..c2a7b21dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,9 @@ repos: rev: v1.41.0 hooks: - id: typos + # Reference data holds tokenizer fragments ("other" from " G"+"other"), which + # --fix silently rewrites into real words. + exclude: '^tests/data/.*\.json$' - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.45.0 hooks: diff --git a/Makefile b/Makefile index a522adc96..1aaa70d25 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ else ifeq ($(TEST_TYPE),integration) # Single-invocation integration = the union of the CI smoke + compile jobs, so # `make tests TEST_TYPE=integration` locally keeps the compile coverage that CI # splits into its own job. (compile is not attention-marked, so it stays in.) -MARK_EXPR := -m "not (distributed or upstream or attention)" +MARK_EXPR := -m "not (distributed or upstream or attention or model_quality)" else ifeq ($(TEST_TYPE),unit) MARK_EXPR := -m "not upstream" else @@ -119,7 +119,7 @@ endif RESULTS_DIR ?= . .PHONY: help test tests run-one aiu-setup perf-tests coverage print-test-type \ - test-smoke test-attention test-attention-shard test-distributed \ + test-smoke test-compile test-model-quality test-attention test-attention-shard test-distributed \ test-upstream test-upstream-distributed test-upstream-model \ tests-single-card tests-multi-card @@ -171,12 +171,15 @@ run-one: ## Internal: one pytest invocation for the resolved MARK_EXPR/JUNIT_ARG echo "Running tests for TEST_TYPE=$(TEST_TYPE) MARK_OVERRIDE=$(MARK_OVERRIDE)..."; \ $(COVERAGE_ENV) uv run --active --no-sync pytest $(PYTEST_ARGS) $(MARK_EXPR) $(UPSTREAM_ARG) $(JUNIT_ARGS) -test-smoke: ## Run the smoke marker combo (non-distributed, non-upstream, non-attention, non-compile). - $(MAKE) run-one MARK_OVERRIDE='not (distributed or upstream or attention or compile)' JUNIT_XML=$(JUNIT_XML) +test-smoke: ## Run the smoke marker combo (non-distributed, non-upstream, non-attention, non-compile, non-model-quality). + $(MAKE) run-one MARK_OVERRIDE='not (distributed or upstream or attention or compile or model_quality)' JUNIT_XML=$(JUNIT_XML) test-compile: ## Run the torch.compile marker combo (its own job; slow). $(MAKE) run-one MARK_OVERRIDE='compile and not (distributed or upstream)' JUNIT_XML=$(JUNIT_XML) +test-model-quality: ## Run the product-model output-quality gates (its own job; loads 8B-31B models). + $(MAKE) run-one MARK_OVERRIDE='model_quality and not (distributed or upstream)' JUNIT_XML=$(JUNIT_XML) + test-attention: ## Run the decoder-attention marker combo (attention minus the encoder split), one process. $(MAKE) run-one MARK_OVERRIDE='attention and not encoder_attention and not (distributed or upstream)' JUNIT_XML=$(JUNIT_XML) @@ -217,11 +220,12 @@ test-upstream-model: ## Run the upstream+model (non-distributed) marker combo. # Single-card / multi-card split, grouping the 6 marker combos above by how many cards they need. # Each suite gets its own junit-/junit-.xml subdir, matching GHA's artifact-name/file-name layout (_test_matrix.yaml) so a Jenkins run's JUnit paths line up 1:1 with a GHA run's. -tests-single-card: ## Run the non-distributed marker combos (smoke/compile/attention/encoder-attention/upstream/upstream-model). Needs 1 card. +tests-single-card: ## Run the non-distributed marker combos (smoke/compile/model-quality/attention/encoder-attention/upstream/upstream-model). Needs 1 card. mkdir -p "$(RESULTS_DIR)"; \ rc=0; \ mkdir -p "$(RESULTS_DIR)/junit-test-smoke" && $(MAKE) test-smoke JUNIT_XML="$(RESULTS_DIR)/junit-test-smoke/junit-test-smoke.xml" || rc=1; \ mkdir -p "$(RESULTS_DIR)/junit-test-compile" && $(MAKE) test-compile JUNIT_XML="$(RESULTS_DIR)/junit-test-compile/junit-test-compile.xml" || rc=1; \ + mkdir -p "$(RESULTS_DIR)/junit-test-model-quality" && $(MAKE) test-model-quality JUNIT_XML="$(RESULTS_DIR)/junit-test-model-quality/junit-test-model-quality.xml" || rc=1; \ for i in $$(seq 0 $$(( $(ATTN_SHARDS) - 1 ))); do \ mkdir -p "$(RESULTS_DIR)/junit-test-attention-shard-$$i" && $(MAKE) test-attention-shard ATTN_SHARD_ID=$$i JUNIT_XML="$(RESULTS_DIR)/junit-test-attention-shard-$$i/junit-test-attention-shard-$$i.xml" || rc=1; \ done; \ diff --git a/docs/contributing/README.md b/docs/contributing/README.md index c1263283f..de33cfe19 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -82,6 +82,31 @@ pytest -m "attention and upstream" pytest --upstream -m "attention" ``` +#### Model Output Quality Gate + +The `model_quality` marker gates the product models on their output: each is loaded +**compiled** (the platform default) and compared against a CPU HF reference ā€” greedy +token ids plus per-token probabilities for the decoders +(`tests/e2e/test_model_quality.py`), cosine similarity for the embedding models +(`tests/e2e/test_encoder_models.py`). + +```bash +make test-model-quality # the whole gate, one card +``` + +The models are too large to run through transformers in CI, so the references live in +`tests/data/` and are regenerated by hand where the weights are cached: + +```bash +python tests/data/generate_decoder_output_refs.py --models ibm-granite/granite-4.1-8b +python tests/data/generate_encoder_embed_refs.py +``` + +Regenerate only when the *expected* output changes (new model or prompt), never to make a +failing test pass ā€” that is the regression the gate exists to catch. Prompts are per +model: `MODEL_PROMPTS` in the generator says which models are restricted and why. +`SPYRE_TEST_ABS_TOL` (default `0.08`) sets the decoder probability tolerance. + #### Upstream Test Integration Upstream tests are cloned from the vLLM repository at the commit pinned in `pyproject.toml`, fetching only the `tests/` directory. The clone happens on demand, the first time a run asks for upstream tests (see the marker gate above). Cloned tests are cached in `~/.cache/vllm-upstream-tests` (or `$XDG_CACHE_HOME/vllm-upstream-tests`) with separate worktrees per commit, allowing multiple vLLM versions to be tested simultaneously. All upstream tests run with `VLLM_PLUGINS=spyre_inference,spyre_inference_ops` set automatically. Pointing the plugin at a vLLM checkout instead of the cache is the one case that still needs the flag by hand: `pytest -p spyre_testing_plugin.pytest_plugin -m upstream` from the checkout root. See `tests/plugin/spyre_testing_plugin/pytest_plugin.py` for implementation details. diff --git a/pyproject.toml b/pyproject.toml index e02ff2fde..2fb238ca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -264,6 +264,7 @@ markers = [ "attention: SpyreAttentionImpl tests (tests/attention/test_spyre_attn.py, tests/attention/test_spyre_encoder_attn.py); slow, runs in its own CI job", "encoder_attention: encoder-attention subset (tests/attention/test_spyre_encoder_attn.py); also marked attention, split into its own CI job", "compile: torch.compile end-to-end tests (tests/e2e/test_compile.py); slow, runs in its own CI job", + "model_quality: product-model output gates vs cached HF references (tests/e2e/test_model_quality.py and test_encoder_models.py::test_encoder_embed_models_compiled); loads 8B-31B models, runs in its own CI job", "rotary: Spyre RoPE custom-op tests (tests/custom_ops/test_rotary_embedding.py)", "fp8: FP8 quantization tests (tests/custom_ops/test_fp8.py)", ] diff --git a/tests/data/decoder_output_refs.json b/tests/data/decoder_output_refs.json new file mode 100644 index 000000000..65ddaeec7 --- /dev/null +++ b/tests/data/decoder_output_refs.json @@ -0,0 +1,542 @@ +{ + "google/gemma-4-31B": { + "dtype": "float32", + "max_tokens": 16, + "results": [ + { + "logprobs": [ + -1.186606, + -1.509682, + -1.086679, + -1.072163, + -2.122931, + -2.515233, + -0.972521, + -1.722148, + -0.336788, + -0.343377, + -1.098584, + -0.801154, + -0.545086, + -0.649821, + -0.777465, + -0.235408 + ], + "prompt": "What are IBMs main businesses?", + "text": "\n\nWhat are the main businesses of IBM?\n\nWhat are the main businesses of", + "token_ids": [ + 108, + 3689, + 659, + 506, + 1689, + 8634, + 529, + 31209, + 236881, + 108, + 3689, + 659, + 506, + 1689, + 8634, + 529 + ], + "tokens": [ + "\n\n", + "What", + " are", + " the", + " main", + " businesses", + " of", + " IBM", + "?", + "\n\n", + "What", + " are", + " the", + " main", + " businesses", + " of" + ] + }, + { + "logprobs": [ + -1.829299, + -1.579614, + -1.214549, + -2.250874, + -0.600629, + -2.059917, + -0.471437, + -0.932735, + -1.361293, + -0.381795, + -1.460487, + -0.791934, + -0.959909, + -0.507025, + -0.303709, + -0.436555 + ], + "prompt": "The capital of France is", + "text": " a city of romance, art, and culture. It is also a city of", + "token_ids": [ + 496, + 3207, + 529, + 30875, + 236764, + 1610, + 236764, + 532, + 6540, + 236761, + 1030, + 563, + 992, + 496, + 3207, + 529 + ], + "tokens": [ + " a", + " city", + " of", + " romance", + ",", + " art", + ",", + " and", + " culture", + ".", + " It", + " is", + " also", + " a", + " city", + " of" + ] + }, + { + "logprobs": [ + -0.236248, + -0.423639, + -0.031093, + -1.6e-05, + -0.70528, + -0.530978, + -0.247519, + -1.565231, + -0.161889, + -0.029527, + -0.112657, + -0.049126, + -0.016049, + -0.003028, + -0.043576, + -0.004413 + ], + "prompt": "Q: What is the largest planet in our solar system?\nA:", + "text": " Jupiter\n\nQ: What is the second largest planet in our solar system?\n", + "token_ids": [ + 52895, + 108, + 236935, + 236787, + 2900, + 563, + 506, + 1855, + 7488, + 13401, + 528, + 1023, + 10321, + 1458, + 236881, + 107 + ], + "tokens": [ + " Jupiter", + "\n\n", + "Q", + ":", + " What", + " is", + " the", + " second", + " largest", + " planet", + " in", + " our", + " solar", + " system", + "?", + "\n" + ] + } + ] + }, + "ibm-granite/granite-3.3-8b-instruct": { + "dtype": "float32", + "max_tokens": 16, + "results": [ + { + "logprobs": [ + -0.000607, + -0.749758, + -0.106717, + -3e-05, + -0.088363, + -0.000342, + -0.696137, + -0.000183, + -0.585119, + -0.475057, + -0.21888, + -0.00091, + -3e-06, + -0.41757, + -0.003762, + -0.305773 + ], + "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup.\n\n### Response:", + "text": "\n\n1. Gather ingredients: 1 whole chicken, 2", + "token_ids": [ + 203, + 203, + 35, + 32, + 570, + 3790, + 42568, + 44, + 225, + 35, + 9844, + 663, + 21217, + 30, + 225, + 36 + ], + "tokens": [ + "\n", + "\n", + "1", + ".", + " G", + "ather", + " ingredients", + ":", + " ", + "1", + " whole", + " ch", + "icken", + ",", + " ", + "2" + ] + }, + { + "logprobs": [ + -0.011008, + -0.0722, + -0.248327, + -0.110021, + -0.000184, + -2.7e-05, + -5e-06, + -0.000195, + -0.061382, + -1e-06, + -0.213029, + -0.396524, + -2e-06, + -0.394157, + -0.950517, + -0.376066 + ], + "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nWhat are the main businesses of IBM?\n\n### Response:", + "text": "\nIBM, or International Business Machines Corporation, has diversified its business portfolio", + "token_ids": [ + 203, + 36671, + 30, + 556, + 21554, + 17265, + 488, + 11386, + 13180, + 30, + 1401, + 34142, + 1639, + 2819, + 12590, + 31397 + ], + "tokens": [ + "\n", + "IBM", + ",", + " or", + " International", + " Business", + " M", + "achines", + " Corporation", + ",", + " has", + " divers", + "ified", + " its", + " business", + " portfolio" + ] + }, + { + "logprobs": [ + -0.001327, + -0.615451, + -0.000136, + -0.000182, + -0.879656, + -0.529216, + -0.394932, + -0.133598, + -0.267801, + -0.004337, + -0.013194, + -4.6e-05, + -2e-06, + -0.1755, + -0.0, + -0.254457 + ], + "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nConvert char to string in Java.\n\n### Response:", + "text": "\nIn Java, a character is represented by the `char` data type,", + "token_ids": [ + 203, + 383, + 4949, + 30, + 312, + 5341, + 438, + 17718, + 810, + 322, + 529, + 1594, + 82, + 706, + 827, + 30 + ], + "tokens": [ + "\n", + "In", + " Java", + ",", + " a", + " character", + " is", + " represented", + " by", + " the", + " `", + "char", + "`", + " data", + " type", + "," + ] + } + ] + }, + "ibm-granite/granite-4.1-8b": { + "dtype": "float32", + "max_tokens": 16, + "results": [ + { + "logprobs": [ + -0.302353, + -0.299105, + -6.4e-05, + -0.083195, + -0.014468, + -9e-06, + -0.006912, + -0.190234, + -0.263843, + -0.093166, + -0.036535, + -0.008035, + -0.346234, + -3e-06, + -0.001928, + -0.717883 + ], + "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup.\n\n### Response:", + "text": " \n1. **Gather Ingredients**: Collect all necessary ingredients such as chicken breasts", + "token_ids": [ + 720, + 16, + 13, + 3146, + 38, + 1894, + 52275, + 96618, + 21153, + 682, + 5995, + 14293, + 1778, + 439, + 16553, + 37449 + ], + "tokens": [ + " \n", + "1", + ".", + " **", + "G", + "ather", + " Ingredients", + "**:", + " Collect", + " all", + " necessary", + " ingredients", + " such", + " as", + " chicken", + " breasts" + ] + }, + { + "logprobs": [ + -0.701019, + -0.014904, + -0.251564, + -0.000126, + -0.0, + -1e-06, + -0.002019, + -0.002093, + -0.062779, + -0.073797, + -0.088115, + -0.000268, + -0.009194, + -0.085598, + -0.057024, + -0.298725 + ], + "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nWhat are the main businesses of IBM?\n\n### Response:", + "text": " \n\nIBM (International Business Machines Corporation) is a multinational technology company with a diverse", + "token_ids": [ + 4815, + 68838, + 320, + 34746, + 8184, + 45004, + 13332, + 8, + 374, + 264, + 69026, + 5557, + 2883, + 449, + 264, + 17226 + ], + "tokens": [ + " \n\n", + "IBM", + " (", + "International", + " Business", + " Machines", + " Corporation", + ")", + " is", + " a", + " multinational", + " technology", + " company", + " with", + " a", + " diverse" + ] + }, + { + "logprobs": [ + -0.083338, + -0.150565, + -1.2e-05, + -1e-05, + -1.02652, + -0.382205, + -6.1e-05, + -0.001031, + -2.5e-05, + -1e-06, + -1e-05, + -7.1e-05, + -0.006363, + -0.026268, + -0.281138, + -0.042108 + ], + "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nConvert char to string in Java.\n\n### Response:", + "text": " \n\nTo convert a character to a string in Java, you can use the `", + "token_ids": [ + 4815, + 1271, + 5625, + 264, + 3752, + 311, + 264, + 925, + 304, + 8102, + 11, + 499, + 649, + 1005, + 279, + 1595 + ], + "tokens": [ + " \n\n", + "To", + " convert", + " a", + " character", + " to", + " a", + " string", + " in", + " Java", + ",", + " you", + " can", + " use", + " the", + " `" + ] + } + ] + } +} diff --git a/tests/data/generate_decoder_output_refs.py b/tests/data/generate_decoder_output_refs.py new file mode 100644 index 000000000..27e5a68ae --- /dev/null +++ b/tests/data/generate_decoder_output_refs.py @@ -0,0 +1,131 @@ +# Copyright 2026 The Spyre-Inference Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Write CPU HF greedy references for tests/e2e/test_model_quality.py. + +The models are too large to run through transformers in CI, so the references are +generated here and checked in. Each run merges into the existing file. + + python tests/data/generate_decoder_output_refs.py + python tests/data/generate_decoder_output_refs.py --models ibm-granite/granite-4.1-8b +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +# Model ids must match tests/e2e/test_model_quality.py. +DECODER_MODELS = [ + "ibm-granite/granite-3.3-8b-instruct", + "ibm-granite/granite-4.1-8b", + "google/gemma-4-31B", +] + +# Must stay under MAX_NUM_BATCHED_TOKENS (test_model_quality.py) so each prefill lands +# in a single compiled bucket. +_TEMPLATE = ( + "Below is an instruction that describes a task. Write a response that " + "appropriately completes the request.\n\n### Instruction:\n{}\n\n### Response:" +) +PROMPTS = [ + _TEMPLATE.format("Provide a list of instructions for preparing chicken soup."), + _TEMPLATE.format("What are the main businesses of IBM?"), + _TEMPLATE.format("Convert char to string in Java."), +] + +# gemma-4 drifts from HF as the prompt grows, because torch-spyre runs RMSNorm in fp16: +# on the prompts above its first-token probability is 0.65 against HF's 0.84 and the +# continuation diverges, while short prompts match token for token. Neither the reference +# dtype (fp16 CPU HF agrees with fp32 to <0.002) nor torch.compile (eager deviates just +# as far) is involved, so drop this entry once torch-spyre normalises in fp32. +MODEL_PROMPTS = { + "google/gemma-4-31B": [ + "What are IBMs main businesses?", + "The capital of France is", + "Q: What is the largest planet in our solar system?\nA:", + ], +} + +MAX_TOKENS = 16 +_ROUND = 6 + +OUT_PATH = Path(__file__).parent / "decoder_output_refs.json" + + +def generate_reference(model_id: str, dtype: torch.dtype) -> dict[str, Any]: + tokenizer = AutoTokenizer.from_pretrained(model_id) + model = AutoModelForCausalLM.from_pretrained(model_id, dtype=dtype) + model.eval() + # The test runs with ignore_eos=True, so the reference needs all MAX_TOKENS steps. + model.generation_config.eos_token_id = None + + results = [] + for prompt in MODEL_PROMPTS.get(model_id, PROMPTS): + input_ids = tokenizer(prompt, return_tensors="pt").input_ids + with torch.inference_mode(): + output = model.generate( + input_ids, + do_sample=False, + max_new_tokens=MAX_TOKENS, + return_dict_in_generate=True, + output_scores=True, + ) + # normalize_logits gives logprobs over the vocabulary, matching what vLLM reports. + logprobs = model.compute_transition_scores( + output.sequences, output.scores, normalize_logits=True + )[0] + new_token_ids = output.sequences[0, input_ids.shape[1] :] + + results.append( + { + "prompt": prompt, + "text": tokenizer.decode(new_token_ids), + "token_ids": [int(t) for t in new_token_ids], + "tokens": [tokenizer.decode(t) for t in new_token_ids], + "logprobs": [round(float(lp), _ROUND) for lp in logprobs], + } + ) + print(f" {prompt!r}\n -> {results[-1]['text']!r}", flush=True) + + return { + "max_tokens": MAX_TOKENS, + "dtype": str(dtype).removeprefix("torch."), + "results": results, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--models", nargs="+", default=DECODER_MODELS) + parser.add_argument("--dtype", default="float32", choices=["float32", "bfloat16", "float16"]) + parser.add_argument("--out", type=Path, default=OUT_PATH) + args = parser.parse_args() + + data = json.loads(args.out.read_text()) if args.out.exists() else {} + for model_id in args.models: + print(f"Generating {model_id} ...", flush=True) + data[model_id] = generate_reference(model_id, getattr(torch, args.dtype)) + # Written per model: each one takes minutes and is easy to interrupt. + args.out.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") + print(f"Wrote {args.out}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/test_encoder_models.py b/tests/e2e/test_encoder_models.py index ed3d807fc..95aef4bab 100644 --- a/tests/e2e/test_encoder_models.py +++ b/tests/e2e/test_encoder_models.py @@ -91,6 +91,18 @@ def _hf_last_token_embeddings(model: str, prompts: list[str]) -> list[list[float @pytest.mark.parametrize("model", EMBEDDING_MODELS) def test_encoder_embed_models(model: str) -> None: """Spyre embeddings match cached HF references within cosine tolerance.""" + _assert_embeddings_match_refs(model, enforce_eager=True) + + +@pytest.mark.model_quality +@pytest.mark.uses_subprocess +@pytest.mark.parametrize("model", EMBEDDING_MODELS) +def test_encoder_embed_models_compiled(model: str) -> None: + """Same models and references, compiled rather than eager.""" + _assert_embeddings_match_refs(model, enforce_eager=False) + + +def _assert_embeddings_match_refs(model: str, enforce_eager: bool) -> None: ref = _REFERENCES.get(model) if ref is None: pytest.skip(f"No HF ref for {model}; run tests/data/generate_encoder_embed_refs.py") @@ -101,7 +113,7 @@ def test_encoder_embed_models(model: str) -> None: runner="pooling", max_model_len=64, max_num_seqs=1, - enforce_eager=True, + enforce_eager=enforce_eager, ) outputs = llm.embed(prompts) assert len(outputs) == len(prompts) diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py new file mode 100644 index 000000000..5b017586e --- /dev/null +++ b/tests/e2e/test_model_quality.py @@ -0,0 +1,121 @@ +# Copyright 2026 The Spyre-Inference Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Output-quality gate for the product decoder models: compiled Spyre output against a +cached CPU HF reference, comparing token ids and per-token probabilities. + +Prompts and references: ``python tests/data/generate_decoder_output_refs.py`` +""" + +from __future__ import annotations + +import json +import math +import os +from pathlib import Path +from typing import Any + +import pytest +from vllm import LLM, RequestOutput, SamplingParams + +pytestmark = [pytest.mark.model_quality, pytest.mark.uses_subprocess] + +DECODER_MODELS = [ + "ibm-granite/granite-3.3-8b-instruct", + "ibm-granite/granite-4.1-8b", + "google/gemma-4-31B", +] + +# fp16 on device reorders accumulation against the fp32 reference, so probabilities are +# compared with a tolerance. Same default as sendnn-inference's TEST_ABS_TOL. +ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.08")) + +MAX_MODEL_LEN = 256 +MAX_NUM_SEQS = 2 +# Caps the compiled buckets (platform.py) and so warmup; every prompt fits one bucket. +MAX_NUM_BATCHED_TOKENS = 64 + +_REF_PATH = Path(__file__).parent.parent / "data" / "decoder_output_refs.json" +_REFERENCES: dict = json.loads(_REF_PATH.read_text()) if _REF_PATH.exists() else {} + + +@pytest.mark.parametrize("model", DECODER_MODELS) +def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> None: + """Compiled Spyre output matches the cached HF reference for `model`.""" + ref = _REFERENCES.get(model) + assert ref is not None, ( + f"No HF reference for {model} in {_REF_PATH.name}; regenerate with " + f"`python tests/data/generate_decoder_output_refs.py --models {model}`" + ) + + monkeypatch.setenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "36000") + + prompts = [result["prompt"] for result in ref["results"]] + max_tokens = ref["max_tokens"] + + engine = LLM( + model=model, + enforce_eager=False, + max_model_len=MAX_MODEL_LEN, + max_num_seqs=MAX_NUM_SEQS, + max_num_batched_tokens=MAX_NUM_BATCHED_TOKENS, + ) + + outputs = engine.generate( + prompts, + SamplingParams( + temperature=0.0, + max_tokens=max_tokens, + logprobs=0, # logprob of the sampled token only + ignore_eos=True, # the reference is a fixed-length run with EOS disabled + ), + use_tqdm=False, + ) + + assert [output.prompt for output in outputs] == prompts, "Model output contained wrong prompt!" + for hf_result, output in zip(ref["results"], outputs): + _compare_against_hf(model, hf_result, output) + + +def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOutput) -> None: + completion = output.outputs[0] + token_ids = list(completion.token_ids) + logprobs = [completion.logprobs[i][t].logprob for i, t in enumerate(token_ids)] + + print(f"\n{model} prompt: {hf_result['prompt']!r}") + print(f" HF: {hf_result['text']!r}") + print(f" Spyre: {completion.text!r}") + + assert len(token_ids) == len(hf_result["token_ids"]), ( + f"{model}: generated {len(token_ids)} tokens, reference has {len(hf_result['token_ids'])}" + ) + + for step, (hf_id, hf_logprob, token_id, logprob) in enumerate( + zip(hf_result["token_ids"], hf_result["logprobs"], token_ids, logprobs) + ): + hf_prob, prob = math.exp(hf_logprob), math.exp(logprob) + probs_close = math.isclose(hf_prob, prob, abs_tol=ABS_TOL) + detail = ( + f"step {step}: token {token_id} ({completion.logprobs[step][token_id].decoded_token!r}," + f" p={prob:.4f}) vs HF {hf_id} ({hf_result['tokens'][step]!r}, p={hf_prob:.4f})" + ) + + if hf_id != token_id: + # Greedy paths only diverge legitimately on a near-tie, and past that point + # the prefixes differ, so no later token is comparable. + assert probs_close, f"{model}: wrong token, {detail}" + print(f" diverged on a near-tie at {detail}; not comparing further") + return + + assert probs_close, f"{model}: probability differs by more than {ABS_TOL}, {detail}" From b8a5c155cfd98934cb3e8595b0ba4c14fce883e0 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Thu, 3 Sep 2026 11:06:12 +0200 Subject: [PATCH 02/18] add revisions Signed-off-by: Yannick Schnider --- .../cache_config/hf_models_and_datasets.yaml | 9 ++++++--- .github/cache_config/manage_cache.py | 20 ++++++++++++++++--- tests/data/decoder_output_refs.json | 3 +++ tests/data/generate_decoder_output_refs.py | 17 ++++++++++++---- tests/e2e/test_model_quality.py | 3 +++ 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/.github/cache_config/hf_models_and_datasets.yaml b/.github/cache_config/hf_models_and_datasets.yaml index 57afbfa6a..fca0dd94d 100644 --- a/.github/cache_config/hf_models_and_datasets.yaml +++ b/.github/cache_config/hf_models_and_datasets.yaml @@ -22,7 +22,10 @@ public_models: - BAAI/bge-reranker-v2-m3 - ibm-ai-platform/micro-g3.3-8b-instruct-1b - ibm-granite/granite-4.1-3b - - ibm-granite/granite-3.3-8b-instruct - - ibm-granite/granite-4.1-8b - - google/gemma-4-31B + - repo: ibm-granite/granite-3.3-8b-instruct + revision: 51dd4bc2ade4059a6bd87649d68aa11e4fb2529b + - repo: ibm-granite/granite-4.1-8b + revision: 1504002f650e656a0a3789d99574df12e3e94ed0 + - repo: google/gemma-4-31B + revision: 5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89 - qwrt/Swedish0.1M diff --git a/.github/cache_config/manage_cache.py b/.github/cache_config/manage_cache.py index 77f94e8df..524fe3c3d 100644 --- a/.github/cache_config/manage_cache.py +++ b/.github/cache_config/manage_cache.py @@ -64,16 +64,30 @@ def _run_public(config, config_file): return print(f"šŸ“‹ Found {len(models)} public model(s) to cache:", models) failed_models = [] - for repo_id in models: + for entry in models: + if isinstance(entry, str): + repo_id, revision = entry, None + else: + repo_id, revision = entry["repo"], entry["revision"] print(f"\nšŸš€ Processing: {repo_id}...") try: - snapshot_download(repo_id, local_files_only=True, ignore_patterns=["*.pt", "*.bin"]) + snapshot_download( + repo_id, + revision=revision, + local_files_only=True, + ignore_patterns=["*.pt", "*.bin"], + ) print(f"āœ… {repo_id}: already cached") continue except LocalEntryNotFoundError: pass try: - snapshot_download(repo_id, local_files_only=False, ignore_patterns=["*.pt", "*.bin"]) + snapshot_download( + repo_id, + revision=revision, + local_files_only=False, + ignore_patterns=["*.pt", "*.bin"], + ) print(f"āœ… {repo_id}: downloaded and cached") except Exception as e: print(f"āš ļø Warning: failed to cache {repo_id}: {e}") diff --git a/tests/data/decoder_output_refs.json b/tests/data/decoder_output_refs.json index 65ddaeec7..e8d8b82c3 100644 --- a/tests/data/decoder_output_refs.json +++ b/tests/data/decoder_output_refs.json @@ -2,6 +2,7 @@ "google/gemma-4-31B": { "dtype": "float32", "max_tokens": 16, + "revision": "5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89", "results": [ { "logprobs": [ @@ -182,6 +183,7 @@ "ibm-granite/granite-3.3-8b-instruct": { "dtype": "float32", "max_tokens": 16, + "revision": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b", "results": [ { "logprobs": [ @@ -362,6 +364,7 @@ "ibm-granite/granite-4.1-8b": { "dtype": "float32", "max_tokens": 16, + "revision": "1504002f650e656a0a3789d99574df12e3e94ed0", "results": [ { "logprobs": [ diff --git a/tests/data/generate_decoder_output_refs.py b/tests/data/generate_decoder_output_refs.py index 27e5a68ae..ea0ea64f4 100644 --- a/tests/data/generate_decoder_output_refs.py +++ b/tests/data/generate_decoder_output_refs.py @@ -38,6 +38,12 @@ "google/gemma-4-31B", ] +MODEL_REVISIONS = { + "ibm-granite/granite-3.3-8b-instruct": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b", + "ibm-granite/granite-4.1-8b": "1504002f650e656a0a3789d99574df12e3e94ed0", + "google/gemma-4-31B": "5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89", +} + # Must stay under MAX_NUM_BATCHED_TOKENS (test_model_quality.py) so each prefill lands # in a single compiled bucket. _TEMPLATE = ( @@ -69,9 +75,9 @@ OUT_PATH = Path(__file__).parent / "decoder_output_refs.json" -def generate_reference(model_id: str, dtype: torch.dtype) -> dict[str, Any]: - tokenizer = AutoTokenizer.from_pretrained(model_id) - model = AutoModelForCausalLM.from_pretrained(model_id, dtype=dtype) +def generate_reference(model_id: str, revision: str, dtype: torch.dtype) -> dict[str, Any]: + tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision) + model = AutoModelForCausalLM.from_pretrained(model_id, revision=revision, dtype=dtype) model.eval() # The test runs with ignore_eos=True, so the reference needs all MAX_TOKENS steps. model.generation_config.eos_token_id = None @@ -105,6 +111,7 @@ def generate_reference(model_id: str, dtype: torch.dtype) -> dict[str, Any]: print(f" {prompt!r}\n -> {results[-1]['text']!r}", flush=True) return { + "revision": revision, "max_tokens": MAX_TOKENS, "dtype": str(dtype).removeprefix("torch."), "results": results, @@ -121,7 +128,9 @@ def main() -> None: data = json.loads(args.out.read_text()) if args.out.exists() else {} for model_id in args.models: print(f"Generating {model_id} ...", flush=True) - data[model_id] = generate_reference(model_id, getattr(torch, args.dtype)) + data[model_id] = generate_reference( + model_id, MODEL_REVISIONS[model_id], getattr(torch, args.dtype) + ) # Written per model: each one takes minutes and is easy to interrupt. args.out.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") print(f"Wrote {args.out}", flush=True) diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index 5b017586e..28c0e04fe 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -63,9 +63,12 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No prompts = [result["prompt"] for result in ref["results"]] max_tokens = ref["max_tokens"] + revision = ref["revision"] engine = LLM( model=model, + revision=revision, + tokenizer_revision=revision, enforce_eager=False, max_model_len=MAX_MODEL_LEN, max_num_seqs=MAX_NUM_SEQS, From d97fa5b6c835eb20ee6b33650f7f71ebf578d56a Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Fri, 4 Sep 2026 13:34:55 +0200 Subject: [PATCH 03/18] shave off some runtime by less compiled shapes Signed-off-by: Yannick Schnider --- tests/e2e/test_model_quality.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index 28c0e04fe..49b981c7e 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -42,9 +42,10 @@ ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.08")) MAX_MODEL_LEN = 256 -MAX_NUM_SEQS = 2 +MAX_NUM_SEQS = 3 # Caps the compiled buckets (platform.py) and so warmup; every prompt fits one bucket. MAX_NUM_BATCHED_TOKENS = 64 +COMPILE_SIZES = [MAX_NUM_SEQS, MAX_NUM_BATCHED_TOKENS] _REF_PATH = Path(__file__).parent.parent / "data" / "decoder_output_refs.json" _REFERENCES: dict = json.loads(_REF_PATH.read_text()) if _REF_PATH.exists() else {} @@ -73,6 +74,7 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No max_model_len=MAX_MODEL_LEN, max_num_seqs=MAX_NUM_SEQS, max_num_batched_tokens=MAX_NUM_BATCHED_TOKENS, + compilation_config={"compile_sizes": COMPILE_SIZES}, ) outputs = engine.generate( From 282c9dbb8557914d5a0bee0766fb505257974092 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Fri, 4 Sep 2026 14:56:04 +0200 Subject: [PATCH 04/18] adding llama model Signed-off-by: Yannick Schnider --- .../cache_config/hf_models_and_datasets.yaml | 2 + .github/cache_config/manage_cache.py | 7 +- tests/data/decoder_output_refs.json | 193 +++++++++++++++++- tests/data/generate_decoder_output_refs.py | 2 + tests/e2e/test_model_quality.py | 1 + 5 files changed, 198 insertions(+), 7 deletions(-) diff --git a/.github/cache_config/hf_models_and_datasets.yaml b/.github/cache_config/hf_models_and_datasets.yaml index fca0dd94d..8967de48d 100644 --- a/.github/cache_config/hf_models_and_datasets.yaml +++ b/.github/cache_config/hf_models_and_datasets.yaml @@ -6,6 +6,8 @@ gated_models: - meta-llama/Meta-Llama-3-8B + - repo: meta-llama/Llama-3.1-8B-Instruct + revision: 0e9e39f249a16976918f6564b8830bc894c89659 - meta-llama/Llama-3.2-1B-Instruct - google/gemma-3-1b-it - google/gemma-1.1-2b-it diff --git a/.github/cache_config/manage_cache.py b/.github/cache_config/manage_cache.py index 524fe3c3d..d07eddd41 100644 --- a/.github/cache_config/manage_cache.py +++ b/.github/cache_config/manage_cache.py @@ -112,12 +112,17 @@ def _run_gated(config, config_file): sys.exit(0) print(f"šŸ“‹ Found {len(models)} model(s) to cache:", models) failed_models = [] - for repo_id in models: + for entry in models: + if isinstance(entry, str): + repo_id, revision = entry, None + else: + repo_id, revision = entry["repo"], entry["revision"] print(f"\nšŸš€ Processing: {repo_id}...") try: # snapshot_download automatically reads and uses the HF_HOME env var snapshot_download( repo_id=repo_id, + revision=revision, token=token, force_download=force, ignore_patterns=["*.pt", "*.bin"], diff --git a/tests/data/decoder_output_refs.json b/tests/data/decoder_output_refs.json index e8d8b82c3..f50a1a019 100644 --- a/tests/data/decoder_output_refs.json +++ b/tests/data/decoder_output_refs.json @@ -2,7 +2,6 @@ "google/gemma-4-31B": { "dtype": "float32", "max_tokens": 16, - "revision": "5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89", "results": [ { "logprobs": [ @@ -178,12 +177,12 @@ "\n" ] } - ] + ], + "revision": "5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89" }, "ibm-granite/granite-3.3-8b-instruct": { "dtype": "float32", "max_tokens": 16, - "revision": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b", "results": [ { "logprobs": [ @@ -359,12 +358,12 @@ "," ] } - ] + ], + "revision": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b" }, "ibm-granite/granite-4.1-8b": { "dtype": "float32", "max_tokens": 16, - "revision": "1504002f650e656a0a3789d99574df12e3e94ed0", "results": [ { "logprobs": [ @@ -540,6 +539,188 @@ " `" ] } - ] + ], + "revision": "1504002f650e656a0a3789d99574df12e3e94ed0" + }, + "meta-llama/Llama-3.1-8B-Instruct": { + "dtype": "float32", + "max_tokens": 16, + "results": [ + { + "logprobs": [ + -0.48447, + -0.932041, + -0.928222, + -0.034002, + -0.69502, + -0.002108, + -0.00067, + -0.004681, + -0.565966, + -0.349952, + -0.521094, + -0.55201, + -5.5e-05, + -0.076344, + -1.010146, + -0.003694 + ], + "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup.\n\n### Response:", + "text": " \n\nHere's a step-by-step guide to preparing delicious chicken soup:\n\n1.", + "token_ids": [ + 4815, + 8586, + 596, + 264, + 3094, + 14656, + 30308, + 8641, + 311, + 20646, + 18406, + 16553, + 19724, + 1473, + 16, + 13 + ], + "tokens": [ + " \n\n", + "Here", + "'s", + " a", + " step", + "-by", + "-step", + " guide", + " to", + " preparing", + " delicious", + " chicken", + " soup", + ":\n\n", + "1", + "." + ] + }, + { + "logprobs": [ + -0.489573, + -0.563746, + -1.127867, + -0.279811, + -0.700401, + -0.172913, + -0.405786, + -0.011894, + -0.274383, + -0.763821, + -1.057111, + -0.333751, + -1.217825, + -0.639492, + -0.759504, + -0.141694 + ], + "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nWhat are the main businesses of IBM?\n\n### Response:", + "text": " \nIBM is a multinational technology and consulting company that operates in various sectors, including", + "token_ids": [ + 720, + 68838, + 374, + 264, + 69026, + 5557, + 323, + 31831, + 2883, + 430, + 27149, + 304, + 5370, + 26593, + 11, + 2737 + ], + "tokens": [ + " \n", + "IBM", + " is", + " a", + " multinational", + " technology", + " and", + " consulting", + " company", + " that", + " operates", + " in", + " various", + " sectors", + ",", + " including" + ] + }, + { + "logprobs": [ + -0.337628, + -1.170689, + -0.002479, + -0.026265, + -0.631445, + -0.105513, + -0.064008, + -0.027309, + -0.001081, + -0.000404, + -0.003033, + -0.078346, + -0.043913, + -0.082205, + -0.045746, + -0.765835 + ], + "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nConvert char to string in Java.\n\n### Response:", + "text": " \n\nTo convert a char to a string in Java, you can use the `", + "token_ids": [ + 4815, + 1271, + 5625, + 264, + 1181, + 311, + 264, + 925, + 304, + 8102, + 11, + 499, + 649, + 1005, + 279, + 1595 + ], + "tokens": [ + " \n\n", + "To", + " convert", + " a", + " char", + " to", + " a", + " string", + " in", + " Java", + ",", + " you", + " can", + " use", + " the", + " `" + ] + } + ], + "revision": "0e9e39f249a16976918f6564b8830bc894c89659" } } diff --git a/tests/data/generate_decoder_output_refs.py b/tests/data/generate_decoder_output_refs.py index ea0ea64f4..328dca1ca 100644 --- a/tests/data/generate_decoder_output_refs.py +++ b/tests/data/generate_decoder_output_refs.py @@ -36,12 +36,14 @@ "ibm-granite/granite-3.3-8b-instruct", "ibm-granite/granite-4.1-8b", "google/gemma-4-31B", + "meta-llama/Llama-3.1-8B-Instruct", ] MODEL_REVISIONS = { "ibm-granite/granite-3.3-8b-instruct": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b", "ibm-granite/granite-4.1-8b": "1504002f650e656a0a3789d99574df12e3e94ed0", "google/gemma-4-31B": "5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89", + "meta-llama/Llama-3.1-8B-Instruct": "0e9e39f249a16976918f6564b8830bc894c89659", } # Must stay under MAX_NUM_BATCHED_TOKENS (test_model_quality.py) so each prefill lands diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index 49b981c7e..2105cc1c9 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -35,6 +35,7 @@ "ibm-granite/granite-3.3-8b-instruct", "ibm-granite/granite-4.1-8b", "google/gemma-4-31B", + "meta-llama/Llama-3.1-8B-Instruct", ] # fp16 on device reorders accumulation against the fp32 reference, so probabilities are From ea46caefe2b07f244ae1252159239a8e85f99718 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Fri, 4 Sep 2026 16:25:02 +0200 Subject: [PATCH 05/18] fix(tests): judge greedy divergence on the same token in both distributions Signed-off-by: Yannick Schnider --- tests/e2e/test_model_quality.py | 48 ++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index 2105cc1c9..eca7f1a89 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -42,6 +42,11 @@ # compared with a tolerance. Same default as sendnn-inference's TEST_ABS_TOL. ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.08")) +# Enough of the distribution that HF's greedy token is present even when Spyre picks a +# different one -- `_compare_against_hf` needs p(HF token) under *Spyre* to tell a +# near-tie from two distributions that disagree. 20 is vLLM's default `max_logprobs`. +NUM_LOGPROBS = 20 + MAX_MODEL_LEN = 256 MAX_NUM_SEQS = 3 # Caps the compiled buckets (platform.py) and so warmup; every prompt fits one bucket. @@ -83,7 +88,7 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No SamplingParams( temperature=0.0, max_tokens=max_tokens, - logprobs=0, # logprob of the sampled token only + logprobs=NUM_LOGPROBS, # sampled token plus enough to locate HF's ignore_eos=True, # the reference is a fixed-length run with EOS disabled ), use_tqdm=False, @@ -111,17 +116,46 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu zip(hf_result["token_ids"], hf_result["logprobs"], token_ids, logprobs) ): hf_prob, prob = math.exp(hf_logprob), math.exp(logprob) - probs_close = math.isclose(hf_prob, prob, abs_tol=ABS_TOL) detail = ( f"step {step}: token {token_id} ({completion.logprobs[step][token_id].decoded_token!r}," f" p={prob:.4f}) vs HF {hf_id} ({hf_result['tokens'][step]!r}, p={hf_prob:.4f})" ) if hf_id != token_id: - # Greedy paths only diverge legitimately on a near-tie, and past that point - # the prefixes differ, so no later token is comparable. - assert probs_close, f"{model}: wrong token, {detail}" - print(f" diverged on a near-tie at {detail}; not comparing further") + # Greedy paths only diverge legitimately on a near-tie. Judge that on the HF + # token in *both* distributions, never on the two sampled tokens' own + # probabilities: those agree whenever the models are equally confident, so + # HF at p=0.9 on one token and Spyre at p=0.9 on another -- a total + # disagreement -- would read as a tie. Past this step the prefixes differ, + # so no later token is comparable either way. + spyre_hf = completion.logprobs[step].get(hf_id) + assert spyre_hf is not None, ( + f"{model}: wrong token and HF's token is outside Spyre's top " + f"{NUM_LOGPROBS}, so the distributions disagree outright, {detail}" + ) + spyre_hf_prob = math.exp(spyre_hf.logprob) + assert math.isclose(spyre_hf_prob, hf_prob, abs_tol=ABS_TOL), ( + f"{model}: wrong token and p(HF token) differs by more than {ABS_TOL} " + f"(Spyre {spyre_hf_prob:.4f} vs HF {hf_prob:.4f}), {detail}" + ) + # A tie also means Spyre itself ranks the two level. Without this, a flat HF + # distribution (its own argmax at p=0.1) would excuse Spyre being confidently + # elsewhere at p=0.85, since p(HF token) still matches at 0.1 in both. + # Bound is 2*ABS_TOL, not ABS_TOL: HF picked its token, so it led there + # (p_hf(spyre token) <= hf_prob), and each of the two may drift by ABS_TOL in + # the opposite direction, which is what flipped the argmax in the first place. + tie_tol = 2 * ABS_TOL + assert math.isclose(prob, spyre_hf_prob, abs_tol=tie_tol), ( + f"{model}: wrong token, and Spyre puts it {prob - spyre_hf_prob:.4f} > " + f"{tie_tol} above HF's token (p={spyre_hf_prob:.4f}), so this is not a " + f"near-tie, {detail}" + ) + print( + f" diverged on a near-tie at {detail}; p(HF token) on Spyre " + f"{spyre_hf_prob:.4f}; not comparing further" + ) return - assert probs_close, f"{model}: probability differs by more than {ABS_TOL}, {detail}" + assert math.isclose(hf_prob, prob, abs_tol=ABS_TOL), ( + f"{model}: probability differs by more than {ABS_TOL}, {detail}" + ) From ec8d084723622183ca4221bc2fbaf1ba9cb7d485 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Fri, 4 Sep 2026 16:31:15 +0200 Subject: [PATCH 06/18] test(e2e): assert model-quality prompts fit the largest compiled prefill bucket Signed-off-by: Yannick Schnider --- tests/data/generate_decoder_output_refs.py | 4 ++-- tests/e2e/test_model_quality.py | 23 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/data/generate_decoder_output_refs.py b/tests/data/generate_decoder_output_refs.py index 328dca1ca..a555fe6b4 100644 --- a/tests/data/generate_decoder_output_refs.py +++ b/tests/data/generate_decoder_output_refs.py @@ -46,8 +46,8 @@ "meta-llama/Llama-3.1-8B-Instruct": "0e9e39f249a16976918f6564b8830bc894c89659", } -# Must stay under MAX_NUM_BATCHED_TOKENS (test_model_quality.py) so each prefill lands -# in a single compiled bucket. +# Must stay within MAX_NUM_BATCHED_TOKENS (test_model_quality.py) so each prefill lands +# in a single compiled bucket; that test asserts it before building its engine. _TEMPLATE = ( "Below is an instruction that describes a task. Write a response that " "appropriately completes the request.\n\n### Instruction:\n{}\n\n### Response:" diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index eca7f1a89..ee27eda63 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -72,6 +72,8 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No max_tokens = ref["max_tokens"] revision = ref["revision"] + _assert_prompts_fit_prefill_bucket(model, revision, prompts) + engine = LLM( model=model, revision=revision, @@ -99,6 +101,27 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No _compare_against_hf(model, hf_result, output) +def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[str]) -> None: + """Fail loudly if a prompt outgrew the largest compiled prefill bucket. + + Nothing else does: `next_bucket` stick-aligns past the end of the ladder + (spyre_shape_bucketer.py), so an over-long prompt is not an error but an uncompiled + shape -- it recompiles inside generate(), and the raised + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS lets that grind for hours instead of failing. Run + before the engine is built so an edited prompt costs seconds, not a warmup. + """ + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(model, revision=revision) + for prompt in prompts: + num_tokens = len(tokenizer(prompt).input_ids) + assert num_tokens <= MAX_NUM_BATCHED_TOKENS, ( + f"{model}: prompt is {num_tokens} tokens, past the largest compiled bucket " + f"({MAX_NUM_BATCHED_TOKENS}) -- it would recompile at generate() time. Shorten " + f"it, or raise MAX_NUM_BATCHED_TOKENS here and in the generator: {prompt!r}" + ) + + def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOutput) -> None: completion = output.outputs[0] token_ids = list(completion.token_ids) From ed6d7e4ff1fa4d1c030892ac10de73d30f8f8d53 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Fri, 4 Sep 2026 16:49:43 +0200 Subject: [PATCH 07/18] test(e2e): scale probability tolerance to reference confidence, report step coverage Signed-off-by: Yannick Schnider --- docs/contributing/README.md | 10 +++++- tests/e2e/test_model_quality.py | 56 +++++++++++++++++++++++++-------- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/docs/contributing/README.md b/docs/contributing/README.md index de33cfe19..ec925321d 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -105,7 +105,15 @@ python tests/data/generate_encoder_embed_refs.py Regenerate only when the *expected* output changes (new model or prompt), never to make a failing test pass ā€” that is the regression the gate exists to catch. Prompts are per model: `MODEL_PROMPTS` in the generator says which models are restricted and why. -`SPYRE_TEST_ABS_TOL` (default `0.08`) sets the decoder probability tolerance. +`SPYRE_TEST_ABS_TOL` (default `0.08`) and `SPYRE_TEST_REL_TOL` (default `0.5`) set the +decoder probability tolerance: the stricter of the two applies, so a low-confidence +reference token is held to a fraction rather than to the same absolute margin. + +A greedy path that diverges from HF on a near-tie cannot be compared past the split, so +each decoder case prints how many reference steps it actually matched +(`matched 32/48 reference steps (16/16, 0/16, 16/16 per prompt)`). Read that line rather +than the pass/fail alone: a prompt that stops after a step or two gates very little, and +the fix is a prompt whose greedy path is confident, not a looser tolerance. #### Upstream Test Integration diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index ee27eda63..9e95a3448 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -41,6 +41,12 @@ # fp16 on device reorders accumulation against the fp32 reference, so probabilities are # compared with a tolerance. Same default as sendnn-inference's TEST_ABS_TOL. ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.08")) +# ABS_TOL alone is not a uniform bound: it holds p=0.999 to 8% but lets p=0.08 land +# anywhere in [0, 0.16], a 2x relative error, so the gate is loosest exactly where the +# reference is least certain. Below the ABS_TOL/REL_TOL crossover the bound goes +# relative, holding a low-confidence token to the same *fraction* instead of the same +# margin. At the defaults the crossover is p=0.16, so nothing above it changes. +REL_TOL = float(os.environ.get("SPYRE_TEST_REL_TOL", "0.5")) # Enough of the distribution that HF's greedy token is present even when Spyre picks a # different one -- `_compare_against_hf` needs p(HF token) under *Spyre* to tell a @@ -97,8 +103,20 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No ) assert [output.prompt for output in outputs] == prompts, "Model output contained wrong prompt!" - for hf_result, output in zip(ref["results"], outputs): + matched = [ _compare_against_hf(model, hf_result, output) + for hf_result, output in zip(ref["results"], outputs) + ] + # A prompt that diverges early verifies only the steps before the split, so a green + # case is not automatically a well-covered one. Printed (PYTEST_ARGS carries -s) so + # the coverage a run actually achieved is visible without having to fail first. + per_prompt = ", ".join(f"{n}/{max_tokens}" for n in matched) + print( + f"\n{model}: matched {sum(matched)}/{len(prompts) * max_tokens} reference steps " + f"({per_prompt} per prompt). Prompts that stop after a step or two diverged on a " + f"near-tie and gate little -- see MODEL_PROMPTS in " + f"tests/data/generate_decoder_output_refs.py." + ) def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[str]) -> None: @@ -122,7 +140,16 @@ def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[ ) -def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOutput) -> None: +def _prob_tol(reference_prob: float) -> float: + """Tolerance for one probability comparison, tightening as the reference gets small. + + ``min`` and not ``max``: this only ever tightens ABS_TOL, never loosens it, so the + bound is the stricter of "within ABS_TOL" and "within REL_TOL of the reference". + """ + return min(ABS_TOL, REL_TOL * reference_prob) + + +def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOutput) -> int: completion = output.outputs[0] token_ids = list(completion.token_ids) logprobs = [completion.logprobs[i][t].logprob for i, t in enumerate(token_ids)] @@ -139,6 +166,7 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu zip(hf_result["token_ids"], hf_result["logprobs"], token_ids, logprobs) ): hf_prob, prob = math.exp(hf_logprob), math.exp(logprob) + tol = _prob_tol(hf_prob) detail = ( f"step {step}: token {token_id} ({completion.logprobs[step][token_id].decoded_token!r}," f" p={prob:.4f}) vs HF {hf_id} ({hf_result['tokens'][step]!r}, p={hf_prob:.4f})" @@ -157,28 +185,30 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu f"{NUM_LOGPROBS}, so the distributions disagree outright, {detail}" ) spyre_hf_prob = math.exp(spyre_hf.logprob) - assert math.isclose(spyre_hf_prob, hf_prob, abs_tol=ABS_TOL), ( - f"{model}: wrong token and p(HF token) differs by more than {ABS_TOL} " + assert abs(spyre_hf_prob - hf_prob) <= tol, ( + f"{model}: wrong token and p(HF token) differs by more than {tol:.4f} " f"(Spyre {spyre_hf_prob:.4f} vs HF {hf_prob:.4f}), {detail}" ) # A tie also means Spyre itself ranks the two level. Without this, a flat HF # distribution (its own argmax at p=0.1) would excuse Spyre being confidently # elsewhere at p=0.85, since p(HF token) still matches at 0.1 in both. - # Bound is 2*ABS_TOL, not ABS_TOL: HF picked its token, so it led there - # (p_hf(spyre token) <= hf_prob), and each of the two may drift by ABS_TOL in + # Bound is doubled: HF picked its token, so it led there + # (p_hf(spyre token) <= hf_prob), and each of the two may drift by `tol` in # the opposite direction, which is what flipped the argmax in the first place. - tie_tol = 2 * ABS_TOL - assert math.isclose(prob, spyre_hf_prob, abs_tol=tie_tol), ( + tie_tol = 2 * tol + assert abs(prob - spyre_hf_prob) <= tie_tol, ( f"{model}: wrong token, and Spyre puts it {prob - spyre_hf_prob:.4f} > " - f"{tie_tol} above HF's token (p={spyre_hf_prob:.4f}), so this is not a " - f"near-tie, {detail}" + f"{tie_tol:.4f} above HF's token (p={spyre_hf_prob:.4f}), so this is not " + f"a near-tie, {detail}" ) print( f" diverged on a near-tie at {detail}; p(HF token) on Spyre " f"{spyre_hf_prob:.4f}; not comparing further" ) - return + return step - assert math.isclose(hf_prob, prob, abs_tol=ABS_TOL), ( - f"{model}: probability differs by more than {ABS_TOL}, {detail}" + assert abs(hf_prob - prob) <= tol, ( + f"{model}: probability differs by more than {tol:.4f}, {detail}" ) + + return len(token_ids) From ca12d395dd142a8c8e821a4de8f53ba888e275a4 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Fri, 4 Sep 2026 17:19:48 +0200 Subject: [PATCH 08/18] refactor(ci): share the cache-config entry parser, treat missing revision as unpinned Signed-off-by: Yannick Schnider --- .github/cache_config/manage_cache.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/cache_config/manage_cache.py b/.github/cache_config/manage_cache.py index a6650c94b..70825de42 100644 --- a/.github/cache_config/manage_cache.py +++ b/.github/cache_config/manage_cache.py @@ -56,6 +56,12 @@ def main(): _run_gated(config, config_file) +def _parse_entry(entry): + if isinstance(entry, str): + return entry, None + return entry["repo"], entry.get("revision") + + def _run_public(config, config_file): """Cache public_models entries.""" models = config.get("public_models", []) @@ -65,10 +71,7 @@ def _run_public(config, config_file): print(f"šŸ“‹ Found {len(models)} public model(s) to cache:", models) failed_models = [] for entry in models: - if isinstance(entry, str): - repo_id, revision = entry, None - else: - repo_id, revision = entry["repo"], entry["revision"] + repo_id, revision = _parse_entry(entry) print(f"\nšŸš€ Processing: {repo_id}...") try: snapshot_download( @@ -113,10 +116,7 @@ def _run_gated(config, config_file): print(f"šŸ“‹ Found {len(models)} model(s) to cache:", models) failed_models = [] for entry in models: - if isinstance(entry, str): - repo_id, revision = entry, None - else: - repo_id, revision = entry["repo"], entry["revision"] + repo_id, revision = _parse_entry(entry) print(f"\nšŸš€ Processing: {repo_id}...") try: # snapshot_download automatically reads and uses the HF_HOME env var From ac3fd0cc71c818d412911c20fa7e7751b1770d98 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Fri, 4 Sep 2026 18:11:09 +0200 Subject: [PATCH 09/18] cosmetics Signed-off-by: Yannick Schnider --- .pre-commit-config.yaml | 4 ++-- Makefile | 5 ++++- pyproject.toml | 2 +- tests/e2e/test_model_quality.py | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2a7b21dd..e23caa9b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,8 +16,8 @@ repos: rev: v1.41.0 hooks: - id: typos - # Reference data holds tokenizer fragments ("other" from " G"+"other"), which - # --fix silently rewrites into real words. + # Reference data holds tokenizer fragments -- "Gather" arrives as " G"+"ather", + # "Machines" as " M"+"achines" -- which --fix silently rewrites into real words. exclude: '^tests/data/.*\.json$' - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.45.0 diff --git a/Makefile b/Makefile index 60f74a9a8..b224d2e2a 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,10 @@ else ifeq ($(TEST_TYPE),integration) # model, far too slow for the integration tier. MARK_EXPR := -m "not (distributed or upstream or attention or probe or model_quality)" else ifeq ($(TEST_TYPE),unit) -MARK_EXPR := -m "not upstream" +# Mirrors the CI jobs the `unit` tier schedules (_test_matrix.yaml test_types): +# smoke + compile + attention + encoder-attention + distributed. model_quality is +# regression/trunk only there, so it stays out here too. +MARK_EXPR := -m "not (upstream or model_quality)" else # The validation above already rejected any type outside VALID_TEST_TYPES, so # a value that reaches here IS valid but has no marker mapping above -- i.e. a diff --git a/pyproject.toml b/pyproject.toml index b1cd25b95..f7cd6a2f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -265,7 +265,7 @@ markers = [ "encoder_attention: encoder-attention subset (tests/attention/test_spyre_encoder_attn.py); also marked attention, split into its own CI job", "probe: torch-spyre backend probes (tests/probes/); run in their own CI job, excluded from integration", "uses_subprocess: tests that spawn a subprocess engine; the plugin runs them first, before the main process initializes the Spyre device", - "model_quality: product-model output gates vs cached HF references (tests/e2e/test_model_quality.py and test_encoder_models.py::test_encoder_embed_models_compiled); loads 8B-31B models, runs in its own CI job", + "model_quality: product-model output gates vs cached HF references (tests/e2e/test_model_quality.py and test_encoder_models.py::test_encoder_embed_models_compiled); compiles models from 125M encoders up to 31B decoders, runs in its own CI job", "rotary: Spyre RoPE custom-op tests (tests/custom_ops/test_rotary_embedding.py)", "fp8: FP8 quantization tests (tests/custom_ops/test_fp8.py)", ] diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index 9e95a3448..b4cb28e7d 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -163,7 +163,7 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu ) for step, (hf_id, hf_logprob, token_id, logprob) in enumerate( - zip(hf_result["token_ids"], hf_result["logprobs"], token_ids, logprobs) + zip(hf_result["token_ids"], hf_result["logprobs"], token_ids, logprobs, strict=True) ): hf_prob, prob = math.exp(hf_logprob), math.exp(logprob) tol = _prob_tol(hf_prob) From a71a01133e41895a2526da537199ab4a47103e8f Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Fri, 4 Sep 2026 23:28:15 +0200 Subject: [PATCH 10/18] add reranker and fp8 tests Signed-off-by: Yannick Schnider --- .../cache_config/hf_models_and_datasets.yaml | 9 +- docs/contributing/README.md | 18 ++- pyproject.toml | 2 +- tests/data/generate_rerank_score_refs.py | 123 ++++++++++++++++++ tests/data/rerank_score_refs.json | 40 ++++++ tests/e2e/test_encoder_models.py | 75 +++++++++-- tests/e2e/test_model_quality.py | 66 ++++++++++ 7 files changed, 320 insertions(+), 13 deletions(-) create mode 100644 tests/data/generate_rerank_score_refs.py create mode 100644 tests/data/rerank_score_refs.json diff --git a/.github/cache_config/hf_models_and_datasets.yaml b/.github/cache_config/hf_models_and_datasets.yaml index 8967de48d..e6897a952 100644 --- a/.github/cache_config/hf_models_and_datasets.yaml +++ b/.github/cache_config/hf_models_and_datasets.yaml @@ -21,13 +21,20 @@ public_models: - ibm-granite/granite-embedding-278m-multilingual - intfloat/multilingual-e5-large - sentence-transformers/all-roberta-large-v1 - - BAAI/bge-reranker-v2-m3 + - repo: BAAI/bge-reranker-v2-m3 + revision: 953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e + - repo: BAAI/bge-reranker-large + revision: 55611d7bca2a7133960a6d3b71e083071bbfc312 - ibm-ai-platform/micro-g3.3-8b-instruct-1b - ibm-granite/granite-4.1-3b - repo: ibm-granite/granite-3.3-8b-instruct revision: 51dd4bc2ade4059a6bd87649d68aa11e4fb2529b - repo: ibm-granite/granite-4.1-8b revision: 1504002f650e656a0a3789d99574df12e3e94ed0 + - repo: ibm-granite/granite-3.3-8b-instruct-FP8 + revision: 4b5990b8d402a75febe0086abbf1e490af494e3d + - repo: ibm-granite/granite-4.1-8b-fp8 + revision: 070021b3608433b6107a00733d561c9779b9937e - repo: google/gemma-4-31B revision: 5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89 - qwrt/Swedish0.1M diff --git a/docs/contributing/README.md b/docs/contributing/README.md index ec925321d..078f921b2 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -87,7 +87,8 @@ pytest --upstream -m "attention" The `model_quality` marker gates the product models on their output: each is loaded **compiled** (the platform default) and compared against a CPU HF reference ā€” greedy token ids plus per-token probabilities for the decoders -(`tests/e2e/test_model_quality.py`), cosine similarity for the embedding models +(`tests/e2e/test_model_quality.py`), cosine similarity for the embedding models and +sigmoid scores plus document ranking for the cross-encoder rerankers (`tests/e2e/test_encoder_models.py`). ```bash @@ -100,14 +101,27 @@ The models are too large to run through transformers in CI, so the references li ```bash python tests/data/generate_decoder_output_refs.py --models ibm-granite/granite-4.1-8b python tests/data/generate_encoder_embed_refs.py +python tests/data/generate_rerank_score_refs.py ``` Regenerate only when the *expected* output changes (new model or prompt), never to make a failing test pass ā€” that is the regression the gate exists to catch. Prompts are per -model: `MODEL_PROMPTS` in the generator says which models are restricted and why. +model: `MODEL_PROMPTS` (decoders) and `MODEL_DOCUMENTS` (rerankers) in the generators say +which models get their own inputs and why. `SPYRE_TEST_ABS_TOL` (default `0.08`) and `SPYRE_TEST_REL_TOL` (default `0.5`) set the decoder probability tolerance: the stricter of the two applies, so a low-confidence reference token is held to a fraction rather than to the same absolute margin. +`SPYRE_TEST_SCORE_ABS_TOL` (default `0.03`) and `SPYRE_TEST_SCORE_REL_TOL` (default +`0.5`) bound a reranker score the same stricter-of-the-two way, which matters more here +than for the decoders because the scores are sigmoids sitting near the rails. The +document ranking is checked separately: all scores can drift the same direction without +reordering anything, and two documents can reorder while both stay inside the bound. + +The FP8 decoder checkpoints (`FP8_DECODER_MODELS`) are load-and-decode cases with no +reference of their own ā€” writing one means dequantizing a compressed-tensors checkpoint on +CPU, which the generator does not do ā€” so they gate the FP8 weight load and the Spyre +`aten._scaled_mm` kernel against breaking outright, while the unquantized siblings gate the +numerics. Each borrows its prompts from the sibling's reference entry. A greedy path that diverges from HF on a near-tie cannot be compared past the split, so each decoder case prints how many reference steps it actually matched diff --git a/pyproject.toml b/pyproject.toml index f7cd6a2f7..e52ff06c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -265,7 +265,7 @@ markers = [ "encoder_attention: encoder-attention subset (tests/attention/test_spyre_encoder_attn.py); also marked attention, split into its own CI job", "probe: torch-spyre backend probes (tests/probes/); run in their own CI job, excluded from integration", "uses_subprocess: tests that spawn a subprocess engine; the plugin runs them first, before the main process initializes the Spyre device", - "model_quality: product-model output gates vs cached HF references (tests/e2e/test_model_quality.py and test_encoder_models.py::test_encoder_embed_models_compiled); compiles models from 125M encoders up to 31B decoders, runs in its own CI job", + "model_quality: product-model output gates vs cached HF references (tests/e2e/test_model_quality.py and the compiled encoder cases in test_encoder_models.py); compiles models from 125M encoders up to 31B decoders, runs in its own CI job", "rotary: Spyre RoPE custom-op tests (tests/custom_ops/test_rotary_embedding.py)", "fp8: FP8 quantization tests (tests/custom_ops/test_fp8.py)", ] diff --git a/tests/data/generate_rerank_score_refs.py b/tests/data/generate_rerank_score_refs.py new file mode 100644 index 000000000..6099e574e --- /dev/null +++ b/tests/data/generate_rerank_score_refs.py @@ -0,0 +1,123 @@ +# Copyright 2026 The Spyre-Inference Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Write CPU HF cross-encoder scores to rerank_score_refs.json for the reranker gates. + +Each run merges into the existing file. + + python tests/data/generate_rerank_score_refs.py + python tests/data/generate_rerank_score_refs.py --models BAAI/bge-reranker-large +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import torch +from transformers import AutoModelForSequenceClassification, AutoTokenizer + +# Model ids must match tests/e2e/test_encoder_models.py. +RERANKER_MODELS = [ + "BAAI/bge-reranker-v2-m3", + "BAAI/bge-reranker-large", +] + +MODEL_REVISIONS = { + "BAAI/bge-reranker-v2-m3": "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e", + "BAAI/bge-reranker-large": "55611d7bca2a7133960a6d3b71e083071bbfc312", +} + +# One query against documents a reranker should separate widely, listed most relevant +# first. The spread is what lets the test gate the *ranking* and not only the individual +# scores, so neighbouring documents have to stay far enough apart that fp16 drift cannot +# swap them: every adjacent pair below differs by at least a factor of 8 in odds. +QUERY = "What is the capital of France?" +DOCUMENTS = [ + "The capital of France is Paris.", + "Paris is the largest city in France by population.", + "The Eiffel Tower stands on the Champ de Mars in Paris.", + "France is a country in Western Europe with about 68 million inhabitants.", + "Berlin is the capital of Germany.", + "The IBM Spyre accelerator runs AI inference workloads.", +] + +# bge-reranker-large saturates: it scores every Paris-adjacent document above 0.9994, so on +# the list above its top five land within 5e-4 of each other and their order is decided by +# fp16 noise rather than by relevance. Give it documents spread across the range it +# actually resolves. Drop this entry only for a model that separates the shared list. +MODEL_DOCUMENTS = { + "BAAI/bge-reranker-large": [ + "The capital of France is Paris.", + "France is a country in Western Europe with about 68 million inhabitants.", + "France moved its seat of government several times in its history.", + "Berlin is the capital of Germany.", + "The IBM Spyre accelerator runs AI inference workloads.", + ], +} + +# Scores span ~1e-5 to ~1 and the test bounds small ones *relatively*, so a fixed number +# of decimals has to keep several significant digits at the bottom of that range. +_ROUND = 8 + +OUT_PATH = Path(__file__).parent / "rerank_score_refs.json" + + +def generate_reference(model_id: str, revision: str) -> dict[str, Any]: + tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision) + model = AutoModelForSequenceClassification.from_pretrained( + model_id, revision=revision, dtype=torch.float32 + ) + model.eval() + + documents = MODEL_DOCUMENTS.get(model_id, DOCUMENTS) + scores = [] + for document in documents: + # The same tokenizer call vLLM's cross-encoder io_processor makes, one pair at a + # time so no padding reaches the reference. + inputs = tokenizer(text=QUERY, text_pair=document, return_tensors="pt") + with torch.inference_mode(): + logit = model(**inputs).logits.reshape(-1) + assert logit.numel() == 1, f"{model_id}: expected num_labels=1, got {logit.numel()}" + # vLLM's PoolerClassify sigmoids a single-label head, so the reference is a + # probability and the test's tolerance is an absolute bound on [0, 1]. + scores.append(round(float(torch.sigmoid(logit)[0]), _ROUND)) + print(f" {document!r}\n -> {scores[-1]:.6f}", flush=True) + + return { + "revision": revision, + "query": QUERY, + "documents": documents, + "scores": scores, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--models", nargs="+", default=RERANKER_MODELS) + parser.add_argument("--out", type=Path, default=OUT_PATH) + args = parser.parse_args() + + data = json.loads(args.out.read_text()) if args.out.exists() else {} + for model_id in args.models: + print(f"Scoring {model_id} ...", flush=True) + data[model_id] = generate_reference(model_id, MODEL_REVISIONS[model_id]) + args.out.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") + print(f"Wrote {args.out}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/tests/data/rerank_score_refs.json b/tests/data/rerank_score_refs.json new file mode 100644 index 000000000..cacf8cebe --- /dev/null +++ b/tests/data/rerank_score_refs.json @@ -0,0 +1,40 @@ +{ + "BAAI/bge-reranker-large": { + "documents": [ + "The capital of France is Paris.", + "France is a country in Western Europe with about 68 million inhabitants.", + "France moved its seat of government several times in its history.", + "Berlin is the capital of Germany.", + "The IBM Spyre accelerator runs AI inference workloads." + ], + "query": "What is the capital of France?", + "revision": "55611d7bca2a7133960a6d3b71e083071bbfc312", + "scores": [ + 0.99946493, + 0.0866667, + 0.00186743, + 0.0005165, + 7.623e-05 + ] + }, + "BAAI/bge-reranker-v2-m3": { + "documents": [ + "The capital of France is Paris.", + "Paris is the largest city in France by population.", + "The Eiffel Tower stands on the Champ de Mars in Paris.", + "France is a country in Western Europe with about 68 million inhabitants.", + "Berlin is the capital of Germany.", + "The IBM Spyre accelerator runs AI inference workloads." + ], + "query": "What is the capital of France?", + "revision": "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e", + "scores": [ + 0.99996591, + 0.9614659, + 0.59719944, + 0.04726163, + 0.00421512, + 1.604e-05 + ] + } +} diff --git a/tests/e2e/test_encoder_models.py b/tests/e2e/test_encoder_models.py index 97e5c16db..b2fe1860c 100644 --- a/tests/e2e/test_encoder_models.py +++ b/tests/e2e/test_encoder_models.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Spyre product embed tests vs cached HF refs and reranker smoke tests. +"""Spyre product encoder tests vs cached HF refs: embeddings, reranker scores, labels. -Regenerate embed refs: ``python tests/data/generate_encoder_embed_refs.py`` +Regenerate refs: ``python tests/data/generate_encoder_embed_refs.py`` and +``python tests/data/generate_rerank_score_refs.py`` """ from __future__ import annotations import json import math +import os from pathlib import Path import pytest @@ -51,10 +53,12 @@ "The quick brown fox jumps over the lazy dog.", ] -# Cross-encoder reranker smoke (classify / score path). One model is enough ā€” -# both BGE variants share XLMRobertaForSequenceClassification. +# Cross-encoder rerankers (classify / score path). The two BGE variants share +# XLMRobertaForSequenceClassification but not their weights or position table (514 vs +# 8194 slots), so each is gated on its own cached scores. RERANKER_MODELS = [ "BAAI/bge-reranker-v2-m3", + "BAAI/bge-reranker-large", ] # Token classification: the model applies its own classifier after casting to @@ -69,9 +73,23 @@ # Match upstream check_embeddings_close(tol=1e-2). COSINE_MIN = 0.99 +# Reranker references are sigmoid probabilities, and a bare absolute bound is a poor gate +# for one: the scores sit near the rails, where it permits an arbitrary relative error. +# Paired with a relative bound (stricter of the two wins, as in test_model_quality.py) a +# saturated reference is held to a fraction instead. Worst measured drift on the reference +# documents is 7e-3 absolute and 13% relative, both on bge-reranker-large, so each bound +# keeps ~4x margin. +SCORE_ABS_TOL = float(os.environ.get("SPYRE_TEST_SCORE_ABS_TOL", "0.03")) +SCORE_REL_TOL = float(os.environ.get("SPYRE_TEST_SCORE_REL_TOL", "0.5")) + _REF_PATH = Path(__file__).parent.parent / "data" / "encoder_embed_refs.json" _REFERENCES: dict = json.loads(_REF_PATH.read_text()) if _REF_PATH.exists() else {} +_RERANK_REF_PATH = Path(__file__).parent.parent / "data" / "rerank_score_refs.json" +_RERANK_REFERENCES: dict = ( + json.loads(_RERANK_REF_PATH.read_text()) if _RERANK_REF_PATH.exists() else {} +) + def _cosine(a: list[float], b: list[float]) -> float: return F.cosine_similarity( @@ -220,17 +238,56 @@ def test_encoder_embed_last_pooling() -> None: @pytest.mark.uses_subprocess @pytest.mark.parametrize("model", RERANKER_MODELS) def test_encoder_rerank_models(model: str) -> None: - """Load reranker and return one finite score via LLM.score().""" + """Spyre reranker scores match the cached HF references within tolerance.""" + _assert_rerank_scores_match_refs(model, enforce_eager=True) + + +@pytest.mark.model_quality +@pytest.mark.uses_subprocess +@pytest.mark.parametrize("model", RERANKER_MODELS) +def test_encoder_rerank_models_compiled(model: str) -> None: + """Same models and references, compiled rather than eager.""" + _assert_rerank_scores_match_refs(model, enforce_eager=False) + + +def _assert_rerank_scores_match_refs(model: str, enforce_eager: bool) -> None: + ref = _RERANK_REFERENCES.get(model) + if ref is None: + pytest.skip(f"No HF ref for {model}; run tests/data/generate_rerank_score_refs.py") + + documents = ref["documents"] + ref_scores = ref["scores"] llm = LLM( model=model, + revision=ref["revision"], + tokenizer_revision=ref["revision"], runner="pooling", max_model_len=64, max_num_seqs=1, - enforce_eager=True, + enforce_eager=enforce_eager, + ) + outputs = llm.score(ref["query"], documents) + assert len(outputs) == len(documents) + + scores = [out.outputs.score for out in outputs] + assert all(math.isfinite(s) for s in scores), f"{model}: non-finite score in {scores}" + + # A reranker is used for its ordering, and every score can drift the same direction + # without disturbing that -- so the ranking is checked apart from the per-score bound, + # which conversely passes on a pair that has swapped inside the tolerance. + order = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True) + ref_order = sorted(range(len(ref_scores)), key=lambda i: ref_scores[i], reverse=True) + assert order == ref_order, ( + f"{model}: ranked documents {order} vs cached HF {ref_order}; " + f"scores {scores} vs {ref_scores}" ) - scores = llm.score("What is Spyre?", "An IBM AI accelerator.") - assert len(scores) == 1 - assert math.isfinite(scores[0].outputs.score) + + for document, score, ref_score in zip(documents, scores, ref_scores, strict=True): + tol = min(SCORE_ABS_TOL, SCORE_REL_TOL * ref_score) + assert abs(score - ref_score) <= tol, ( + f"{model}: score {score:.6f} vs cached HF {ref_score:.6f} (tol {tol:.6f}) " + f"for {document!r}" + ) @pytest.mark.uses_subprocess diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index b4cb28e7d..98089d57c 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -38,6 +38,21 @@ "meta-llama/Llama-3.1-8B-Instruct", ] +# Weight-only FP8 (compressed-tensors) checkpoints, each mapped to the unquantized sibling +# whose reference entry it borrows prompts from -- the smoke test below compares no +# output, it only needs prompts that fit a compiled prefill bucket. Both run end to end on +# Spyre today, so the sibling entries are what gate the numerics. +FP8_DECODER_MODELS = { + "ibm-granite/granite-3.3-8b-instruct-FP8": "ibm-granite/granite-3.3-8b-instruct", + "ibm-granite/granite-4.1-8b-fp8": "ibm-granite/granite-4.1-8b", +} +FP8_REVISIONS = { + "ibm-granite/granite-3.3-8b-instruct-FP8": "4b5990b8d402a75febe0086abbf1e490af494e3d", + "ibm-granite/granite-4.1-8b-fp8": "070021b3608433b6107a00733d561c9779b9937e", +} +# Short: nothing is compared, so the run only has to prove decode advances at all. +FP8_MAX_TOKENS = 8 + # fp16 on device reorders accumulation against the fp32 reference, so probabilities are # compared with a tolerance. Same default as sendnn-inference's TEST_ABS_TOL. ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.08")) @@ -119,6 +134,57 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No ) +@pytest.mark.parametrize("model", FP8_DECODER_MODELS) +def test_fp8_decoder_model_smoke(model: str, monkeypatch: pytest.MonkeyPatch) -> None: + """A compiled FP8 checkpoint loads and decodes. + + Load-and-decode only, with no reference comparison: a reference for a + compressed-tensors checkpoint means dequantizing it on CPU first, which the generator + does not do. So this holds the FP8 weight load and the ``aten._scaled_mm`` kernel + (``custom_ops/fp8_linear_kernel.py``) to running at all rather than to a numerical + bound -- `test_decoder_model_output` gates the unquantized siblings' output. + """ + base = FP8_DECODER_MODELS[model] + base_ref = _REFERENCES.get(base) + assert base_ref is not None, ( + f"No HF reference for {base} in {_REF_PATH.name}, and {model} borrows its prompts; " + f"regenerate with `python tests/data/generate_decoder_output_refs.py --models {base}`" + ) + + monkeypatch.setenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "36000") + + prompts = [result["prompt"] for result in base_ref["results"]] + revision = FP8_REVISIONS[model] + + _assert_prompts_fit_prefill_bucket(model, revision, prompts) + + engine = LLM( + model=model, + revision=revision, + tokenizer_revision=revision, + enforce_eager=False, + max_model_len=MAX_MODEL_LEN, + max_num_seqs=MAX_NUM_SEQS, + max_num_batched_tokens=MAX_NUM_BATCHED_TOKENS, + compilation_config={"compile_sizes": COMPILE_SIZES}, + ) + + outputs = engine.generate( + prompts, + SamplingParams(temperature=0.0, max_tokens=FP8_MAX_TOKENS, ignore_eos=True), + use_tqdm=False, + ) + + assert [output.prompt for output in outputs] == prompts, "Model output contained wrong prompt!" + for output in outputs: + completion = output.outputs[0] + print(f"\n{model} prompt: {output.prompt!r}\n Spyre: {completion.text!r}") + assert len(completion.token_ids) == FP8_MAX_TOKENS, ( + f"{model}: generated {len(completion.token_ids)} of {FP8_MAX_TOKENS} tokens" + ) + assert completion.text.strip(), f"{model}: empty completion for {output.prompt!r}" + + def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[str]) -> None: """Fail loudly if a prompt outgrew the largest compiled prefill bucket. From fbacf76527d363149c80ba5dab5f9937d993ed11 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Mon, 7 Sep 2026 12:10:34 +0200 Subject: [PATCH 11/18] update comments Signed-off-by: Yannick Schnider --- .github/workflows/_test_matrix.yaml | 6 +++--- .pre-commit-config.yaml | 3 ++- Makefile | 18 +++++++++++------- tests/e2e/test_encoder_models.py | 11 ++++++----- tests/e2e/test_model_quality.py | 15 +++++++++------ 5 files changed, 31 insertions(+), 22 deletions(-) diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml index fab0793aa..d7b4214b6 100644 --- a/.github/workflows/_test_matrix.yaml +++ b/.github/workflows/_test_matrix.yaml @@ -283,9 +283,9 @@ jobs: image_label: image_torch_spyre test_target: test-smoke-shard-7 # Product-model output-quality gate. Carved out of the smoke shards - # above and deliberately NOT in `integration`: each case loads and - # compiles an 8B-31B model, too slow for the tier the multi-repo - # integration hook runs. + # above and deliberately NOT in `integration`: every case loads and + # compiles a product model, from the 125M encoders up to the 31B + # decoders, too slow for the tier the multi-repo integration hook runs. - cfg: Spyre model quality tests test_types: regression trunk runs_on: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e23caa9b3..8ebed7266 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,8 @@ repos: hooks: - id: typos # Reference data holds tokenizer fragments -- "Gather" arrives as " G"+"ather", - # "Machines" as " M"+"achines" -- which --fix silently rewrites into real words. + # "Machines" as " M"+"achines" -- which the hook's --write-changes silently rewrites + # into real words. exclude: '^tests/data/.*\.json$' - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.45.0 diff --git a/Makefile b/Makefile index b224d2e2a..0bcf8ddda 100644 --- a/Makefile +++ b/Makefile @@ -99,13 +99,16 @@ else ifeq ($(TEST_TYPE),integration) # compiled (enforce_eager=False) tests/e2e/test_compile.py cases. Probes and the # model-quality gate are excluded here just as the sharded smoke jobs exclude # them, each running in its own CI job: probes must not gate integration on -# strict-xfail flips, and every model-quality case loads and compiles an 8B-31B -# model, far too slow for the integration tier. +# strict-xfail flips, and the model-quality gate compiles a product model per case +# -- encoders from 125M up to the 31B decoders -- far too slow for the integration +# tier. MARK_EXPR := -m "not (distributed or upstream or attention or probe or model_quality)" else ifeq ($(TEST_TYPE),unit) # Mirrors the CI jobs the `unit` tier schedules (_test_matrix.yaml test_types): -# smoke + compile + attention + encoder-attention + distributed. model_quality is -# regression/trunk only there, so it stays out here too. +# the smoke shards (which now carry the compiled e2e cases), the attention shards, +# encoder-attention, the distributed shards and the probe shards -- so probes stay +# in here, unlike the integration tier above. model_quality is regression/trunk +# only there, so it stays out here too. MARK_EXPR := -m "not (upstream or model_quality)" else # The validation above already rejected any type outside VALID_TEST_TYPES, so @@ -193,9 +196,10 @@ test-smoke-shard: ## Run one smoke shard (SMOKE_SHARDS=N SMOKE_SHARD_ID=i). test-smoke-shard-%: $(MAKE) test-smoke-shard SMOKE_SHARD_ID=$* JUNIT_XML=$(JUNIT_XML) -# Carved out of smoke (and so out of integration): each case loads and compiles a -# product model at 8B-31B, which the smoke shards' runtime budget cannot absorb. -test-model-quality: ## Run the product-model output-quality gates (its own job; loads 8B-31B models). +# Carved out of smoke (and so out of integration): every case loads and compiles a +# product model -- a 125M-580M encoder or an 8B-31B decoder -- and the decoders alone +# are more than the smoke shards' runtime budget can absorb. +test-model-quality: ## Run the product-model output-quality gates (its own job; compiles 125M encoders up to 31B decoders). $(MAKE) run-one MARK_OVERRIDE='model_quality and not (distributed or upstream)' JUNIT_XML=$(JUNIT_XML) test-probes: ## Run the torch-spyre backend probes (excluded from integration), unsharded (local full run). diff --git a/tests/e2e/test_encoder_models.py b/tests/e2e/test_encoder_models.py index b2fe1860c..d8cc3eb84 100644 --- a/tests/e2e/test_encoder_models.py +++ b/tests/e2e/test_encoder_models.py @@ -74,11 +74,12 @@ COSINE_MIN = 0.99 # Reranker references are sigmoid probabilities, and a bare absolute bound is a poor gate -# for one: the scores sit near the rails, where it permits an arbitrary relative error. -# Paired with a relative bound (stricter of the two wins, as in test_model_quality.py) a -# saturated reference is held to a fraction instead. Worst measured drift on the reference -# documents is 7e-3 absolute and 13% relative, both on bge-reranker-large, so each bound -# keeps ~4x margin. +# for one: most of them sit just above zero, where 0.03 permits an arbitrary relative +# error -- 400x on the smallest. Paired with a relative bound (stricter of the two wins, +# as in test_model_quality.py) a near-zero reference is held to a fraction instead, while +# the near-one end stays on the absolute bound, which is already tight relatively there. +# Worst measured drift on the reference documents is 7e-3 absolute and 13% relative, both +# on bge-reranker-large, so each bound keeps ~4x margin. SCORE_ABS_TOL = float(os.environ.get("SPYRE_TEST_SCORE_ABS_TOL", "0.03")) SCORE_REL_TOL = float(os.environ.get("SPYRE_TEST_SCORE_REL_TOL", "0.5")) diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index 98089d57c..b57018638 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -70,7 +70,9 @@ MAX_MODEL_LEN = 256 MAX_NUM_SEQS = 3 -# Caps the compiled buckets (platform.py) and so warmup; every prompt fits one bucket. +# Top of COMPILE_SIZES below, which is what caps warmup: passing compile_sizes +# explicitly skips the default buckets platform.py would derive, and platform.py then +# clamps max_num_batched_tokens down to the largest bucket. Every prompt fits this one. MAX_NUM_BATCHED_TOKENS = 64 COMPILE_SIZES = [MAX_NUM_SEQS, MAX_NUM_BATCHED_TOKENS] @@ -188,11 +190,12 @@ def test_fp8_decoder_model_smoke(model: str, monkeypatch: pytest.MonkeyPatch) -> def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[str]) -> None: """Fail loudly if a prompt outgrew the largest compiled prefill bucket. - Nothing else does: `next_bucket` stick-aligns past the end of the ladder - (spyre_shape_bucketer.py), so an over-long prompt is not an error but an uncompiled - shape -- it recompiles inside generate(), and the raised - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS lets that grind for hours instead of failing. Run - before the engine is built so an edited prompt costs seconds, not a warmup. + Nothing else does: past the largest bucket `SpyreShapeBucketer.find_bucket` returns + None (spyre_shape_bucketer.py) and execute_model runs the shape unpadded, so an + over-long prompt is not an error but a Dynamo recompile inside generate() -- and the + raised VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS lets that grind for hours instead of + failing. Run before the engine is built so an edited prompt costs seconds, not a + warmup. """ from transformers import AutoTokenizer From 16b1a41193bca61558205e0db4875025491233c9 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Mon, 7 Sep 2026 13:35:23 +0200 Subject: [PATCH 12/18] pin encoder revisions Signed-off-by: Yannick Schnider --- .../cache_config/hf_models_and_datasets.yaml | 12 +++++--- docs/contributing/README.md | 6 ++++ tests/data/encoder_embed_refs.json | 2 +- tests/data/generate_encoder_embed_refs.py | 19 +++++++++++-- tests/data/generate_rerank_score_refs.py | 15 ++++++++-- tests/e2e/test_encoder_models.py | 28 ++++++++++++++++--- 6 files changed, 68 insertions(+), 14 deletions(-) diff --git a/.github/cache_config/hf_models_and_datasets.yaml b/.github/cache_config/hf_models_and_datasets.yaml index e6897a952..817adfa58 100644 --- a/.github/cache_config/hf_models_and_datasets.yaml +++ b/.github/cache_config/hf_models_and_datasets.yaml @@ -17,10 +17,14 @@ public_models: - mistralai/Mistral-7B-Instruct-v0.3 - BAAI/bge-base-en-v1.5 - sentence-transformers/all-MiniLM-L12-v2 - - ibm-granite/granite-embedding-125m-english - - ibm-granite/granite-embedding-278m-multilingual - - intfloat/multilingual-e5-large - - sentence-transformers/all-roberta-large-v1 + - repo: ibm-granite/granite-embedding-125m-english + revision: 4ab61ffd423be45cd932b21a7c696063d82bf45f + - repo: ibm-granite/granite-embedding-278m-multilingual + revision: a9cb5338491faf32b73dd17b714a31821c021bbf + - repo: intfloat/multilingual-e5-large + revision: 3d7cfbdacd47fdda877c5cd8a79fbcc4f2a574f3 + - repo: sentence-transformers/all-roberta-large-v1 + revision: cf74d8acd4f198de950bf004b262e6accfed5d2c - repo: BAAI/bge-reranker-v2-m3 revision: 953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e - repo: BAAI/bge-reranker-large diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 078f921b2..6691a28d8 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -108,6 +108,12 @@ Regenerate only when the *expected* output changes (new model or prompt), never failing test pass ā€” that is the regression the gate exists to catch. Prompts are per model: `MODEL_PROMPTS` (decoders) and `MODEL_DOCUMENTS` (rerankers) in the generators say which models get their own inputs and why. + +Every gated model is pinned to a revision, in the generator's `MODEL_REVISIONS` and again +in `.github/cache_config/hf_models_and_datasets.yaml`. Each generator writes the revision +it measured into its JSON, and the test loads that revision back ā€” so a reference always +names the weights it was taken from, and an upstream re-upload cannot silently redefine +what the gate compares against. Bumping a pin means regenerating that model's reference. `SPYRE_TEST_ABS_TOL` (default `0.08`) and `SPYRE_TEST_REL_TOL` (default `0.5`) set the decoder probability tolerance: the stricter of the two applies, so a low-confidence reference token is held to a fraction rather than to the same absolute margin. diff --git a/tests/data/encoder_embed_refs.json b/tests/data/encoder_embed_refs.json index e0f0a73a7..1af5337fa 100644 --- a/tests/data/encoder_embed_refs.json +++ b/tests/data/encoder_embed_refs.json @@ -1 +1 @@ -{"ibm-granite/granite-embedding-125m-english":{"embeddings":[[-0.01831,0.00958,-0.02759,-0.01465,-0.01917,-0.01819,-0.00061,-0.02112,-0.01337,0.04932,-0.0282,-0.02612,-0.01843,-0.02734,-0.00497,-0.0166,0.00455,0.00488,0.01215,-0.02673,0.0354,-0.05444,-0.02332,-0.052,-0.03247,-0.01807,0.02454,-0.01648,0.00598,0.0332,-0.01373,-0.02771,0.04199,0.01831,0.04785,-0.02441,-0.00806,-0.00922,0.01184,-0.03491,0.00105,-0.0047,-0.00671,-0.01611,0.01093,0.00084,0.01331,-0.00362,-0.04199,0.03394,0.03687,0.00352,0.00885,-0.00467,0.03687,-0.00371,0.03467,-0.026,-0.02002,-0.01733,-0.01495,0.03223,-0.01587,-0.00036,0.01276,0.04736,0.01941,-0.01953,-0.04565,0.0415,0.01447,-0.026,-0.04468,-0.02673,0.03394,-0.00928,-0.07471,-0.03516,-0.00415,0.02087,0.02112,0.01746,-0.01379,0.02026,-0.03394,-0.02258,0.00577,-0.02917,-0.02393,-0.01514,-0.00668,-0.00983,0.02429,-0.00552,-0.02502,0.00203,-0.02307,0.026,-0.01196,-0.00319,0.00787,-0.00903,0.02063,-0.013,0.04761,0.01337,-0.00255,0.04419,0.02588,0.00867,0.02661,-0.01215,-0.02625,0.04321,0.01483,-0.00167,-0.0152,0.00681,-0.0006,-0.00665,-0.00436,-0.00958,0.01337,-0.03931,0.01917,0.01257,0.00488,0.01495,0.00958,-0.01611,-0.00117,0.0072,0.05371,-0.00854,-0.0105,0.00433,-0.0054,0.05981,-0.02734,-0.02332,0.00171,0.02881,-0.04761,-0.01672,-0.021,-0.00824,0.02734,-0.03809,-0.02185,-0.0144,-0.04663,-0.04663,-0.01257,0.03052,-0.02039,-0.0043,0.05811,-0.00739,-0.02087,0.02283,0.02502,-0.02661,-0.01501,-0.02075,-0.01019,-0.01337,-0.01038,-0.01343,-0.00093,-0.02734,0.02612,-0.02844,-0.00519,0.03833,-0.00916,-0.03369,-0.0076,-0.0293,0.03931,0.00378,0.01196,0.03564,-0.01807,0.01697,-0.00055,-0.02356,-0.00371,0.00577,-0.02551,-0.01257,-0.03174,0.02039,0.02136,-0.01044,0.01105,-0.00934,-0.02319,-0.01929,-0.02502,-0.0238,-0.02612,-0.01294,-0.03687,-0.03906,0.01483,-0.008,-0.02185,0.00163,0.04297,-0.03687,-0.0062,0.00252,-0.04785,-0.33203,0.00879,0.01843,-0.33008,-0.00461,-0.01953,-0.01105,0.02209,-0.05078,-0.00714,-0.03589,0.02368,-0.01428,0.04053,0.03369,-0.01611,-0.00162,0.02576,-0.01709,-0.01123,0.04468,0.0144,-0.05493,0.00153,0.00656,-0.02625,0.00781,-0.01117,-0.03613,-0.02783,0.02771,-0.02954,-0.05835,-0.01514,-0.0199,-0.04565,-0.0354,-0.00198,-0.00421,-0.32617,0.02441,-0.00476,0.02881,-0.0105,-0.02356,0.01019,-0.01685,0.02209,-0.02148,-0.03149,0.01019,-0.00711,0.01385,0.03906,-0.0152,0.00157,-0.01489,-0.03247,0.02905,-0.01782,0.01831,-0.00063,-0.00063,0.02356,-0.01422,-0.04614,-0.04321,0.04028,-0.01624,0.00333,-0.00687,0.01019,-0.01422,-0.01624,-0.00842,-0.03296,0.04297,-0.01447,0.01782,0.00793,-0.04883,0.03101,0.03467,-0.00879,-0.00177,0.02112,-0.00763,-0.0282,0.01385,0.00775,-0.0152,-0.01941,-0.01459,0.02661,0.0293,-0.00824,0.03589,0.02551,0.0188,-0.02673,-0.03113,-0.00598,-0.00916,0.00141,-0.02283,0.01941,-0.00516,-0.04053,-0.00885,-0.02576,-0.06055,-0.01587,-0.00671,0.01868,0.01965,0.01904,0.02173,-0.03345,0.013,-0.02295,-0.00173,-0.02368,-0.03149,-0.02893,-0.00983,-0.03467,0.02893,0.02295,0.01483,0.01562,0.01373,0.00058,-0.02673,0.00331,-0.02234,-0.04565,0.00952,0.0238,0.0141,-0.00562,0.01917,-0.00381,-0.00369,0.01013,0.03687,-0.01807,-0.0094,0.01562,0.0141,-0.04321,-0.03564,0.00977,0.00394,0.00047,-0.0127,0.03418,-0.02515,-0.01648,0.01343,0.00085,0.01501,0.021,-0.01648,0.03149,-0.01221,-0.04639,-0.01733,0.02356,-0.04004,-0.01343,0.01154,-0.00842,0.02283,0.06641,-0.01562,-0.02832,0.0083,-0.03064,0.00766,-0.01453,0.01416,0.01807,-0.01129,-0.021,0.03589,-0.00983,-0.02686,0.021,-0.0062,-0.02673,-0.00903,0.00113,0.13574,-0.03882,-0.0332,0.0481,-0.03247,0.0415,-0.00873,-0.02881,0.02356,0.03003,0.0097,-0.02649,0.00616,-0.0007,-0.00476,-0.07227,0.00879,0.01941,0.02307,0.41602,-0.01263,-0.04468,0.01556,-0.03369,-0.02917,0.03809,-0.01953,0.00903,-0.03223,0.05933,-0.05054,-0.02173,0.0009,0.02063,-0.03687,-0.00144,-0.02454,-0.03516,0.00291,-0.02368,0.01105,-0.00021,0.00504,-0.00879,0.01917,0.04346,0.00861,-0.01648,0.03613,0.00613,-0.0152,-0.021,0.04004,0.0354,-0.0017,-0.03687,-0.03149,-0.0437,-0.00433,0.03857,0.00534,-0.00958,-0.0013,0.02014,-0.02429,0.01422,-0.00307,0.00123,-0.01758,-0.01093,-0.04712,-0.02722,0.02783,-0.0076,-0.03833,0.00653,0.00439,-0.02844,-0.04077,-0.02649,0.05664,-0.05396,-0.04175,-0.03882,-0.02551,-0.01117,0.01184,-0.03931,0.02307,0.02832,-0.00412,-0.00775,0.01416,0.0025,-0.02026,-0.02661,-0.00446,0.00389,0.06006,4e-05,-0.01331,-0.03857,0.0293,-0.01929,-0.02173,-0.01031,0.02075,0.00696,0.03882,-0.02161,0.02014,-0.05566,0.02612,-0.04443,0.02686,0.00952,0.03662,0.0332,0.00922,0.03516,0.01709,-0.01483,0.03735,-0.00482,-0.04346,0.00543,0.01624,0.01196,0.0094,-0.01697,0.01758,-0.03564,0.05444,-0.0354,-0.00806,0.01917,0.02173,0.00504,-0.07715,-0.01312,0.02734,0.0332,-0.04053,0.03345,0.03613,-0.00504,-0.00696,-0.01123,0.0166,-0.05249,-0.02478,-0.00182,-0.008,0.05078,-0.03516,0.04932,-0.03882,0.00546,0.00635,-0.04785,0.02332,-0.05688,-0.00616,-0.0061,-0.01495,-0.00824,-0.03662,-0.052,-0.02844,0.03931,0.01685,0.02478,-0.01819,-0.04663,0.00143,-0.04712,0.03113,0.00854,0.02258,-0.02087,0.05762,0.01575,-0.0076,0.03491,-0.05615,-0.04126,0.01459,0.05469,-0.02441,-0.02161,-0.00842,-0.03662,-0.03369,0.02173,0.00328,0.00187,-0.01373,-0.02075,-0.00626,0.00164,-0.00158,-0.02368,-0.03442,-0.00388,-0.01538,-0.02429,-0.05005,0.0376,0.00632,-0.03735,-0.02844,-0.00276,-0.02222,0.02283,-0.0166,-0.00074,0.0043,0.01343,-0.0004,-0.01587,0.05103,0.02161,-0.01263,-0.0108,0.02905,0.00848,0.01575,-0.00885,0.02917,-0.00757,0.02234,0.02087,-0.02026,-0.03857,0.03296,0.0152,0.02368,0.02014,-0.01276,0.00288,0.00867,-0.00083,0.04858,-0.00723,-0.02148,0.00197,0.02405,0.01855,0.00317,-0.02234,-0.00854,-0.00334,-0.01324,-0.02039,0.05103,-0.02063,0.00385,-0.01221,-0.01587,-0.04224,-0.0152,-0.01514,0.01416,0.00272,-0.00177,0.01343,0.00421,-0.01648,-0.01013,-0.00653,-0.02844,0.00662,-0.00366,-0.01831,-0.02759,0.0024,-0.03442,0.01721,-0.00301,0.01538,0.00064,-0.03296,0.01758,0.01129,-0.00836,-0.03589,-0.00433,0.01373,-0.02832,0.00211,-0.00055,-0.0155,-0.00702,0.0376,0.00086,0.00479,-0.04004,-0.00574,-0.02844,-0.04224,0.02539,0.0304,-0.01257,0.02356,-0.04858,-0.013,-0.03906,-0.01532,-0.0076,-0.00577,0.02502,-0.01587,-0.02722,-0.0105,-0.00964,0.00879,-0.01123,-0.03198,0.00668,-0.04346,-0.03369,-0.04297,0.00439,-0.00272,0.03809,0.02087,-0.00476,-0.02576,0.02283,0.01294,0.03149,-0.03149,0.01868,-0.00934,0.01215,0.02661,0.02087,0.01105,0.00461,-0.01367,-0.01556,-0.00519,0.03979,-0.0238,-0.02722,-0.02808,-0.01056,-0.04785,-0.02454,-5e-05,0.00546,0.01147,0.02039,-0.00037,0.00452,-0.01831,0.00372,-0.01624,0.00378,-0.00013,0.03906,0.01337,-0.01031],[-0.00702,-0.00351,-0.01978,-0.0415,-0.00403,-0.03064,-0.01019,-0.02258,0.00726,0.1084,-0.03223,-0.03418,-0.01892,-0.00562,-0.02283,0.0072,-0.04614,0.0282,-0.00166,0.00677,0.00534,-0.02234,-0.02014,-0.04907,-0.04028,0.00094,-0.02075,-0.03113,-0.00365,0.01868,0.01697,0.01733,0.00836,0.00601,0.04663,-0.03516,-0.00485,-0.00793,0.02258,-0.0061,0.01306,0.05078,-0.04346,-0.00842,-0.01892,-0.00946,0.02087,-0.00033,-0.02673,0.04102,0.01434,-0.01129,0.0047,-0.01611,0.01477,0.00873,0.00995,0.0238,0.01117,0.01154,-0.05591,0.04688,-0.00726,-0.00616,-0.00647,0.03711,0.05005,-0.01611,-0.00952,0.04565,0.01178,-0.03711,-0.00298,-0.02454,-0.00467,0.00156,-0.0564,-0.03833,-0.0437,0.0072,-0.00494,-0.0199,-0.02588,0.03467,0.01178,-0.03113,-0.00583,-0.02136,-0.00677,-0.0166,-0.03113,-0.00842,0.02539,0.0127,-0.01117,0.00018,0.02832,-0.02393,0.01483,0.00702,0.0022,-0.00011,-0.01562,0.01508,0.05225,-0.00916,-0.00812,0.00995,0.04346,0.00035,-0.01038,-0.01227,-0.04175,0.00467,0.01404,0.02454,-0.01404,-0.00032,-0.01324,0.01294,-0.01154,-0.04663,0.01373,0.01648,0.03296,0.03711,0.0032,0.01636,0.01685,-0.02808,-0.00623,-0.02698,0.02722,-0.00334,-0.01404,-0.03223,-0.00772,0.03418,-0.06836,0.01648,-0.00173,0.0152,-0.0119,-0.01166,-0.03198,-0.02673,-0.00504,-0.01129,-0.08154,-0.01349,-0.00128,-0.01636,-0.01074,0.00179,-0.00885,-0.01129,0.01233,-0.03833,-0.01007,0.00121,-0.02563,-0.05493,-0.02136,-0.04688,-0.00772,-0.01758,-0.00708,-0.00334,-0.00467,-0.03088,0.00041,-0.06445,-0.00024,-0.02795,-0.04565,0.00916,-0.02307,-0.0304,0.03027,-0.02014,0.03516,0.02832,0.00662,0.01819,0.00607,0.00167,-0.00458,0.01166,-0.0166,-0.05273,-0.01172,-0.00266,0.00371,-0.00397,0.03833,-0.01099,0.01501,-0.03027,-0.02075,-0.00325,-0.00586,-0.00601,-0.05396,0.00497,0.03442,-0.02087,-0.02856,-0.01416,-0.01276,-0.02502,-0.00317,-0.02771,-0.00331,-0.31641,0.0498,0.04883,-0.31055,-0.00397,-0.00964,-0.00346,-0.04688,-0.00159,0.03442,-0.04346,0.03076,-0.02954,0.00156,-0.00025,0.00989,-0.03198,0.02014,-0.04688,-0.01221,0.03638,-0.01337,-0.05005,0.00708,0.00589,-0.01733,0.0282,-0.01941,-0.01025,-0.00252,0.03491,-0.02869,-0.08789,-0.01733,-0.01929,-0.01733,-0.02405,-0.03125,-0.01251,-0.29883,-0.00482,-0.01251,0.05908,0.01007,-0.026,0.02979,-0.01843,-0.01685,-0.06152,-0.01178,0.02454,-0.01709,0.00531,0.02344,-0.026,-0.00217,0.01709,0.01892,0.03076,0.01038,0.01965,-0.00519,-0.00513,-0.00928,0.00571,-0.01062,-0.02588,-0.00333,-0.01483,0.01514,-0.05005,-0.0238,-0.04395,-0.00357,0.04517,-0.04883,0.05029,-0.0097,0.05444,0.01178,-0.01227,0.02222,0.00193,-0.00903,-0.01324,0.02502,-0.04785,0.00867,0.00232,0.00769,-0.05347,-0.01624,-0.0038,0.0032,0.0282,0.01611,0.06592,0.02747,0.00653,0.00046,-0.0271,-0.02014,0.02454,-0.01483,-0.04395,0.01093,0.00522,-0.04688,-0.00147,-0.00412,0.00052,-0.02039,-0.03931,-0.00818,0.0354,0.00772,0.02429,-0.02954,-0.00351,-0.02637,0.00099,-0.01093,-0.00455,-0.03369,-0.03589,-0.05298,0.03516,0.05347,0.05029,-0.00638,0.00928,-0.03223,-0.02905,0.0481,-0.02307,-0.00403,-0.03369,0.02441,0.01636,0.02014,0.0293,-0.02661,-0.01373,0.026,0.06885,-0.0166,-0.03784,0.04565,0.03516,-0.03955,0.00078,0.03564,-0.02869,-0.02002,0.00668,0.0271,-0.04077,-0.00319,-0.03174,0.01453,-0.02454,-0.021,-0.03149,-0.00066,0.00439,-0.03882,0.00491,0.03491,-0.00867,-0.0034,-0.02466,-0.03027,0.01178,0.02832,0.00377,0.00885,0.03882,-0.03027,-0.02441,0.00995,-0.00717,0.05103,-0.01294,0.0097,0.02795,0.00301,-0.03931,0.00182,-0.01672,0.00586,-0.00304,-0.00757,0.18652,-0.0625,-0.02222,0.03638,-0.03931,0.01831,0.03296,0.00325,-0.0332,0.04248,-0.01587,-0.03198,-0.00491,-0.01819,0.02307,-0.06055,-0.01459,0.03198,0.00191,0.35156,0.01575,-0.00013,0.01709,-0.03027,-0.02307,0.02136,-0.02173,0.00861,-5e-05,0.06494,-0.07031,-0.01434,0.01434,-0.01575,-0.05664,-0.00473,-9e-05,-0.05005,-0.00607,-0.01599,0.01355,0.02222,-0.00739,0.01892,-0.05176,0.04346,0.01843,-0.03149,-0.04932,0.03687,-0.03955,0.00531,-0.01276,0.05396,0.02454,-3e-05,-0.06396,-0.00772,-0.02637,0.0249,-0.00012,-0.00331,0.00412,0.02722,-0.05615,0.00522,0.01251,0.00092,-0.00327,0.01953,-0.0376,-0.04297,0.0304,0.0304,-0.00861,-0.00397,0.04712,-0.04565,-0.0481,-0.04858,0.04053,-0.05664,-0.02649,-0.06299,0.0166,0.00172,0.01318,-0.02722,-0.01941,0.03345,-0.05127,-0.00133,0.04883,-0.00702,-0.02197,0.01807,0.01196,0.02393,0.04492,0.00812,0.05298,-0.00022,-0.0072,-0.03442,-0.03174,-0.04907,-0.00922,-0.03369,0.01746,-0.00595,-0.04663,-0.04614,0.03369,-0.05029,-0.00574,0.00037,0.00934,0.02185,-0.0166,0.0293,0.02087,-0.01062,0.00766,-0.00867,-0.00909,0.0459,0.013,-0.00359,0.04028,-0.01404,-0.0354,-0.00439,0.03113,-0.0304,-0.00934,0.00082,0.0293,-0.00149,-0.04395,-0.01477,0.01575,0.05176,-0.02429,0.013,0.03882,-0.00247,-0.01221,-0.02051,-0.00674,-0.01129,0.02832,-0.00033,0.02722,0.0119,-0.005,0.02661,-0.06128,0.00635,0.02612,-0.00439,0.0415,-0.06641,-0.02771,0.04077,0.01215,-0.00681,-0.00391,-0.0271,-0.00766,0.01892,-0.02026,0.00485,0.0304,-0.004,-0.00074,-0.01807,0.01892,-0.03589,0.0166,-0.00436,0.00192,0.00946,-0.00084,0.0069,-0.01355,-0.01123,-0.02502,0.05127,-0.03882,0.00497,0.02832,-0.01782,-0.03491,0.03882,-0.0293,0.01172,-0.021,-0.052,-0.03076,0.03491,-0.00126,-0.04688,-0.04028,0.01093,-0.03271,-0.00169,-0.00327,0.02832,-0.03711,-0.03027,-0.04004,-0.05591,0.02649,-0.02283,-0.00897,0.03247,-0.00377,0.01575,-0.02026,-0.02405,0.02332,0.01636,-0.01111,-0.02283,0.0069,0.00167,0.0069,0.00237,0.04883,0.00739,0.01459,0.01929,0.00153,-0.02832,0.03784,0.03638,0.03015,-0.01459,-0.02441,0.02246,0.03735,-0.03516,0.00861,0.00094,-0.03979,0.02356,-0.00589,-0.00339,0.01807,-0.04614,-0.0199,-0.02344,-0.03662,-0.00787,0.06299,0.01245,-0.01538,-0.02356,-0.04199,-0.01758,0.00641,-0.04688,0.02795,0.01843,0.04175,0.02197,0.00574,-0.01501,-0.02344,-0.02661,-0.01245,0.00056,-0.0105,-0.02234,0.00519,0.04614,0.02148,0.00479,-0.00781,-0.0498,0.00928,0.00574,0.01831,-0.01196,-0.02588,-0.02283,0.00327,-0.01624,0.00243,0.02087,-0.00879,0.03125,-0.00739,0.00928,-0.0332,-0.01746,0.01379,-0.01007,-0.04907,-0.01544,0.00198,0.00934,-0.02795,0.01544,-0.02881,0.00479,-0.00714,-0.04272,-0.02661,0.00854,-0.01807,-0.03931,-0.04492,-0.01892,0.00025,0.01965,-0.0238,-0.00077,0.05127,-0.00201,0.0072,0.00586,-0.00952,0.00141,0.03784,0.00427,0.02539,-0.01147,0.02563,0.02136,-0.00891,-0.00598,0.00635,-0.03174,0.013,-0.03223,0.03088,0.01514,0.02747,-0.02673,-0.02002,0.01025,0.02014,0.03064,-0.02307,-0.0564,-0.01843,-0.01129,-0.00638,-0.00337,0.02124,-0.03076,0.02039,-0.01324,-0.06689,-0.01685,0.00665,-0.01855,-0.03467,-0.00705,0.03418,0.03198,-0.00394]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."]},"ibm-granite/granite-embedding-278m-multilingual":{"embeddings":[[-0.03931,0.04541,0.01062,0.01904,0.0752,-0.04492,0.07568,0.05591,0.05127,0.03247,0.00861,0.00191,0.02673,0.02856,0.01685,0.08447,0.07861,-0.06934,-0.03296,-0.0177,-0.01111,0.02661,-0.01257,0.05908,-0.0094,-0.05591,-0.01782,-0.01245,-0.03979,0.02466,0.00394,0.0304,-0.01978,0.03662,-0.00313,-0.00684,0.02551,0.03345,-0.02734,-0.00039,-0.01965,0.03394,0.01599,0.01093,-0.01379,0.0155,-0.0177,0.04004,0.00458,0.03247,-0.05396,0.0141,0.00218,-0.00365,-0.04272,0.01526,0.06445,0.06592,-0.06445,0.0354,-0.0188,-3e-05,0.01685,-0.00327,0.02942,0.00763,0.06592,-0.06079,0.01337,0.05103,-0.00173,0.00668,0.05518,0.07178,-0.03955,0.00354,-0.02856,0.02319,0.00879,-0.00638,-0.04321,0.03589,-0.01709,0.01917,-0.03394,0.0155,0.02576,0.03076,0.05396,-0.03345,0.03015,-0.01685,-0.04346,-0.00226,-0.00313,-0.00365,0.01746,0.01538,0.05762,-0.03613,0.02759,-0.00052,0.00188,0.01361,0.02246,-0.00565,0.03345,-0.04663,0.0238,-0.0116,0.03979,-0.02979,0.05396,0.0304,0.03589,0.01172,0.01685,0.00069,0.05762,0.02051,0.07764,-0.02832,0.04907,-0.00885,0.02124,0.021,-0.0238,0.04175,0.03931,-0.01068,-0.00977,-0.08154,-0.02917,0.02222,-0.06348,-0.02942,-0.00644,0.01135,0.01019,0.00842,-0.013,-0.01782,-0.02271,0.04126,-0.01013,-0.03638,-0.02393,0.04199,0.03906,0.02051,-0.02917,-0.00885,0.05396,0.03711,-0.05469,0.03809,-0.01843,-0.00668,-0.01062,0.01318,0.0376,-0.008,0.00897,0.021,0.00659,-0.02832,-0.00056,0.05786,0.0141,-0.00209,-0.01831,0.02014,0.00824,-0.04053,-0.02722,0.04224,-0.00195,0.00729,-0.00964,-0.01025,0.02026,-0.02527,-0.11084,0.0354,0.0038,-0.00337,0.0006,-0.04834,-0.00775,0.03857,0.01556,0.00319,0.00287,0.15527,0.0791,-0.03394,-0.05225,0.00235,-0.08691,0.00616,0.0069,0.05322,0.02246,-0.0332,0.00159,0.021,0.05908,0.06006,0.01538,0.04858,-0.02271,0.03247,0.03711,0.05518,0.01599,-0.03101,0.2041,-0.00069,0.00107,-0.06079,0.04541,0.04053,0.01398,0.0199,0.00068,0.02515,0.01941,0.04175,0.05322,0.02161,0.05273,0.04102,-0.02893,-0.05957,0.07617,0.00308,0.02405,0.0332,0.00735,-0.03979,-0.02502,-0.02917,0.0119,0.02734,0.01202,0.0498,0.04224,-0.0437,0.02698,-0.0238,-0.00754,0.00885,0.02527,0.06104,0.07471,0.02234,0.01611,-0.01929,0.06592,0.05664,0.07422,-0.00182,0.00946,-0.00589,0.0376,0.0282,-0.02368,-0.03931,0.04248,0.06641,0.00243,0.06543,0.00934,-0.00916,0.05396,-0.00684,-0.01953,-0.03149,0.01733,0.01282,0.02979,-0.03003,0.0083,0.0304,0.00243,0.07227,0.0188,-0.03662,0.03113,0.00375,-0.02283,0.03369,0.04419,-0.00778,-0.03003,0.02612,0.0415,0.02881,0.02917,0.00342,-0.00127,0.10449,0.04443,-0.00446,0.02539,0.02222,-0.01508,-0.02771,0.0199,-0.01471,0.03345,-0.00365,0.0188,0.05273,-0.01965,0.05078,0.0282,-0.03149,-0.01477,-0.00818,-0.03589,0.03662,0.052,0.0097,0.00285,-0.05469,-0.00256,0.00909,0.00604,0.01208,0.02051,0.01624,0.01349,-0.00211,0.02332,-0.04053,-0.01855,0.12793,-0.04053,0.0072,0.00218,0.07471,-0.00623,-0.00934,0.0108,0.02136,0.0498,-0.03662,0.03198,0.00061,-0.01489,0.00665,-0.0108,0.02832,-0.00221,-0.03198,0.04565,-0.02979,0.00497,0.03467,0.03613,0.04712,-0.00842,-0.01361,0.026,0.01501,0.01471,-0.01538,-0.0304,0.03149,-0.03247,-0.02148,-0.05518,0.02295,-0.00885,0.03101,0.05664,0.01965,0.00787,-0.03015,0.00464,0.01538,0.01556,-0.03711,0.03809,-0.01471,0.00653,-0.01428,-0.07617,0.0141,0.06445,0.0603,-0.00149,-0.00958,0.01019,0.04565,-0.02771,0.02637,0.02417,0.0004,0.04272,-0.03589,0.0022,0.03735,0.04932,-0.02051,0.0097,-0.03271,-0.02283,0.01416,0.03589,0.0354,-0.00287,0.03589,-0.00583,0.03174,-0.02295,-0.01367,0.03271,0.02661,0.03589,-0.02551,0.04663,0.02734,-0.01709,0.03809,0.06396,0.01489,0.00464,0.03955,0.02234,0.0752,0.0062,0.01385,0.00571,0.0031,0.03906,-0.021,0.03491,0.00681,-0.008,0.02576,0.04761,-0.02673,0.02393,-0.04028,0.013,-0.00497,0.01599,0.08984,0.026,-0.0603,-0.0144,-0.02979,0.05322,0.06592,0.08643,0.07422,0.00705,0.01379,0.00452,-0.02832,0.00589,0.0498,0.05542,0.03198,0.04199,-0.01007,-0.02063,0.05884,0.04639,0.0332,0.0188,0.00169,0.03149,0.00818,0.0459,-0.05664,0.0047,-0.0177,-0.01508,0.03052,0.0564,-0.0015,0.01508,0.01831,0.00019,0.02576,0.00699,0.02295,0.0282,-0.04077,-0.00443,-0.01489,-0.02502,0.02112,-0.03418,-0.13672,-0.01276,0.02051,-0.00543,-0.00522,-0.04688,-0.00909,0.00525,-0.05225,0.03198,0.02148,-0.00368,0.03296,-0.09619,0.00305,0.01196,-0.0459,0.02942,0.02612,-0.00235,-0.01398,0.01978,0.05591,0.05396,0.01025,-0.04395,0.0083,0.03247,0.052,-0.02551,0.005,0.02661,-0.01398,0.02527,-0.0376,0.01501,0.026,0.00491,0.09277,0.00455,0.02771,0.05103,-0.04053,0.00592,-0.01263,-0.00909,0.02698,1e-05,-0.01599,0.06396,0.04761,-0.02576,-0.01245,0.00595,0.11426,-0.04004,0.02161,-0.00537,-0.01709,0.05713,-0.00121,0.01135,-0.04297,-0.01044,-0.02527,0.04907,0.07324,0.01929,0.00562,0.01929,0.021,0.03589,-0.02271,-0.00516,0.02551,0.03711,-0.00494,0.04883,-0.0155,0.04517,-0.00653,-0.02893,0.0354,0.03491,0.03638,-0.00934,0.03589,0.01105,-0.02527,-0.02417,-0.00806,-0.0376,-0.03345,-0.00873,0.04541,-0.03296,-0.04492,-0.06543,0.02563,-0.04004,0.00909,-0.04199,0.01459,-0.00311,-0.0177,-0.04248,0.03015,-0.05103,0.01385,-0.04639,-0.01416,0.02478,-0.0238,-0.02954,0.03174,-0.00244,-0.00644,-0.00256,0.03857,0.02283,-0.06006,-0.00885,-0.01349,0.01398,0.013,0.02197,-0.02612,-0.01166,0.00668,0.0304,0.01416,0.01074,0.05713,-0.00699,0.01202,-0.05713,0.03052,-0.02283,-0.01733,-0.02246,0.02161,-0.02136,-0.00552,-0.00436,0.00215,0.02673,0.03955,0.01575,0.00528,0.06201,0.0199,-0.05127,0.00182,0.01733,0.01978,0.05322,-0.0481,0.03662,-0.00769,-0.00824,0.00048,0.05273,-0.01379,-0.00861,-0.00249,0.00362,0.02234,0.01819,-0.00266,0.01013,0.05762,-0.0304,-0.05347,0.01367,0.04077,0.03149,0.06299,-0.01917,0.0069,0.02185,0.04199,0.0354,0.01965,-0.02478,-0.03418,-0.04834,0.00958,-0.00885,0.02124,0.03247,-0.0188,-0.02942,0.0354,0.0708,0.01794,0.00589,-0.01794,-0.00308,0.05518,-0.03015,0.00226,0.03516,-0.03613,-0.02393,0.04663,0.04443,-0.03223,-0.02881,-0.04712,0.05713,0.05469,0.0332,0.06348,-0.02173,-0.06226,0.01556,0.06885,0.02637,0.01428,-0.00071,0.05518,0.0708,0.0058,0.0282,-0.01459,0.01379,0.03784,-0.0083,-0.08252,-0.0354,-0.02209,0.06689,0.0603,-0.00674,-0.05103,0.00873,-0.00388,-0.14941,0.01385,0.00693,-0.00595,-0.01953,-0.00156,0.02429,-0.04614,0.02063,0.0152,0.02368,0.01709,0.02209,-0.03442,0.01953,-0.04199,0.06079,0.0065,0.00337,0.05127,0.0708,0.02405,-0.06348,0.01709,0.00568,-0.05078,0.04468,-0.01636,0.01282,0.02612,-0.0072,0.0177,0.02441,0.02283],[-0.03906,0.04712,-0.02917,-0.00665,0.02295,-0.06201,0.04785,-0.01129,0.05347,0.03174,0.01471,-0.01599,0.02112,-0.02673,-0.01453,-0.00665,0.03442,0.00647,-0.01086,0.00038,-0.0188,0.02795,-0.01459,0.01929,0.0481,0.00223,0.02454,-0.00714,-0.00243,0.00333,0.03125,0.00136,0.00903,0.02014,0.05127,-0.03955,-0.03735,0.02026,-0.05591,0.02148,0.02405,-0.0097,-0.01831,0.06641,-0.08643,0.06177,-0.01892,0.00473,0.01575,0.04614,0.00276,0.03345,0.0141,-0.02979,-0.02002,0.03638,0.0332,0.02466,-0.03345,0.00226,-0.07471,-0.04053,0.02197,-0.02356,0.04199,0.03564,0.07031,0.00278,0.02844,-0.03857,0.01337,-0.00629,0.04053,0.05957,-0.01831,0.0199,-0.04175,0.04639,-0.06396,-0.02026,-0.01562,0.00446,0.03491,-0.00952,-0.01086,0.06079,-0.00415,-0.03516,0.02405,0.00705,0.01868,-0.01367,0.008,0.04102,0.03931,-0.0332,-0.04199,0.05078,0.01575,-0.00745,0.02734,0.00191,0.0199,0.03101,0.0043,-0.00958,0.05713,-0.02185,0.02197,-0.00301,0.03516,0.00537,-0.02234,-0.00577,-0.04077,0.03369,-0.02759,-0.02588,0.03711,0.03687,0.01294,0.0199,0.04761,-0.04395,0.04492,0.01331,0.05054,0.0108,0.03638,-0.0332,-0.06641,-0.03979,-0.03809,-0.00446,-0.00616,-0.0069,-0.01056,0.04614,0.00928,0.02429,0.03369,-0.02429,-0.04468,0.0127,0.02551,-0.00595,0.07373,0.02783,0.04297,0.07617,0.02795,-0.00513,-0.01672,-0.0105,-0.08057,-0.0033,-0.00033,0.0481,-0.03662,0.04932,0.03906,-0.06885,0.05029,0.0105,-0.01135,0.06299,0.05273,0.02441,-0.01392,0.026,0.04858,0.06006,0.02734,-0.09521,0.03149,0.01294,0.02942,0.07031,-0.0199,-0.02112,0.01855,0.01532,-0.03589,0.09326,0.06299,0.09131,0.02881,-0.01025,0.02161,0.0097,0.0304,0.05176,-0.01172,-0.00751,-0.01483,0.00659,-0.04248,0.02466,-0.026,-0.05054,0.01648,-0.04199,0.01453,-0.03088,0.02722,0.01544,-0.0155,0.03149,-0.00296,0.04321,-0.00169,0.01398,-0.00668,0.0354,-0.01379,0.01746,0.07275,0.01721,0.06738,-0.0752,0.01257,0.03418,0.02161,0.01746,-0.03003,-0.02551,-0.0006,-0.01685,0.0013,-0.02905,0.02844,0.05444,-0.01196,-0.04736,0.07617,-0.01013,0.02515,0.01068,-0.02271,0.00507,-0.06885,-0.02527,-0.00482,-0.01917,0.04663,0.02441,-0.03064,-0.05444,0.03613,-0.05273,0.0332,0.02759,0.08594,0.0034,0.04297,-0.09863,0.00256,0.02014,0.03223,-0.00253,0.03662,0.03516,0.03247,-0.02222,-0.02478,-0.0072,-0.01831,0.01697,-0.0238,0.08398,-0.00415,-0.02734,-0.03125,0.00836,0.06982,0.0332,0.01172,0.00525,0.00577,0.00861,0.02539,0.00473,-0.03369,0.0354,0.00626,0.02856,0.06201,0.00549,0.05029,0.00726,0.05835,0.04297,-0.02356,-0.02905,-0.0007,-0.00537,-0.01141,-0.00632,0.02759,-0.00375,0.0282,0.04224,-0.03809,-0.05273,0.04883,0.04199,-0.01843,-0.00121,0.02942,0.01483,0.00952,0.06836,0.00293,0.04224,-0.00903,0.00699,-0.04053,-0.02466,-0.05054,0.00793,-0.00342,-0.02612,0.04224,-0.01868,-0.03088,-0.01892,-0.00089,0.05103,0.04224,-0.08398,-0.0083,0.05469,-0.02417,0.03979,-0.03027,-0.04517,-0.01965,0.05127,-0.07031,-0.00146,-0.01056,-0.00035,0.00989,0.02161,-0.0199,0.0459,-0.04077,0.05444,0.03125,0.0481,0.06445,0.03149,-0.01007,0.02795,-0.00455,-0.03955,0.02576,-0.02698,-0.05054,-0.03442,-0.00946,0.04102,-0.01929,0.08105,0.04932,0.05444,0.02515,-0.04004,-0.0238,0.0144,0.03662,0.05151,-0.01624,0.01599,0.04907,-0.03088,0.01758,0.01221,0.0835,-0.0437,0.03149,0.00983,0.01917,-0.02979,-0.02722,0.01672,0.02783,-0.02051,0.0238,-0.0564,0.01306,0.06079,0.05566,0.0481,0.05273,0.05762,-0.00705,-0.00072,0.00275,-0.00308,0.02051,0.04077,0.07861,0.02698,-0.01562,-0.06055,0.04932,-0.03064,-0.0437,0.07617,0.02942,0.03882,-0.00415,-0.02856,-0.05713,0.03516,-0.02148,-0.0009,0.01428,0.0029,0.05127,0.02258,-0.01575,-0.02161,-0.00604,0.02539,0.01459,-0.00111,0.01471,0.03931,-0.02722,0.04297,0.0459,0.01361,0.02295,0.04932,-0.00632,0.0437,-0.00284,0.01361,0.06885,0.00751,0.00766,0.01172,0.05396,-0.02002,-0.03345,-0.02039,0.02271,-0.00446,-0.00647,-0.08594,0.01483,0.01367,0.06787,0.00604,0.07227,-0.01331,0.01398,-0.05713,-0.01483,0.026,-0.00246,0.03491,-0.01904,-0.002,-0.00182,-0.0166,0.06885,-0.00861,0.0152,0.021,0.02734,-0.01331,-0.05957,0.06299,0.01599,0.00699,-0.04541,-0.03174,-0.06445,0.02332,-0.01978,-0.01294,0.05444,0.01367,0.01062,0.0001,0.02197,0.01965,0.02441,-0.05054,0.04468,-0.01331,-0.02759,0.03491,-0.01636,-0.08887,-0.03442,-0.00375,0.04761,-0.00604,0.0105,-0.01697,0.02466,-0.01068,0.03223,0.03088,0.00147,-0.03101,-0.04248,-0.02637,0.05396,0.01758,0.02551,0.00074,-0.01379,-0.04199,0.01733,0.04639,0.02612,-0.00028,0.00635,-0.04321,0.00014,0.03088,0.06543,0.0437,-0.02271,-0.03662,0.04102,0.0076,0.06592,0.00172,-0.00873,0.11035,0.01038,0.02539,0.05054,0.02795,0.008,-0.02405,0.00372,0.02966,-0.04688,-0.06396,0.00488,0.06006,-0.01855,0.00313,-0.00299,0.03613,-0.03687,-0.01544,-0.00977,0.03003,0.01263,0.03271,0.03955,0.01831,0.00494,-0.02515,0.04346,0.02942,0.01807,0.05566,-0.04883,0.01007,0.03027,-0.04126,0.07275,0.01843,-0.03442,0.06885,-0.00989,-0.04175,0.04736,0.02612,-4e-05,-0.0119,0.06006,0.01184,0.04175,0.00555,0.02722,0.00155,-0.08398,0.01379,0.00046,-0.01904,0.12109,0.03247,-0.06055,0.00745,-0.01007,0.07617,-0.03223,-0.04468,-0.0332,0.03271,-0.00087,0.03882,-0.02637,0.05957,0.008,-0.00638,-0.01392,-0.05566,-0.03125,-0.01685,-0.03003,0.00357,0.0249,0.03394,0.02942,-0.00854,0.05005,0.03198,0.03442,0.0177,0.06128,-0.02173,0.09277,-0.0415,-0.0625,-0.04858,-0.01782,0.01611,0.01245,-0.03711,0.03442,-0.01794,-0.01782,0.01501,-0.00879,0.04443,-0.00013,0.0332,-0.07764,0.03149,0.02002,0.00684,0.03833,0.01233,0.05396,0.02405,0.04956,0.01965,0.00772,0.05811,-0.02466,-0.04443,0.04956,-0.02332,-0.05566,0.00012,-0.02039,-0.01501,0.04443,-0.0249,0.05566,0.01965,0.05957,-0.0282,-0.00836,0.00653,0.0304,0.05591,-0.0144,0.0001,0.03687,0.01471,0.03564,0.00552,0.02356,0.04492,0.03394,-0.00043,-0.01501,0.05396,0.0304,0.04321,0.01166,0.01483,0.05566,0.02185,0.02014,0.0199,-0.01746,0.0166,0.04419,0.00221,0.02405,-0.00696,0.04419,-0.01965,-0.02161,0.01471,-0.00238,0.00218,0.03589,0.10791,0.05469,-0.06177,0.04565,-0.02283,-0.01001,0.02026,0.06641,0.04443,-0.0304,-0.01807,0.03687,0.02588,0.02637,0.0791,0.05835,0.02234,0.04419,-0.00995,0.0007,-0.01483,-0.04053,-0.00427,0.02917,-0.04077,-0.02673,-0.02966,-0.02917,0.06592,0.04639,-0.02405,0.04468,0.00818,-0.03101,0.05811,0.02271,0.02222,0.01398,-0.01062,0.0054,-0.0127,0.03198,-0.05078,-0.01208,0.06641,0.01599,0.00714,0.00476,0.01031,0.10791,-0.02222,0.01721,0.03931,-0.00266,0.00516,-0.06201,0.0625,-0.01294,-0.01782,0.04736,-0.00127,0.06348,0.02856,0.0332,0.03882,0.02124,0.00854]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."]},"intfloat/multilingual-e5-large":{"embeddings":[[0.00615,0.00167,-0.01884,-0.03221,0.02153,-0.02805,-0.03668,0.03462,0.05085,-0.03967,0.02994,0.02673,-0.01631,-0.03023,-0.00784,-0.02785,-0.01527,0.01416,-0.0175,-0.0024,0.01564,-0.01382,-0.03142,-0.05291,-0.01336,-0.00906,-0.05366,-0.03229,0.00018,-0.02371,0.02445,0.0198,-0.057,-0.02765,-0.01388,0.04315,0.01792,0.04386,-0.04145,0.02259,-0.04027,0.06549,-0.00615,-0.04685,-0.02545,0.01882,-7e-05,0.00995,-0.00658,0.0094,0.01294,0.03544,-0.01205,-0.00679,-0.05014,0.02546,-0.03369,-0.00695,-0.06792,0.02104,-0.00758,-0.01068,0.02602,-0.00884,-0.05043,0.04765,0.04374,0.01041,-0.0627,0.01121,-0.01663,0.02877,-0.00805,-0.01111,-0.0205,-0.00844,0.01198,-0.00926,0.02134,-0.04331,0.06926,-0.00373,0.02077,-0.00268,0.04092,0.05955,0.05645,0.03653,0.04056,-0.0068,-0.02024,0.04116,0.04302,-0.03086,-0.06146,-0.0102,0.02745,0.01578,-0.0087,0.02171,-0.02511,0.01118,0.04607,-0.04068,-0.01473,0.02825,0.04167,0.03878,0.01572,0.01375,0.02937,0.06091,0.01171,-0.03897,-0.02686,-0.0017,-0.01511,-0.0206,-0.01555,0.02105,0.03059,0.05445,-0.00398,-0.02821,0.03217,-0.0198,0.00498,0.01906,-0.03535,-0.00161,0.01966,0.0339,-0.04975,-0.00636,-0.03221,-0.01732,-0.03673,0.01371,-0.0158,0.00554,0.00017,0.03149,0.03001,-0.03474,-0.00475,-0.03925,-0.06055,0.02426,0.0156,-0.02372,0.01525,-0.03527,-0.01349,0.01126,0.04015,-0.02439,-0.00961,0.02856,0.01881,-0.0098,-0.03581,-0.04898,-0.02589,-0.02458,0.00259,-0.01206,0.02079,0.03067,0.00878,0.01893,-0.0057,-0.01867,-0.00036,-0.06085,0.0105,0.03999,0.03616,0.01615,0.04167,-0.02559,0.00959,0.04495,0.03505,-0.04797,-0.0236,0.04523,0.01781,-0.00457,0.04794,0.03589,0.04838,-0.03042,-0.01687,0.00771,-0.02514,0.02846,0.01811,0.02622,-0.04418,-0.04573,-0.00331,0.01715,-0.04608,0.03864,-0.00672,-0.018,-0.03178,-0.03277,0.04316,-0.021,-0.02684,0.00573,0.01766,0.01733,-0.03753,-0.04431,-0.0218,-0.04106,0.02515,-0.02909,0.02671,0.04492,0.00073,0.03602,0.02872,0.02141,0.04402,0.02366,0.07388,0.04,0.01923,0.01851,-0.02748,-0.01402,0.00598,-0.02162,-0.02918,0.00522,0.03035,0.06531,-0.03152,0.02151,0.00102,-0.0669,-0.00033,-0.01244,0.03099,-0.02126,0.06486,-0.00309,0.01367,0.00166,-0.03883,-0.0096,0.02686,-0.01297,0.03665,-0.02066,-0.00011,-0.01652,0.01446,0.01598,-0.00921,0.05124,0.03684,0.02166,-0.00765,-0.04292,-0.0186,-0.03449,-0.02408,-0.04944,-0.06127,-0.01436,-0.02312,-0.00964,-0.04073,-0.02219,-0.01258,0.01835,-0.01879,-0.02844,0.01862,-0.03649,-0.022,-0.00534,-0.01025,-0.01194,0.04166,0.0023,0.01103,-0.01967,0.09679,0.00777,0.03473,0.03102,-0.05535,-0.08699,-0.03839,0.04657,-0.03918,-0.04668,0.02044,0.02931,-0.03045,-0.05196,0.01273,0.03193,-0.06546,-0.0224,0.0303,0.04325,0.01745,-0.0319,-0.05916,-0.00678,-0.0362,-0.01359,-0.03348,0.04909,-0.04637,-0.00442,-0.03869,0.00256,-0.0091,0.0323,0.00699,-0.03583,-0.00141,-0.03772,0.0364,-0.03694,-0.00036,0.05587,-0.03022,0.01914,-0.04939,0.03388,0.02687,-0.00381,0.02428,-0.00716,-0.05017,0.01245,0.01753,-0.01649,0.01085,-0.03297,0.02048,0.01497,0.00411,-0.03754,0.04071,0.00336,-0.02853,0.06207,-0.00445,-0.01898,0.02212,-0.01418,0.00869,0.02196,0.00822,-0.02099,-0.04342,-0.00806,-0.01612,0.02104,-0.04276,-0.04005,0.00753,-0.00583,-0.01166,0.00018,-0.00525,-0.01323,-0.08784,-0.01248,0.0087,-0.06877,-0.04399,-0.03193,0.01033,-0.03153,0.0083,-0.03041,0.15939,0.00369,0.01998,-0.05045,-0.01189,0.04456,0.0318,0.02134,0.04552,0.01934,-0.03649,-0.00998,0.02344,-0.0149,0.04619,0.05614,0.00407,0.03745,0.0283,0.00314,0.02897,-0.01966,0.05961,0.0096,-0.05195,-0.03497,0.00896,0.02183,-0.04654,0.0342,-0.01183,0.0266,-0.02103,-0.00451,0.02802,0.01168,0.02302,-0.06055,0.03326,0.0066,-0.03371,-0.00198,-0.03486,0.06223,-0.00114,0.03318,0.035,-0.01025,-0.033,0.03541,0.00551,0.01503,-0.02654,-0.03598,-0.02727,-0.02694,-0.0154,-0.06813,0.0187,0.01968,0.06567,0.02885,0.01509,-0.04656,0.02185,0.01838,0.00224,-0.05228,-0.03437,0.00359,-0.03251,-0.00873,-0.03066,0.02017,0.0496,-0.00101,-0.02414,0.04115,0.03096,0.0125,0.01225,-0.01952,-0.00629,-0.0021,0.05267,-0.03803,0.00979,0.00656,-0.02996,0.01079,0.02984,0.02911,-0.0254,0.03443,-0.00182,-0.0528,-0.00454,-0.01863,-0.02927,-0.01361,-0.02177,0.02903,-0.04105,-0.01436,-0.02122,-0.01872,0.00244,0.03449,0.02727,0.00095,0.02517,5e-05,0.04947,-0.00208,0.03659,0.0357,0.01634,-0.04529,0.0405,0.01845,-0.01805,0.02499,-0.02753,0.06908,-0.01798,0.0303,-0.02546,0.00353,0.04697,0.03999,0.02565,-0.03321,-0.02968,0.03114,0.0114,-0.02264,0.05427,-0.02309,-0.01277,0.04318,-0.01662,0.0375,-0.03253,0.03411,0.01434,0.02483,-0.04909,-0.04621,-0.02879,-0.03238,0.0281,-0.04326,0.02168,0.00135,-0.01895,-0.02044,-0.00199,0.00882,0.0467,-0.038,-0.01491,0.00713,0.02949,0.08934,0.03923,0.03082,-0.0556,0.03442,0.04656,0.01178,-0.01369,-0.0359,-0.00317,0.0165,0.03202,-0.02372,-0.02483,0.04296,-0.05164,0.0388,-0.00114,0.03858,0.01901,-0.05343,-0.02259,0.00472,0.01412,-0.03766,0.00163,0.0276,-0.01545,0.06877,-0.0334,-0.00281,-0.03326,-0.00625,0.03139,0.0168,-0.01824,-0.03339,0.00804,-0.03694,-0.00283,-0.0191,-0.03522,-0.01633,-0.00378,0.03659,0.00572,0.06713,0.01236,-0.06875,0.00218,0.03009,0.031,0.01589,0.03229,0.03717,0.04021,-0.02505,0.02784,0.02861,0.02398,-0.03976,-0.04332,-0.03079,0.07925,0.01692,-0.01335,-0.02087,0.0103,-0.02611,-0.00049,-0.0713,-0.02071,-0.01673,0.04088,-0.0074,0.01223,-0.02923,0.04267,-0.00196,0.02846,-0.01718,-0.0108,0.0252,-0.03355,-0.01167,0.02645,0.01847,-0.00764,-0.02294,0.03671,0.04711,-0.04833,-0.00658,0.01129,-0.01064,-0.04095,-0.05825,0.02147,0.02263,-0.03088,0.00999,-0.02612,-0.0002,0.02627,-0.05376,-0.00558,-0.04673,0.0439,-0.027,-0.00552,-0.05778,0.03202,0.04894,-0.02891,-0.03764,-0.02056,-0.02949,-0.03207,0.01705,0.00691,0.02757,0.01405,0.03422,-0.04516,0.05298,-0.0214,-0.03076,-0.02831,0.0001,-0.03295,-0.00082,-0.02642,-0.04646,-0.02201,-0.01685,-0.01091,-0.04717,0.02315,-0.01265,-0.04558,-0.06804,-0.01355,0.03169,-0.03092,-0.00136,0.03555,0.02648,-0.00366,-0.03205,0.0166,0.02467,0.03169,0.00783,0.03284,-0.02549,-0.00702,0.04325,-0.04913,-0.00615,-0.01033,-0.01693,0.06183,-0.00409,0.02603,0.00841,0.01077,0.00269,0.01264,0.00167,-0.02049,0.00447,-0.05125,-0.05859,-0.02217,-0.0244,-0.03446,-0.04205,-0.00539,0.03457,0.02077,-0.02666,0.03425,0.02147,-0.02431,0.01696,-0.02137,-0.01403,-0.0261,0.02934,-0.01227,-0.01623,-0.00687,0.03435,-0.03354,0.00529,0.01695,-0.04586,-0.03223,-0.02635,0.01901,-0.01786,-0.00221,0.03267,-0.01975,0.02599,-0.00909,-0.00334,-0.01975,0.04265,-0.0342,0.00266,-0.02881,0.02719,-0.01439,0.01482,0.05184,0.01778,0.03,-0.07998,-0.05054,-0.04743,0.05319,-0.02988,-0.02204,0.02126,-0.00235,0.03775,-0.04423,0.04813,-0.01801,0.03014,-0.08231,-0.01351,-0.00466,-0.04255,-0.00617,-0.00554,-0.02872,0.01829,-0.00609,-0.0465,0.01261,0.03835,0.02638,-0.01093,-0.00171,0.03271,0.03421,-0.01843,0.00153,0.00913,0.02476,-0.03444,0.00221,0.01112,0.03416,0.03274,-0.00648,0.02359,-0.03598,-0.05186,-0.01576,-0.04442,0.0109,0.03642,-0.03093,0.04372,-0.00046,0.01294,-0.00861,-0.00414,-0.0453,0.0214,-0.05431,-0.00383,-0.0212,0.0344,-0.0017,-0.03341,-0.00681,-0.03029,-0.02644,-0.03305,0.02823,0.05523,0.02451,0.03289,-0.00016,-0.04674,0.0203,0.01865,0.00989,-0.03898,-0.01892,-0.03093,-0.03522,-0.01432,-0.0524,-0.03018,-0.02161,0.02863,0.03719,-0.02986,0.02648,-0.00214,0.04029,-0.01852,-0.00938,-0.00501,0.03258,-0.00685,-0.01942,-0.0135,0.01719,0.02572,-0.00758,0.0167,0.00859,-0.01627,-0.05348,0.0355,-0.03085,-0.00022,0.03865,-0.05134,-0.03863,-0.01656,-0.02716,0.00337,-0.02018,0.04672,-0.03461,-0.00563,0.01592,-0.00138,0.02164,0.01698,0.0256,0.02619,0.02859,-0.01706,-0.03702,-0.02659,-0.01661,-0.0227,-0.00344,0.02523,-0.02006,0.0081,-0.01022,0.00937,-0.02498,-0.07168,0.03724,-0.0122,0.04004,-0.00867,0.00592,-0.04052,0.00315,-0.0341,0.01146,-0.07161,0.04147,0.01907,-0.03241,-0.01396,0.02179,0.06182,-0.04631,0.04859,-0.00359,-0.04783,-0.02793,-0.0342,-0.03299,0.03834,-0.02407,-0.02525,-0.00041,-0.0372,0.03359,0.00067,-0.03844,0.0479,0.01996,-0.00293,0.01322,-0.00658,0.02476,-0.00072,-0.00677,0.01804,-0.0576,0.03539,-0.01186,-0.01977,0.01781,0.00802,-0.03949,-0.01356,0.02769,0.03701,-0.04249,0.01506,0.0521,-0.01934,0.05178,0.03863,0.02595,0.03125,0.02879,0.01489,-0.00325,0.0311,0.0201,0.05376,0.0145,0.01407,-0.03218,0.01808,-0.02966,0.03742,-0.04484,0.04111,0.06778,-0.02128,-0.02059,0.00531,-0.02297,-0.00817,0.00135,-0.00885,0.01699,-0.03618,-0.00299,-0.02229,-0.05462,0.00601,-0.02324,0.03343,0.04094,-0.00466,0.01443,0.02148,0.03247,0.02243,0.03655,-0.0471,-0.01349,0.04415,0.0376,0.03211,-0.00811,0.01018,-0.04028,0.0433,-0.05766,0.02474,-0.04351,-0.02233,0.00075,0.01573,0.00695,0.01393,-0.01471,-0.05068,0.04218,0.02377,0.02173,0.01943,-0.02215,0.00109,-0.02419,0.02293],[0.03846,-0.0082,-0.00921,-0.02049,0.03584,-0.00922,0.00239,0.05344,0.05032,-0.03811,0.0011,0.03783,-0.06407,-0.04176,0.01019,-0.00751,-0.00255,-0.00436,0.00156,-0.03799,0.02109,-0.00684,0.01454,-0.01424,-0.03091,-0.04024,0.02576,-0.01973,-0.01787,-0.03813,-0.01958,-0.00714,-0.03606,-0.04513,-0.00805,0.02994,0.03516,-0.01577,-0.03248,0.01195,-0.02364,0.02526,-0.00402,-0.01722,0.00034,0.03871,0.04856,0.00153,-0.03923,0.00818,-0.0246,-0.00379,-0.00817,-0.0081,-0.01533,-0.00799,-0.00553,-0.0174,-0.028,-0.01267,-0.03008,-0.02038,-0.00394,-0.0224,-0.05102,0.03389,0.03188,-0.0121,-0.0522,0.03167,0.0167,0.02915,-0.0119,-0.03779,0.0002,-0.02102,0.02669,-0.01683,0.04762,0.01064,0.07822,-0.00896,-0.00347,-0.02252,0.03228,0.04546,0.01981,0.00847,0.04402,0.01891,-0.00772,0.04674,0.03865,-0.00523,-0.03102,0.00949,0.02012,0.02886,0.01917,-0.00809,-0.00408,0.02538,0.01319,-0.04884,-0.03445,0.06332,0.01959,0.03309,0.01632,0.02214,0.06101,0.03052,-0.01289,-0.01785,-0.00822,-0.01567,-0.0133,-0.01967,-0.00718,0.01587,0.05309,0.0174,0.01217,-0.02263,0.03331,-0.00693,0.01562,0.03049,0.0047,0.06342,0.03621,0.02228,-0.01601,-0.05516,-0.07028,-0.05228,0.00454,0.01021,-0.01607,0.03895,0.01881,0.01285,0.02272,-0.03872,-0.0324,-0.07229,-0.03093,0.02382,-0.01966,-0.03185,0.01974,-0.02843,-0.04072,0.01126,0.02838,-0.02311,0.02693,0.0554,0.01074,0.0093,-0.04094,-0.02215,-0.00925,-0.02206,-0.01555,-0.03603,0.03611,0.03322,0.01989,-0.03424,-0.03098,-0.095,0.0094,-0.04766,0.03406,0.01623,0.04803,0.01081,0.02131,-0.01541,0.02439,0.02984,0.00327,-0.02181,-0.01283,-0.00599,0.06374,0.03159,0.02941,0.00732,-0.00017,-0.03511,-0.00779,-0.00279,-0.00334,0.01702,0.01239,-0.00758,-0.00403,-0.05993,0.00675,0.02629,-0.02605,-0.01759,0.00797,-0.01698,-0.03316,-0.06832,0.00377,-0.04204,-0.03728,0.00784,0.06657,0.01635,-0.03653,-0.03448,-0.0042,-0.00504,0.04644,0.0107,0.02565,0.05564,0.00885,0.02644,0.05212,0.00242,0.03536,0.0192,0.02383,0.01901,-0.01006,0.02705,-0.04279,-0.00606,0.01002,-0.00333,-0.05587,-0.00189,0.01561,0.0552,-0.00953,0.04897,-0.01361,-0.00384,0.04547,-0.02567,0.02247,-0.00893,0.02218,-0.01858,0.00663,0.03462,-0.03407,-0.02493,-0.01628,0.02987,0.04452,0.02077,-0.00532,-0.02901,0.00884,0.05931,-0.01644,0.02545,0.00471,0.03595,0.01567,-0.02338,-0.00839,-0.05281,-0.00585,-0.00971,-0.04897,-0.01035,-0.0088,0.03115,-0.02032,0.01065,0.00115,-0.00542,-0.01686,-0.02556,0.01987,-0.01151,-0.01397,0.00202,0.01282,-0.01807,0.00456,0.00372,0.01989,-0.06425,0.05562,-0.0227,0.03642,0.01535,-0.03691,-0.06583,-0.02022,0.02946,-0.05371,-0.02705,0.00887,0.03171,-0.06397,-0.01168,-0.02988,0.00561,-0.06922,-0.01226,0.01551,0.06984,0.05458,-0.02257,-0.04874,-0.02622,-0.01035,-0.02683,0.00419,0.0367,-0.03065,-0.0451,-0.04322,0.00292,0.00097,0.01853,0.05196,-0.0196,0.01295,-0.01488,0.01254,-0.01084,0.00515,0.00257,-0.04237,0.01804,-0.03397,0.03957,0.02916,-0.00373,0.02016,-0.03168,-0.04602,-0.03368,0.05328,-0.05221,0.02208,-0.01633,0.044,0.05692,0.02358,-0.01209,0.05275,0.0004,-0.04706,0.05451,0.03488,0.01114,0.01685,0.02183,0.0415,0.00107,0.02898,-0.04218,-0.04362,-0.01915,0.01537,-0.02492,-0.05637,-0.04821,0.02953,0.00898,-0.01376,0.00898,-0.00765,-0.06303,-0.09321,0.01205,-0.00143,-0.04368,-0.03241,-0.04265,0.00778,-0.03445,0.01385,-0.00417,0.17389,0.02144,0.01811,-0.04282,-0.02938,0.02029,0.05084,0.03387,0.02313,0.02912,-0.02514,0.00368,0.02531,-0.03619,0.03223,0.01939,0.02452,0.04334,0.04004,-0.03382,0.0725,-0.00347,0.05657,0.01201,-0.05239,-0.05421,0.06037,0.06205,-0.02382,0.01883,-0.006,0.00015,-0.0547,0.02826,0.01621,-0.00555,0.00126,-0.00604,0.04162,-0.0451,-0.0413,0.00824,-0.06537,0.01002,-0.00597,0.03765,0.02738,-0.02459,-0.01068,-0.00148,0.02199,0.07132,-0.02875,-0.01683,0.00299,-0.00196,-0.04945,-0.04756,-0.00539,0.00111,-0.00861,0.02563,-0.02761,-0.0386,0.02013,0.04481,-0.02944,-0.02613,-0.03505,-0.03158,-0.01601,-0.00712,-7e-05,0.00155,0.02369,-0.0324,-0.01394,0.03892,0.04372,-0.00193,0.04418,-0.03928,-0.04163,-0.00398,0.06629,0.0264,0.05103,0.00748,-0.0492,0.02804,0.03052,-0.00105,-0.03316,0.02228,0.00037,-0.01572,0.0066,0.01587,-0.01735,-0.00703,-0.01271,0.01407,-0.00865,-0.0341,0.01685,-0.03788,-0.00733,0.04954,0.01142,-0.01359,0.04169,-0.01214,0.0261,-0.04434,0.02192,0.03339,0.03089,-0.06181,0.02282,0.01409,-0.00954,0.04259,-0.05463,0.02098,-0.01203,0.01701,-0.00795,0.00068,0.00697,-0.00506,0.00558,-0.0235,-0.04607,0.02162,0.02121,-0.01804,0.06836,-0.02559,-0.0312,0.0399,-0.00942,0.04542,-0.03989,0.04967,0.00898,0.03946,-0.02289,-0.04089,0.0011,-0.01858,0.00709,-0.02093,0.01581,0.00608,-0.0314,0.03741,-0.01064,0.005,0.00783,-0.0476,-0.04019,-0.02989,-0.00365,0.08605,0.04807,0.03505,-0.03834,-0.01646,0.03454,0.03367,-0.02465,-0.02021,0.02289,0.03244,-0.01561,-0.00029,-0.02201,0.03637,-0.0198,0.02235,-0.02434,0.05452,0.02209,-0.04978,-0.02817,0.02683,0.00599,-0.04426,-0.04146,-6e-05,-0.04405,0.08753,-0.01529,-0.00884,-0.04071,0.00535,0.01481,0.02575,-0.0072,-0.0473,0.008,-0.02536,-0.04046,-0.00286,-0.0297,-0.02313,0.01337,0.02312,0.02692,0.00824,-0.02022,-0.04903,-0.01026,0.00556,0.03029,0.05101,0.01232,0.00249,0.03748,-0.03579,0.04941,-0.0029,0.04438,-0.01277,-0.00476,0.03104,0.01169,0.03661,-0.02662,-0.04389,-0.00172,-0.01408,0.00323,-0.02027,-0.03216,-0.02045,0.02354,-0.00234,0.02916,-0.01754,0.00328,0.00314,0.02663,0.00437,0.00045,0.00535,-0.01126,-0.012,0.00719,0.04475,-0.02448,0.01864,0.02765,0.03468,-0.01558,-0.03756,0.03619,-0.00548,-0.04546,-0.03343,-0.00926,-0.00686,-0.03879,0.05985,-0.06223,0.0305,0.02729,-0.03702,-0.01569,-0.02114,-0.00262,-0.01627,-0.03333,-0.04311,0.07162,0.04769,0.00949,-0.0426,-0.01389,-0.00998,-0.04531,0.03624,-0.00778,-0.02647,0.02361,0.02894,-0.03759,0.03943,-0.03981,-0.03441,-0.03085,0.02417,-0.01954,0.00281,-0.04189,-0.0699,0.00481,-0.04181,-0.0118,-0.01496,0.0203,-0.04305,-0.02346,-0.0034,-0.02306,0.00302,0.0012,0.00477,0.04665,0.01654,0.0164,-0.01789,0.03566,0.00419,0.01069,-0.0121,0.00968,-0.02772,-0.01241,0.0375,-0.02738,0.00448,-0.00186,-0.00761,-0.03128,0.02369,0.03182,0.05314,0.01423,-0.0056,0.03644,0.00363,0.00432,0.0198,-0.02121,-0.034,-0.01198,-0.0029,-0.01168,-0.05005,-0.01002,0.05525,0.05908,-0.04571,0.03257,-0.02228,-0.04558,0.00035,0.01208,-0.03366,-0.02763,0.03849,-0.01452,-0.01949,-0.04626,0.00885,0.00317,-0.01688,0.01821,-0.05876,-0.0209,-0.04364,0.03995,-0.0058,-0.0252,0.02031,-0.02688,0.02564,-0.0177,0.04474,-0.03797,0.00526,-0.02617,-0.02875,-0.0162,0.01981,-0.04636,-0.01443,0.00673,0.03683,-0.03811,-0.06851,-0.04211,-0.01595,0.02201,-0.01535,-0.02497,0.00274,0.00713,0.04303,-0.0389,0.03208,-0.00082,-0.00909,-0.05695,0.02611,0.02654,0.01378,-0.01442,-0.00709,-0.07004,0.02546,0.04437,-0.02081,0.02367,-0.01867,0.02155,-0.00377,-0.02557,0.04564,0.05545,-0.00715,0.01833,-0.00652,0.00443,-0.01437,-0.03383,0.01849,0.00272,0.03644,0.00863,0.01915,-0.03701,-0.07782,-0.01782,-0.04097,0.03454,0.00914,-0.05547,0.06169,-0.04774,-0.00566,-0.00554,-0.04663,-0.00346,0.03982,-0.01269,0.01321,-0.0036,0.0255,-0.00477,0.0054,0.03494,-0.04744,-0.04603,-0.03342,0.02808,0.06497,0.03598,0.04581,0.02919,-0.04386,0.02736,0.09067,0.00201,-0.05136,-0.04285,-0.02123,-0.02951,-0.03727,-0.03569,-0.00046,-0.01406,0.05342,-0.00236,-0.0192,-0.00725,0.00241,0.0394,-0.02234,-0.01723,0.01885,0.00991,-0.02232,-0.03134,-0.01029,0.04766,-0.01141,-0.02619,0.04905,-0.01742,-0.0156,-0.06988,0.02335,-0.03614,0.01286,0.0327,-0.02571,-0.06076,-0.01087,-0.02113,0.01315,-0.03591,0.02044,-0.01217,-0.03432,-0.00891,-0.00375,0.01765,0.00592,0.02714,-0.01618,-0.00579,0.00692,-0.02262,-0.04021,0.00358,0.01012,-0.03977,0.01806,0.00885,0.0413,-0.04,-0.02114,-0.00548,-0.0637,0.03599,-0.01827,0.05482,-0.01294,0.02388,-0.01964,-0.01468,-0.0318,0.0453,-0.07325,0.00374,0.04224,-0.01943,-0.01983,-0.00196,0.05563,-0.00797,0.01338,-0.00046,-0.03666,-0.00474,-0.0165,-0.00462,-0.03112,0.00709,-0.06224,-0.03842,-0.04052,0.02456,-0.034,-0.0335,0.03863,0.02719,0.00488,-0.00227,-0.01322,0.02791,-0.01461,0.00433,0.04981,-0.04388,0.03438,-0.05131,-0.03741,-0.01007,0.01641,-0.02147,-0.02365,-0.00543,0.04933,-0.01421,0.00839,0.01756,0.00882,0.03674,-0.01625,0.05141,0.03255,0.02371,0.00419,-0.04884,0.0134,0.02299,0.03823,0.02117,0.01411,-0.04217,0.04015,0.00781,0.0088,-0.03898,0.03052,0.01402,-0.02349,-0.01019,0.05048,-0.03606,-0.00628,0.02164,-0.02057,0.04572,-0.02141,0.00474,-0.02469,-0.0389,0.00954,-0.03281,0.04934,0.01655,-0.03643,0.01387,0.04636,-0.00695,0.00747,0.02263,-0.03546,0.00307,0.0339,0.02223,0.03965,-0.0283,-0.01036,-0.00708,-0.01042,-0.04451,0.0087,-0.01809,-0.04373,0.021,0.01272,0.01337,0.00774,0.00897,0.02052,0.0461,-0.00381,0.03194,-0.00241,0.03502,-0.02532,-0.04624,0.00254]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."]},"sentence-transformers/all-roberta-large-v1":{"embeddings":[[-0.00491,0.0044,-0.01892,-0.00247,-0.07814,-0.06722,0.00618,0.04673,-0.02171,0.02503,-0.00043,0.0242,0.01465,0.03645,-0.03645,-0.0303,0.0148,-0.02533,-0.05983,-0.05134,0.01746,0.07743,0.04035,0.06142,-0.03255,-0.02727,-0.01221,0.00277,-0.00847,-0.02716,0.01255,0.00519,0.02993,-0.01681,-0.00143,-0.03417,-0.0274,-0.00213,-0.01146,-0.00556,0.01481,0.0074,-0.03304,0.03185,-0.01974,0.00986,-0.01792,0.04463,0.05752,-0.03997,0.07135,-0.00506,-0.02994,0.00729,-0.02176,-0.05994,0.0436,-0.06935,0.05496,0.06513,0.00618,0.03998,0.00477,0.00872,-0.01157,0.02082,0.00239,0.01787,-0.02335,0.00173,0.02505,-0.0253,-0.0048,-0.05145,-0.01483,-0.02045,0.02061,0.03904,-0.0242,0.03605,0.00746,0.05673,-0.02943,-0.03818,0.0057,-0.00487,-0.01773,0.03736,0.04141,-0.01267,0.01949,-0.09329,-0.0586,-0.01511,0.01865,0.00241,0.00344,-0.02159,-0.02566,0.03557,0.004,0.04771,0.03518,0.02876,-0.01231,-0.02624,0.00157,0.02732,-0.02482,0.00629,-0.03364,-0.02953,-0.01454,0.00136,-0.02454,0.02143,-0.02548,0.00873,0.00365,0.04238,0.003,0.03508,-0.00857,-0.01997,-0.00211,-0.03049,-0.0091,0.04757,0.00166,0.0182,0.0371,0.01116,0.02716,-0.02996,0.03532,-0.00068,0.01254,0.04435,0.01503,-0.01619,-0.0509,-0.04084,-0.02074,0.01607,0.01155,0.05921,0.02744,-0.01229,-0.00266,0.00486,-0.00701,0.01625,-0.04082,-0.00256,-0.00917,0.0095,0.00071,0.0086,0.01242,0.00685,-0.00454,-0.0298,-0.00142,-0.04007,-0.04089,-0.02946,-0.03479,0.01798,0.06451,0.04752,0.01171,0.02314,0.01749,-0.00481,0.01423,0.01982,0.0163,-0.01267,0.01271,0.03126,0.03253,-0.03753,-0.00359,-0.0126,-0.02904,0.00883,0.0046,0.01535,-0.01568,-0.00207,-0.03463,-0.02127,0.03125,-0.02658,0.00492,-0.04955,-0.00759,0.0269,-0.08531,0.01391,-0.01611,0.04433,0.01175,-0.00047,0.03434,0.00643,0.01327,-0.03346,0.01086,0.00132,0.00855,0.04931,-0.03122,-0.06766,-0.00312,0.01679,0.02908,0.00318,0.00156,-0.05245,-0.01896,0.01812,0.00441,0.00719,0.0047,-0.00169,0.01055,0.03316,-0.02826,-0.06428,0.00171,0.06791,-0.01168,0.02612,-0.03324,0.03362,-0.03644,0.02252,0.01491,0.04821,-0.02506,-0.04503,-0.04295,0.0499,0.0426,-0.05623,-0.0477,0.02001,-0.03126,-0.02243,0.00375,-0.0028,-0.03933,0.03446,0.00091,0.01792,-0.0067,0.00154,0.00762,0.02367,0.05458,-0.0094,-0.00325,0.03178,0.02171,0.04649,-0.01136,-0.00953,0.03607,-0.02213,-0.00294,0.01412,-0.00882,-0.01808,0.01598,-0.02923,0.02937,-0.04259,-0.05132,0.03444,-0.02586,-0.01684,-0.01645,-0.00298,-0.05119,-0.01414,-0.01932,0.05487,0.00152,-0.01417,0.03513,-0.03364,-0.04596,-0.01979,-0.00108,0.01705,0.03152,-0.01226,0.11015,-0.02259,-0.00777,-0.01998,-0.02853,0.02265,0.00672,0.05379,-0.03633,-0.04947,-0.04784,-0.06021,0.00587,-0.04636,0.00057,0.05334,0.01505,0.04036,-0.04575,-0.01661,-0.02811,0.02913,0.05085,-0.02248,0.02317,0.02474,-0.01465,0.03192,-0.05312,-0.00058,-0.01229,0.00217,0.02721,0.03574,0.03714,-0.01278,-0.01241,0.02278,-0.04196,-0.04708,-0.04374,-0.01287,-0.00865,-0.01168,0.0415,-0.04438,0.0311,0.02402,0.00413,-0.02244,-0.01726,0.06536,0.00535,0.01371,-0.02013,0.01311,-0.02689,0.00247,-0.02445,-0.03718,0.00307,0.00774,0.00495,-0.00145,-0.02505,-0.06807,0.05282,-0.01866,0.0034,0.06462,0.02268,-0.01834,0.00437,0.03333,-0.03028,-0.00205,-0.01992,0.00291,-0.00312,0.02033,0.01145,0.04449,0.07244,-0.01434,0.01164,-0.01788,-0.00801,-0.02117,-0.03054,0.0146,0.01765,-0.04504,0.00633,0.0427,-0.01145,0.01272,-0.04937,0.03976,-0.02522,-0.0062,-0.02807,0.02329,0.04338,-0.00469,0.03403,0.00379,0.02278,-0.01415,-0.0505,0.00756,-0.05988,-0.04672,0.01345,0.03528,0.02346,-0.02717,0.02367,0.0109,0.03678,-0.00212,0.00956,-0.0298,0.00592,-0.03096,-0.02238,0.02461,0.00253,0.01747,0.02629,-0.01784,0.01282,-0.00496,0.02501,0.01547,-0.02617,0.00983,-0.00499,-0.01497,0.01642,-0.02056,0.01205,0.06724,0.01422,0.01044,0.02634,0.01067,0.01288,0.01041,-0.00132,-0.0265,-0.03179,-0.01864,0.02567,-0.02476,-0.01453,-0.02537,0.01648,-0.03386,-0.00404,-0.01197,0.02813,0.01349,-0.00103,0.0362,0.03468,-0.01717,-0.00592,0.03509,0.03049,-0.05479,0.02247,0.00641,0.01327,0.05331,0.00286,-0.04148,0.02662,-0.02991,-0.01189,0.03,-0.01124,-0.01759,0.07745,0.02257,0.00103,0.0138,-0.05063,-0.03517,-0.03791,0.00638,0.0025,0.02377,0.00432,-0.05742,0.03785,0.00496,0.0258,0.00969,0.01271,0.02467,0.00955,-0.02789,0.0023,0.00495,0.03569,-0.05401,0.06956,-0.00585,-0.01595,0.01418,0.01695,0.03314,0.00375,-0.0264,0.02842,-0.00581,0.06678,0.07884,0.02801,-0.00835,0.04728,-0.04482,0.00496,-0.03334,-0.00353,-0.01707,-0.02384,0.01986,-0.02282,-0.01359,0.03213,-0.00551,-0.0134,0.06444,0.0102,0.01717,0.036,0.0449,-0.05578,0.02189,-0.05393,0.05831,-0.00837,0.04365,-0.03047,0.04218,0.04157,0.04729,-0.05191,0.04156,0.00472,-0.00231,0.04871,0.01498,-0.01352,-0.00074,-0.05809,0.00414,-0.01312,0.02515,0.04828,-0.029,0.00185,-0.02006,0.03079,-0.02273,-0.00054,0.00124,-0.04775,-0.03523,0.06123,0.02369,0.03288,-0.00942,-0.02377,0.03404,-0.01834,0.03162,-0.04765,0.00311,-0.04369,0.01712,-0.0046,-0.02907,-0.0196,0.01807,-0.02918,0.02673,-0.01424,0.06018,0.02108,0.05122,0.05261,-0.0844,0.01294,-0.00105,0.01756,0.06813,0.01816,-0.01834,-0.01305,0.04269,0.02993,-0.05113,-0.00208,0.01542,0.05871,-0.01004,-0.00253,0.02197,-0.00239,0.04582,-0.02667,-0.02511,-0.00493,-0.03509,-0.01359,-0.00577,-0.00793,-0.08161,0.01071,-0.02469,-0.00321,-0.02586,0.01736,0.04034,0.03213,-0.0275,0.01922,0.04221,-0.01884,0.0041,0.05143,-0.04363,-0.02603,0.01359,-0.04034,-0.02981,-0.0028,-0.00479,0.02249,0.02391,-0.03841,0.05699,0.00909,-0.05007,0.01069,0.01308,0.01425,0.01592,-0.01341,-0.03907,-0.05654,-0.0339,-0.02548,0.01985,0.00424,0.00145,-0.02947,0.0172,0.02241,0.02027,-0.01031,-0.01832,-0.10862,0.01336,0.00109,-0.09422,-0.01585,0.03974,-0.00537,-0.01127,-0.05054,0.04651,-0.03548,0.01287,-0.00876,0.08663,-0.00255,-0.00923,-0.01168,-0.00829,0.02034,-0.03456,-0.00957,0.02267,0.01669,0.01538,0.00694,0.02215,-0.01303,0.01227,0.01423,-0.01576,0.02942,-0.02075,0.05806,0.00468,-0.07045,-0.03456,0.07849,-0.02087,0.03461,0.01593,0.01949,0.00697,0.02198,0.02225,0.01448,-0.03578,0.03795,-0.02227,-0.0208,-0.01368,0.08223,-0.02417,-0.02438,0.00678,0.05668,-0.04544,0.01442,-0.02251,-0.0069,0.03637,-0.03406,-0.01678,-0.00843,-0.03108,0.04477,0.01902,-0.02718,-0.01966,-0.04527,-0.03346,0.0193,-0.03969,0.02866,-0.04977,-0.0432,-0.0015,-0.06911,-0.00447,-0.02955,0.02724,-0.0512,-0.00296,-0.03071,0.00101,-0.01681,0.03946,0.03542,-0.00468,-0.02902,0.0628,0.00848,-0.00765,-0.02137,-0.06543,-0.00419,0.00129,-0.02534,0.04327,0.03223,-0.00096,0.034,0.03474,-0.04742,0.08326,-0.02815,0.02141,-0.03308,0.00944,-0.04045,-0.02303,-0.0151,-0.01923,-0.03251,-0.01503,-0.01013,0.05736,0.00123,0.04242,0.04085,-0.08235,-0.00573,-0.00472,0.01922,8e-05,-0.00753,-0.03246,0.06445,0.02563,0.03713,0.02073,0.00281,-0.01773,-0.00507,-0.0161,-0.05207,0.0207,-0.01122,0.00498,0.05607,0.02406,-0.0077,0.00524,0.00656,0.02172,-0.00582,0.002,0.0163,0.00321,0.00288,-0.04978,0.03,-0.0309,0.0502,-0.00924,0.0218,-0.02395,0.00244,0.00688,0.01555,0.02774,-0.00523,0.00513,-0.05242,0.00226,-0.0107,0.01536,-0.06243,-0.04384,-0.02613,-0.0093,0.00854,-0.05516,-0.04737,-0.01029,0.0075,0.01335,0.00357,-0.0241,0.04834,-0.03318,0.00376,0.04555,0.01395,0.02176,0.00248,-0.00834,-0.0281,-0.01321,0.01123,0.00135,-0.0306,0.00553,-0.02512,-0.07398,-0.01681,0.02158,-0.01458,0.03779,-0.02546,0.01265,0.03814,0.00501,0.01211,-0.00943,0.02332,-0.00755,-0.01952,0.06056,-0.0167,0.02276,0.00015,0.03737,0.00537,0.02031,-0.03587,-0.002,-0.00012,0.02132,0.07326,-0.05212,-0.01719,0.02474,-0.01287,-0.05148,-0.02772,0.00794,0.01174,-0.0036,0.04666,-0.07608,-0.02778,0.03752,0.03669,0.03921,-0.04757,0.03959,0.05085,0.00134,-0.00052,0.008,-0.05218,-0.05433,-0.03486,0.00802,-0.06204,-0.03135,0.02423,0.00268,0.00185,-0.04034,-0.0165,-0.03715,-0.01869,0.00485,0.04019,0.03224,0.03855,0.04126,-0.02699,-0.01076,0.03594,-0.02552,0.00764,0.01708,0.00122,0.02882,0.06046,0.02527,0.0274,0.00398,0.00739,-0.02137,-1e-05,0.00693,0.03125,-0.01413,-0.00583,0.05256,-0.02371,0.0205,0.03769,0.048,0.02885,-0.06224,0.02187,-0.02163,0.04458,0.0028,0.04348,-0.03598,-0.0406,0.00882,0.04196,-0.05882,-0.02998,-0.03239,-0.02962,0.01559,0.00703,-0.06205,-0.0125,0.02488,-0.01178,0.0424,0.02942,0.00555,-0.0195,-0.01782,0.0299,0.01069,-0.05646,0.00072,-0.01938,-0.02975,-0.00091,-0.05657,0.0558,0.00578,-0.04121,-0.00179,0.04888,0.01047,0.00592,0.00888,0.02142,0.01432,0.01216,-0.01609,0.03247,-0.0326,0.03299,-0.03135,0.04498,0.04766,0.03267,0.06434,0.01058,0.02177,-0.03881,0.01728,-0.03258,-0.03123,-0.05876,-0.03556,-0.03094,0.03852,0.02517,-0.01034,-0.02895,-0.00488,0.02061,0.00195,-0.00882,-0.02741,0.02004,0.00786,-0.0326,0.00172,0.00652,0.01467,-0.02109,-0.01082],[-0.00206,0.04185,-0.04616,-0.03325,-0.01363,0.0082,0.01742,0.05534,0.01351,-0.01256,0.0137,-0.03019,0.00824,-0.03269,-0.01796,0.03263,-0.00355,0.0102,0.01736,0.00741,0.00189,-0.06432,-0.04325,0.04347,0.05882,-0.00604,0.02799,0.05457,-0.04737,-0.03622,0.02471,0.08941,-0.01274,0.02435,0.01284,0.03211,-0.01611,-0.03103,-0.01731,0.00477,0.01518,0.00549,-0.02322,0.00771,0.03693,-0.01392,-0.06142,0.03588,0.01451,0.00468,-0.01784,-0.01676,0.01256,-0.00215,0.00603,0.04483,-0.00245,0.01215,0.06297,-0.00953,0.05738,0.01576,0.04132,0.02367,0.00396,0.01046,-0.02265,-0.00082,0.01129,0.00746,0.06808,0.02221,-0.00141,-0.00015,-0.05808,0.0123,-0.00018,-0.04353,-0.04387,0.01658,0.0369,0.01736,-0.04124,-0.00343,-0.00814,-0.00035,0.01998,-0.0409,-0.01798,-0.07698,0.00107,0.01193,0.02141,-0.00631,-0.02103,0.02256,-0.00209,-0.03015,-0.00204,-0.05704,-0.05389,0.02812,0.07857,0.00886,-0.03011,0.0054,0.02466,-0.00949,0.05027,-0.0005,-0.0035,0.0802,-0.02775,-0.06988,-0.01802,0.00885,0.03015,-0.02436,-0.01613,0.01159,-0.0055,-0.03583,0.03152,0.02712,-0.02054,-0.01463,-0.10985,-0.02312,-0.00278,0.04664,0.04068,0.02423,0.01606,-0.03462,0.03663,0.00702,0.03483,-0.01558,0.01751,-0.04317,-0.00442,-0.0077,-0.01749,-0.07771,0.0298,-0.02019,0.00635,-0.00566,0.05242,0.00437,-0.04975,-0.01857,-0.02515,-0.03745,-0.01927,-0.01457,-0.01866,-0.03556,-0.00124,-0.05705,0.03255,0.03204,-0.0137,0.0852,0.03619,-0.00916,-0.02803,0.01214,-0.01861,-0.01464,0.0049,0.01991,0.06227,-0.01888,0.01824,-0.00231,0.02944,0.04604,-0.0331,0.02263,0.0051,-0.00735,0.00551,0.0072,0.03683,0.0038,0.02445,0.00724,-0.0013,0.04939,-0.02201,-0.00442,0.02748,-0.0205,-0.02536,0.01176,-0.06234,0.02924,-0.04934,0.08147,0.00588,-0.03138,0.03934,0.02914,0.04506,-0.00323,-0.04797,-0.03986,-0.00735,0.05942,-0.02488,-0.02243,-0.07256,0.02463,0.00706,0.00536,0.00423,0.04708,0.04336,-0.00569,-0.02316,0.02415,-0.02535,0.00019,0.01234,0.01061,-0.00099,0.09205,0.00833,0.00127,0.0048,-0.04443,-0.00441,0.03757,-0.02662,0.004,0.03018,-0.00481,0.02466,0.06229,0.00082,-0.00233,-0.03372,-0.00677,-0.01455,0.05469,0.02689,-0.0046,-0.05951,-0.00254,0.00873,0.06474,-0.00062,-0.00091,0.00322,-0.00839,-0.05954,-0.04145,-0.03907,0.00237,0.04903,0.03624,0.0301,-0.00821,0.0074,0.01214,-0.00045,-0.01308,0.02098,-0.03085,0.00782,0.04569,0.01247,-0.00435,0.01807,-0.01932,-0.00761,0.01233,-0.00909,-0.01043,-0.00562,-0.05074,0.01968,0.06434,0.00438,-0.0325,-0.01599,-0.00363,0.00043,0.00717,-0.00278,-0.04382,-0.03562,-0.03171,-0.08286,0.00014,0.01623,-0.02824,-0.03717,0.00498,-0.04826,-0.00263,-0.05056,-0.04194,-0.06332,0.06747,0.00319,-0.0241,0.08014,0.04745,0.03,-0.02278,0.01141,0.00233,-0.05887,-0.0393,0.00773,-0.04164,0.05626,-0.03491,0.03835,-0.05679,-0.0455,0.0033,0.05183,-0.02316,0.00121,-0.01228,0.03183,0.00488,0.01489,0.0265,0.03702,0.03868,-0.00911,0.06631,-0.01747,-0.03507,0.00137,-0.00545,-0.04762,0.01688,-0.01308,-0.00574,-0.03176,0.0231,-0.03408,-0.02792,-0.04425,0.05368,0.02461,-0.00494,-0.03715,-0.02371,-0.00814,-0.00642,-0.03314,0.01627,-0.00674,0.0027,-0.00055,0.04965,-0.01034,-0.0039,0.0244,-0.01923,0.02178,0.03745,-0.0033,-0.00019,-0.01738,-0.01694,-0.01221,-0.00066,-0.03128,-0.0155,0.03564,0.02341,-0.02074,-0.00488,0.017,0.01026,-0.08508,-0.01792,-0.03493,-0.01157,0.03419,-0.00872,-0.01336,-0.0178,0.0033,0.02473,-0.01584,-0.00609,0.07753,-0.05776,-0.09227,0.00851,0.00028,0.02119,0.09262,0.02519,0.01116,0.01204,0.00134,0.00788,0.00674,-0.00923,-0.02457,-0.04842,-0.01902,0.01435,0.00651,0.04692,0.02628,-0.02251,-0.04578,0.03277,0.03743,-0.0183,-0.0186,-0.03579,-0.04275,-0.00343,0.03249,0.01309,0.00876,-0.04225,-0.01847,-0.01134,0.00493,-0.02431,0.00356,0.00173,-0.0063,-0.00407,-0.05433,-0.0562,0.04584,0.02586,-0.04907,0.01275,-0.01099,-0.01046,0.02472,0.00286,-0.0039,-0.03374,0.00101,0.01051,0.01628,-0.00488,-0.07494,-0.02079,0.0633,-0.01488,-0.00093,-0.02143,-0.02613,0.00891,-0.0145,0.0108,0.01447,0.01659,-0.01807,-0.02359,-0.02848,-0.02412,0.05532,0.07516,-0.02723,0.02305,-0.04249,-0.03747,-0.03154,-0.0061,0.03429,-0.00176,-0.02662,0.02845,-0.01058,0.07094,-0.0081,-0.02039,-0.04784,0.04751,-0.01331,-0.02918,-0.02033,0.07103,0.0086,0.01157,0.0155,-0.02949,0.03427,0.00323,0.03064,0.03099,-0.02994,-0.02622,0.05816,0.07138,-0.01573,-0.03923,0.02165,-0.03942,-0.04769,0.02503,-0.06287,0.00134,-0.035,-0.01707,0.00082,0.00742,-0.00899,0.02446,-0.03779,-0.04808,0.01033,0.01803,-0.02838,-0.00682,-0.01022,-0.03513,-0.00151,0.03024,-0.01573,0.01315,-0.00141,0.01937,0.01113,0.03308,0.00821,0.03881,0.05997,0.02937,0.00052,0.03524,0.01438,-0.00951,0.0479,0.01645,0.02437,0.04108,-0.04179,0.01595,0.00915,0.01474,-0.01479,-0.01138,-0.00546,-0.00542,-0.02009,0.01217,-0.00839,-0.00528,0.03692,-0.02462,0.04018,0.01027,-0.02305,0.01413,-0.03221,-0.01619,-0.00366,0.00257,-0.01686,-0.01958,0.04608,-0.04776,-0.0189,-0.01264,-0.01905,0.00327,-0.00213,0.01189,-0.01534,0.04735,-0.02226,0.02636,0.00891,-0.02736,-0.0681,0.01168,-0.03668,0.04577,0.0606,-0.00887,0.02025,0.00978,-0.01444,0.01215,-0.03604,0.02607,-0.01037,0.00013,0.01262,0.00477,-0.01335,0.00707,-0.00663,-0.03453,-0.01628,0.01052,0.00832,-0.06043,0.03126,0.03244,0.00244,0.0503,-0.00291,0.0403,0.0198,-0.02373,0.03449,0.04352,0.02152,-0.04059,0.01888,-0.01627,-0.03998,0.02578,-0.0159,-0.00028,-0.0416,0.00671,0.00205,0.02315,0.00656,-0.00318,-0.01772,-0.02864,0.02008,-0.0263,0.01811,-0.0233,0.01575,-0.04911,-0.01261,0.02936,0.00213,0.02135,0.03473,-0.04622,0.04333,-0.00737,-0.03358,0.0227,0.01581,-0.00803,0.04156,0.01076,-0.01992,0.04343,-0.01923,-0.04269,-0.01079,0.01713,-0.03031,0.00322,0.02112,0.00213,-0.02852,0.04839,0.04973,0.05696,0.05122,-0.02071,0.00917,0.0374,0.00752,0.05741,0.01297,-0.0085,0.05919,-0.09915,-0.00748,-0.02673,-0.05797,-0.04767,0.05608,0.00457,0.03995,0.00935,-0.0533,0.02858,-0.00054,0.06385,0.00376,0.01214,0.05129,-0.01525,0.02155,0.01229,0.01694,-0.02441,-0.05561,0.02383,0.05626,-0.01144,-0.00459,0.00665,-0.04026,-0.02969,-0.00016,-0.03639,-0.0002,0.01423,-0.01365,0.0454,0.02831,0.04642,0.00878,0.10063,-0.04216,0.04849,-0.01032,-0.04384,-0.01958,0.03792,-0.0294,0.04493,-0.01516,-0.02431,0.01484,-0.05908,-0.0253,-0.00734,-0.0294,-0.0166,-0.01069,-0.01722,-0.01005,0.03595,0.00291,-0.05385,-0.02243,0.04624,-0.02963,0.0159,0.03085,-0.03689,-0.01771,0.01029,-0.01832,-0.02373,-0.04424,0.03274,-0.00124,-0.029,0.00258,0.00913,-0.00039,0.01209,0.03608,0.00296,0.02905,-0.01307,0.0118,-0.01356,0.00902,-0.04151,0.01593,-0.06927,0.02683,-0.02332,0.00528,0.07809,0.04598,0.00329,-0.01174,-0.02161,0.0144,0.03986,-0.00577,-0.00054,0.01055,0.10328,-0.00854,0.04123,-0.0281,-0.01627,-0.00809,0.00506,-0.01148,-0.00705,-0.02379,-0.00815,0.01779,-0.02028,0.01839,0.00314,0.00357,-0.00945,0.0606,-0.02172,-0.00588,-0.00081,-0.00645,-0.0295,0.0403,0.02714,0.03324,-0.02197,0.02591,0.07449,-0.02155,-0.03062,-0.01114,-0.02688,-0.00346,-0.03228,-0.03849,0.00818,-0.02564,0.01415,-0.01518,0.00811,-0.0192,-0.03064,-0.0161,-0.01336,-0.01244,-0.03178,0.03565,0.01717,0.02071,-0.03996,-0.04877,-0.04514,-0.00076,0.03818,-0.0116,-0.0291,-0.02425,-0.02174,0.02149,0.00177,0.02188,-0.01879,0.04391,-0.00093,0.01344,0.05639,-0.03074,-0.01048,0.02899,-0.01762,0.01265,-0.04332,-0.00886,0.04306,-0.02101,-0.04151,-0.02396,0.03238,0.00911,0.02717,0.01738,-0.01189,-0.03267,0.00315,-0.03517,-0.05413,0.0594,0.00687,0.02194,0.05496,-0.05165,-0.00918,0.05983,-0.01215,-0.01962,-0.03941,-0.00014,-0.01077,0.00628,0.02216,-0.01699,-0.04789,0.01132,-0.03225,0.00092,-0.01141,-0.01038,-0.05065,0.03999,-0.00573,0.06075,0.03056,0.0105,0.01771,-0.03294,-0.00462,0.02377,0.0391,-0.04897,0.0438,0.02617,8e-05,0.00045,-0.03178,-0.03441,-0.05191,-0.02425,-0.01991,-0.02728,0.03142,0.01688,0.00766,0.0004,-0.01117,-0.03277,-0.01741,-0.00079,-0.01226,0.06248,0.03794,-0.03267,0.01789,0.00371,0.01055,-0.04564,-0.03566,-0.01644,0.01897,-0.00794,-0.03457,-0.00889,-0.03042,-0.01716,-0.0378,-0.05054,-0.02853,0.02834,-0.00845,0.03893,0.02269,0.05895,0.04078,0.01726,-0.01072,-0.00909,0.05516,0.00596,0.02647,-0.00682,0.01746,0.03387,0.00924,0.06054,0.01094,-0.0156,-0.05887,-0.01205,0.04112,0.00109,0.05043,-0.03037,-0.01829,-0.00996,-0.00122,0.00906,-0.01136,-0.00464,0.01854,-0.06368,-0.01129,-0.0293,0.002,0.00344,0.00424,-0.00762,-0.02214,-0.01772,0.04501,-0.04189,-0.00136,-0.02585,0.00197,0.00762,-0.0319,0.01055,0.0245,-0.00664,-0.03415,0.02254,0.03979,-0.03625,-0.02966,0.04111,0.01457,0.02157,0.02104,-0.00403,0.00085,-0.00641,0.04651,-0.0203,0.01873,0.02191,0.02907,-0.03252,-0.03089,0.09486,0.02133,0.04798,-0.01348,-0.00205,-0.0566,-0.02041,0.03671,-0.01997,-0.01283,-0.00655,0.0332,0.04795,-0.01107,-0.01457,0.01461,0.01305,0.05409,-0.01398,0.01082]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."]}} \ No newline at end of file +{"ibm-granite/granite-embedding-125m-english":{"embeddings":[[-0.01831,0.00958,-0.02759,-0.01465,-0.01917,-0.01819,-0.00061,-0.02112,-0.01337,0.04932,-0.0282,-0.02612,-0.01843,-0.02734,-0.00497,-0.0166,0.00455,0.00488,0.01215,-0.02673,0.0354,-0.05444,-0.02332,-0.052,-0.03247,-0.01807,0.02454,-0.01648,0.00598,0.0332,-0.01373,-0.02771,0.04199,0.01831,0.04785,-0.02441,-0.00806,-0.00922,0.01184,-0.03491,0.00105,-0.0047,-0.00671,-0.01611,0.01093,0.00084,0.01331,-0.00362,-0.04199,0.03394,0.03687,0.00352,0.00885,-0.00467,0.03687,-0.00371,0.03467,-0.026,-0.02002,-0.01733,-0.01495,0.03223,-0.01587,-0.00036,0.01276,0.04736,0.01941,-0.01953,-0.04565,0.0415,0.01447,-0.026,-0.04468,-0.02673,0.03394,-0.00928,-0.07471,-0.03516,-0.00415,0.02087,0.02112,0.01746,-0.01379,0.02026,-0.03394,-0.02258,0.00577,-0.02917,-0.02393,-0.01514,-0.00668,-0.00983,0.02429,-0.00552,-0.02502,0.00203,-0.02307,0.026,-0.01196,-0.00319,0.00787,-0.00903,0.02063,-0.013,0.04761,0.01337,-0.00255,0.04419,0.02588,0.00867,0.02661,-0.01215,-0.02625,0.04321,0.01483,-0.00167,-0.0152,0.00681,-0.0006,-0.00665,-0.00436,-0.00958,0.01337,-0.03931,0.01917,0.01257,0.00488,0.01495,0.00958,-0.01611,-0.00117,0.0072,0.05371,-0.00854,-0.0105,0.00433,-0.0054,0.05981,-0.02734,-0.02332,0.00171,0.02881,-0.04761,-0.01672,-0.021,-0.00824,0.02734,-0.03809,-0.02185,-0.0144,-0.04663,-0.04663,-0.01257,0.03052,-0.02039,-0.0043,0.05811,-0.00739,-0.02087,0.02283,0.02502,-0.02661,-0.01501,-0.02075,-0.01019,-0.01337,-0.01038,-0.01343,-0.00093,-0.02734,0.02612,-0.02844,-0.00519,0.03833,-0.00916,-0.03369,-0.0076,-0.0293,0.03931,0.00378,0.01196,0.03564,-0.01807,0.01697,-0.00055,-0.02356,-0.00371,0.00577,-0.02551,-0.01257,-0.03174,0.02039,0.02136,-0.01044,0.01105,-0.00934,-0.02319,-0.01929,-0.02502,-0.0238,-0.02612,-0.01294,-0.03687,-0.03906,0.01483,-0.008,-0.02185,0.00163,0.04297,-0.03687,-0.0062,0.00252,-0.04785,-0.33203,0.00879,0.01843,-0.33008,-0.00461,-0.01953,-0.01105,0.02209,-0.05078,-0.00714,-0.03589,0.02368,-0.01428,0.04053,0.03369,-0.01611,-0.00162,0.02576,-0.01709,-0.01123,0.04468,0.0144,-0.05493,0.00153,0.00656,-0.02625,0.00781,-0.01117,-0.03613,-0.02783,0.02771,-0.02954,-0.05835,-0.01514,-0.0199,-0.04565,-0.0354,-0.00198,-0.00421,-0.32617,0.02441,-0.00476,0.02881,-0.0105,-0.02356,0.01019,-0.01685,0.02209,-0.02148,-0.03149,0.01019,-0.00711,0.01385,0.03906,-0.0152,0.00157,-0.01489,-0.03247,0.02905,-0.01782,0.01831,-0.00063,-0.00063,0.02356,-0.01422,-0.04614,-0.04321,0.04028,-0.01624,0.00333,-0.00687,0.01019,-0.01422,-0.01624,-0.00842,-0.03296,0.04297,-0.01447,0.01782,0.00793,-0.04883,0.03101,0.03467,-0.00879,-0.00177,0.02112,-0.00763,-0.0282,0.01385,0.00775,-0.0152,-0.01941,-0.01459,0.02661,0.0293,-0.00824,0.03589,0.02551,0.0188,-0.02673,-0.03113,-0.00598,-0.00916,0.00141,-0.02283,0.01941,-0.00516,-0.04053,-0.00885,-0.02576,-0.06055,-0.01587,-0.00671,0.01868,0.01965,0.01904,0.02173,-0.03345,0.013,-0.02295,-0.00173,-0.02368,-0.03149,-0.02893,-0.00983,-0.03467,0.02893,0.02295,0.01483,0.01562,0.01373,0.00058,-0.02673,0.00331,-0.02234,-0.04565,0.00952,0.0238,0.0141,-0.00562,0.01917,-0.00381,-0.00369,0.01013,0.03687,-0.01807,-0.0094,0.01562,0.0141,-0.04321,-0.03564,0.00977,0.00394,0.00047,-0.0127,0.03418,-0.02515,-0.01648,0.01343,0.00085,0.01501,0.021,-0.01648,0.03149,-0.01221,-0.04639,-0.01733,0.02356,-0.04004,-0.01343,0.01154,-0.00842,0.02283,0.06641,-0.01562,-0.02832,0.0083,-0.03064,0.00766,-0.01453,0.01416,0.01807,-0.01129,-0.021,0.03589,-0.00983,-0.02686,0.021,-0.0062,-0.02673,-0.00903,0.00113,0.13574,-0.03882,-0.0332,0.0481,-0.03247,0.0415,-0.00873,-0.02881,0.02356,0.03003,0.0097,-0.02649,0.00616,-0.0007,-0.00476,-0.07227,0.00879,0.01941,0.02307,0.41602,-0.01263,-0.04468,0.01556,-0.03369,-0.02917,0.03809,-0.01953,0.00903,-0.03223,0.05933,-0.05054,-0.02173,0.0009,0.02063,-0.03687,-0.00144,-0.02454,-0.03516,0.00291,-0.02368,0.01105,-0.00021,0.00504,-0.00879,0.01917,0.04346,0.00861,-0.01648,0.03613,0.00613,-0.0152,-0.021,0.04004,0.0354,-0.0017,-0.03687,-0.03149,-0.0437,-0.00433,0.03857,0.00534,-0.00958,-0.0013,0.02014,-0.02429,0.01422,-0.00307,0.00123,-0.01758,-0.01093,-0.04712,-0.02722,0.02783,-0.0076,-0.03833,0.00653,0.00439,-0.02844,-0.04077,-0.02649,0.05664,-0.05396,-0.04175,-0.03882,-0.02551,-0.01117,0.01184,-0.03931,0.02307,0.02832,-0.00412,-0.00775,0.01416,0.0025,-0.02026,-0.02661,-0.00446,0.00389,0.06006,4e-05,-0.01331,-0.03857,0.0293,-0.01929,-0.02173,-0.01031,0.02075,0.00696,0.03882,-0.02161,0.02014,-0.05566,0.02612,-0.04443,0.02686,0.00952,0.03662,0.0332,0.00922,0.03516,0.01709,-0.01483,0.03735,-0.00482,-0.04346,0.00543,0.01624,0.01196,0.0094,-0.01697,0.01758,-0.03564,0.05444,-0.0354,-0.00806,0.01917,0.02173,0.00504,-0.07715,-0.01312,0.02734,0.0332,-0.04053,0.03345,0.03613,-0.00504,-0.00696,-0.01123,0.0166,-0.05249,-0.02478,-0.00182,-0.008,0.05078,-0.03516,0.04932,-0.03882,0.00546,0.00635,-0.04785,0.02332,-0.05688,-0.00616,-0.0061,-0.01495,-0.00824,-0.03662,-0.052,-0.02844,0.03931,0.01685,0.02478,-0.01819,-0.04663,0.00143,-0.04712,0.03113,0.00854,0.02258,-0.02087,0.05762,0.01575,-0.0076,0.03491,-0.05615,-0.04126,0.01459,0.05469,-0.02441,-0.02161,-0.00842,-0.03662,-0.03369,0.02173,0.00328,0.00187,-0.01373,-0.02075,-0.00626,0.00164,-0.00158,-0.02368,-0.03442,-0.00388,-0.01538,-0.02429,-0.05005,0.0376,0.00632,-0.03735,-0.02844,-0.00276,-0.02222,0.02283,-0.0166,-0.00074,0.0043,0.01343,-0.0004,-0.01587,0.05103,0.02161,-0.01263,-0.0108,0.02905,0.00848,0.01575,-0.00885,0.02917,-0.00757,0.02234,0.02087,-0.02026,-0.03857,0.03296,0.0152,0.02368,0.02014,-0.01276,0.00288,0.00867,-0.00083,0.04858,-0.00723,-0.02148,0.00197,0.02405,0.01855,0.00317,-0.02234,-0.00854,-0.00334,-0.01324,-0.02039,0.05103,-0.02063,0.00385,-0.01221,-0.01587,-0.04224,-0.0152,-0.01514,0.01416,0.00272,-0.00177,0.01343,0.00421,-0.01648,-0.01013,-0.00653,-0.02844,0.00662,-0.00366,-0.01831,-0.02759,0.0024,-0.03442,0.01721,-0.00301,0.01538,0.00064,-0.03296,0.01758,0.01129,-0.00836,-0.03589,-0.00433,0.01373,-0.02832,0.00211,-0.00055,-0.0155,-0.00702,0.0376,0.00086,0.00479,-0.04004,-0.00574,-0.02844,-0.04224,0.02539,0.0304,-0.01257,0.02356,-0.04858,-0.013,-0.03906,-0.01532,-0.0076,-0.00577,0.02502,-0.01587,-0.02722,-0.0105,-0.00964,0.00879,-0.01123,-0.03198,0.00668,-0.04346,-0.03369,-0.04297,0.00439,-0.00272,0.03809,0.02087,-0.00476,-0.02576,0.02283,0.01294,0.03149,-0.03149,0.01868,-0.00934,0.01215,0.02661,0.02087,0.01105,0.00461,-0.01367,-0.01556,-0.00519,0.03979,-0.0238,-0.02722,-0.02808,-0.01056,-0.04785,-0.02454,-5e-05,0.00546,0.01147,0.02039,-0.00037,0.00452,-0.01831,0.00372,-0.01624,0.00378,-0.00013,0.03906,0.01337,-0.01031],[-0.00702,-0.00351,-0.01978,-0.0415,-0.00403,-0.03064,-0.01019,-0.02258,0.00726,0.1084,-0.03223,-0.03418,-0.01892,-0.00562,-0.02283,0.0072,-0.04614,0.0282,-0.00166,0.00677,0.00534,-0.02234,-0.02014,-0.04907,-0.04028,0.00094,-0.02075,-0.03113,-0.00365,0.01868,0.01697,0.01733,0.00836,0.00601,0.04663,-0.03516,-0.00485,-0.00793,0.02258,-0.0061,0.01306,0.05078,-0.04346,-0.00842,-0.01892,-0.00946,0.02087,-0.00033,-0.02673,0.04102,0.01434,-0.01129,0.0047,-0.01611,0.01477,0.00873,0.00995,0.0238,0.01117,0.01154,-0.05591,0.04688,-0.00726,-0.00616,-0.00647,0.03711,0.05005,-0.01611,-0.00952,0.04565,0.01178,-0.03711,-0.00298,-0.02454,-0.00467,0.00156,-0.0564,-0.03833,-0.0437,0.0072,-0.00494,-0.0199,-0.02588,0.03467,0.01178,-0.03113,-0.00583,-0.02136,-0.00677,-0.0166,-0.03113,-0.00842,0.02539,0.0127,-0.01117,0.00018,0.02832,-0.02393,0.01483,0.00702,0.0022,-0.00011,-0.01562,0.01508,0.05225,-0.00916,-0.00812,0.00995,0.04346,0.00035,-0.01038,-0.01227,-0.04175,0.00467,0.01404,0.02454,-0.01404,-0.00032,-0.01324,0.01294,-0.01154,-0.04663,0.01373,0.01648,0.03296,0.03711,0.0032,0.01636,0.01685,-0.02808,-0.00623,-0.02698,0.02722,-0.00334,-0.01404,-0.03223,-0.00772,0.03418,-0.06836,0.01648,-0.00173,0.0152,-0.0119,-0.01166,-0.03198,-0.02673,-0.00504,-0.01129,-0.08154,-0.01349,-0.00128,-0.01636,-0.01074,0.00179,-0.00885,-0.01129,0.01233,-0.03833,-0.01007,0.00121,-0.02563,-0.05493,-0.02136,-0.04688,-0.00772,-0.01758,-0.00708,-0.00334,-0.00467,-0.03088,0.00041,-0.06445,-0.00024,-0.02795,-0.04565,0.00916,-0.02307,-0.0304,0.03027,-0.02014,0.03516,0.02832,0.00662,0.01819,0.00607,0.00167,-0.00458,0.01166,-0.0166,-0.05273,-0.01172,-0.00266,0.00371,-0.00397,0.03833,-0.01099,0.01501,-0.03027,-0.02075,-0.00325,-0.00586,-0.00601,-0.05396,0.00497,0.03442,-0.02087,-0.02856,-0.01416,-0.01276,-0.02502,-0.00317,-0.02771,-0.00331,-0.31641,0.0498,0.04883,-0.31055,-0.00397,-0.00964,-0.00346,-0.04688,-0.00159,0.03442,-0.04346,0.03076,-0.02954,0.00156,-0.00025,0.00989,-0.03198,0.02014,-0.04688,-0.01221,0.03638,-0.01337,-0.05005,0.00708,0.00589,-0.01733,0.0282,-0.01941,-0.01025,-0.00252,0.03491,-0.02869,-0.08789,-0.01733,-0.01929,-0.01733,-0.02405,-0.03125,-0.01251,-0.29883,-0.00482,-0.01251,0.05908,0.01007,-0.026,0.02979,-0.01843,-0.01685,-0.06152,-0.01178,0.02454,-0.01709,0.00531,0.02344,-0.026,-0.00217,0.01709,0.01892,0.03076,0.01038,0.01965,-0.00519,-0.00513,-0.00928,0.00571,-0.01062,-0.02588,-0.00333,-0.01483,0.01514,-0.05005,-0.0238,-0.04395,-0.00357,0.04517,-0.04883,0.05029,-0.0097,0.05444,0.01178,-0.01227,0.02222,0.00193,-0.00903,-0.01324,0.02502,-0.04785,0.00867,0.00232,0.00769,-0.05347,-0.01624,-0.0038,0.0032,0.0282,0.01611,0.06592,0.02747,0.00653,0.00046,-0.0271,-0.02014,0.02454,-0.01483,-0.04395,0.01093,0.00522,-0.04688,-0.00147,-0.00412,0.00052,-0.02039,-0.03931,-0.00818,0.0354,0.00772,0.02429,-0.02954,-0.00351,-0.02637,0.00099,-0.01093,-0.00455,-0.03369,-0.03589,-0.05298,0.03516,0.05347,0.05029,-0.00638,0.00928,-0.03223,-0.02905,0.0481,-0.02307,-0.00403,-0.03369,0.02441,0.01636,0.02014,0.0293,-0.02661,-0.01373,0.026,0.06885,-0.0166,-0.03784,0.04565,0.03516,-0.03955,0.00078,0.03564,-0.02869,-0.02002,0.00668,0.0271,-0.04077,-0.00319,-0.03174,0.01453,-0.02454,-0.021,-0.03149,-0.00066,0.00439,-0.03882,0.00491,0.03491,-0.00867,-0.0034,-0.02466,-0.03027,0.01178,0.02832,0.00377,0.00885,0.03882,-0.03027,-0.02441,0.00995,-0.00717,0.05103,-0.01294,0.0097,0.02795,0.00301,-0.03931,0.00182,-0.01672,0.00586,-0.00304,-0.00757,0.18652,-0.0625,-0.02222,0.03638,-0.03931,0.01831,0.03296,0.00325,-0.0332,0.04248,-0.01587,-0.03198,-0.00491,-0.01819,0.02307,-0.06055,-0.01459,0.03198,0.00191,0.35156,0.01575,-0.00013,0.01709,-0.03027,-0.02307,0.02136,-0.02173,0.00861,-5e-05,0.06494,-0.07031,-0.01434,0.01434,-0.01575,-0.05664,-0.00473,-9e-05,-0.05005,-0.00607,-0.01599,0.01355,0.02222,-0.00739,0.01892,-0.05176,0.04346,0.01843,-0.03149,-0.04932,0.03687,-0.03955,0.00531,-0.01276,0.05396,0.02454,-3e-05,-0.06396,-0.00772,-0.02637,0.0249,-0.00012,-0.00331,0.00412,0.02722,-0.05615,0.00522,0.01251,0.00092,-0.00327,0.01953,-0.0376,-0.04297,0.0304,0.0304,-0.00861,-0.00397,0.04712,-0.04565,-0.0481,-0.04858,0.04053,-0.05664,-0.02649,-0.06299,0.0166,0.00172,0.01318,-0.02722,-0.01941,0.03345,-0.05127,-0.00133,0.04883,-0.00702,-0.02197,0.01807,0.01196,0.02393,0.04492,0.00812,0.05298,-0.00022,-0.0072,-0.03442,-0.03174,-0.04907,-0.00922,-0.03369,0.01746,-0.00595,-0.04663,-0.04614,0.03369,-0.05029,-0.00574,0.00037,0.00934,0.02185,-0.0166,0.0293,0.02087,-0.01062,0.00766,-0.00867,-0.00909,0.0459,0.013,-0.00359,0.04028,-0.01404,-0.0354,-0.00439,0.03113,-0.0304,-0.00934,0.00082,0.0293,-0.00149,-0.04395,-0.01477,0.01575,0.05176,-0.02429,0.013,0.03882,-0.00247,-0.01221,-0.02051,-0.00674,-0.01129,0.02832,-0.00033,0.02722,0.0119,-0.005,0.02661,-0.06128,0.00635,0.02612,-0.00439,0.0415,-0.06641,-0.02771,0.04077,0.01215,-0.00681,-0.00391,-0.0271,-0.00766,0.01892,-0.02026,0.00485,0.0304,-0.004,-0.00074,-0.01807,0.01892,-0.03589,0.0166,-0.00436,0.00192,0.00946,-0.00084,0.0069,-0.01355,-0.01123,-0.02502,0.05127,-0.03882,0.00497,0.02832,-0.01782,-0.03491,0.03882,-0.0293,0.01172,-0.021,-0.052,-0.03076,0.03491,-0.00126,-0.04688,-0.04028,0.01093,-0.03271,-0.00169,-0.00327,0.02832,-0.03711,-0.03027,-0.04004,-0.05591,0.02649,-0.02283,-0.00897,0.03247,-0.00377,0.01575,-0.02026,-0.02405,0.02332,0.01636,-0.01111,-0.02283,0.0069,0.00167,0.0069,0.00237,0.04883,0.00739,0.01459,0.01929,0.00153,-0.02832,0.03784,0.03638,0.03015,-0.01459,-0.02441,0.02246,0.03735,-0.03516,0.00861,0.00094,-0.03979,0.02356,-0.00589,-0.00339,0.01807,-0.04614,-0.0199,-0.02344,-0.03662,-0.00787,0.06299,0.01245,-0.01538,-0.02356,-0.04199,-0.01758,0.00641,-0.04688,0.02795,0.01843,0.04175,0.02197,0.00574,-0.01501,-0.02344,-0.02661,-0.01245,0.00056,-0.0105,-0.02234,0.00519,0.04614,0.02148,0.00479,-0.00781,-0.0498,0.00928,0.00574,0.01831,-0.01196,-0.02588,-0.02283,0.00327,-0.01624,0.00243,0.02087,-0.00879,0.03125,-0.00739,0.00928,-0.0332,-0.01746,0.01379,-0.01007,-0.04907,-0.01544,0.00198,0.00934,-0.02795,0.01544,-0.02881,0.00479,-0.00714,-0.04272,-0.02661,0.00854,-0.01807,-0.03931,-0.04492,-0.01892,0.00025,0.01965,-0.0238,-0.00077,0.05127,-0.00201,0.0072,0.00586,-0.00952,0.00141,0.03784,0.00427,0.02539,-0.01147,0.02563,0.02136,-0.00891,-0.00598,0.00635,-0.03174,0.013,-0.03223,0.03088,0.01514,0.02747,-0.02673,-0.02002,0.01025,0.02014,0.03064,-0.02307,-0.0564,-0.01843,-0.01129,-0.00638,-0.00337,0.02124,-0.03076,0.02039,-0.01324,-0.06689,-0.01685,0.00665,-0.01855,-0.03467,-0.00705,0.03418,0.03198,-0.00394]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."],"revision":"4ab61ffd423be45cd932b21a7c696063d82bf45f"},"ibm-granite/granite-embedding-278m-multilingual":{"embeddings":[[-0.03931,0.04541,0.01062,0.01904,0.0752,-0.04492,0.07568,0.05591,0.05127,0.03247,0.00861,0.00191,0.02673,0.02856,0.01685,0.08447,0.07861,-0.06934,-0.03296,-0.0177,-0.01111,0.02661,-0.01257,0.05908,-0.0094,-0.05591,-0.01782,-0.01245,-0.03979,0.02466,0.00394,0.0304,-0.01978,0.03662,-0.00313,-0.00684,0.02551,0.03345,-0.02734,-0.00039,-0.01965,0.03394,0.01599,0.01093,-0.01379,0.0155,-0.0177,0.04004,0.00458,0.03247,-0.05396,0.0141,0.00218,-0.00365,-0.04272,0.01526,0.06445,0.06592,-0.06445,0.0354,-0.0188,-3e-05,0.01685,-0.00327,0.02942,0.00763,0.06592,-0.06079,0.01337,0.05103,-0.00173,0.00668,0.05518,0.07178,-0.03955,0.00354,-0.02856,0.02319,0.00879,-0.00638,-0.04321,0.03589,-0.01709,0.01917,-0.03394,0.0155,0.02576,0.03076,0.05396,-0.03345,0.03015,-0.01685,-0.04346,-0.00226,-0.00313,-0.00365,0.01746,0.01538,0.05762,-0.03613,0.02759,-0.00052,0.00188,0.01361,0.02246,-0.00565,0.03345,-0.04663,0.0238,-0.0116,0.03979,-0.02979,0.05396,0.0304,0.03589,0.01172,0.01685,0.00069,0.05762,0.02051,0.07764,-0.02832,0.04907,-0.00885,0.02124,0.021,-0.0238,0.04175,0.03931,-0.01068,-0.00977,-0.08154,-0.02917,0.02222,-0.06348,-0.02942,-0.00644,0.01135,0.01019,0.00842,-0.013,-0.01782,-0.02271,0.04126,-0.01013,-0.03638,-0.02393,0.04199,0.03906,0.02051,-0.02917,-0.00885,0.05396,0.03711,-0.05469,0.03809,-0.01843,-0.00668,-0.01062,0.01318,0.0376,-0.008,0.00897,0.021,0.00659,-0.02832,-0.00056,0.05786,0.0141,-0.00209,-0.01831,0.02014,0.00824,-0.04053,-0.02722,0.04224,-0.00195,0.00729,-0.00964,-0.01025,0.02026,-0.02527,-0.11084,0.0354,0.0038,-0.00337,0.0006,-0.04834,-0.00775,0.03857,0.01556,0.00319,0.00287,0.15527,0.0791,-0.03394,-0.05225,0.00235,-0.08691,0.00616,0.0069,0.05322,0.02246,-0.0332,0.00159,0.021,0.05908,0.06006,0.01538,0.04858,-0.02271,0.03247,0.03711,0.05518,0.01599,-0.03101,0.2041,-0.00069,0.00107,-0.06079,0.04541,0.04053,0.01398,0.0199,0.00068,0.02515,0.01941,0.04175,0.05322,0.02161,0.05273,0.04102,-0.02893,-0.05957,0.07617,0.00308,0.02405,0.0332,0.00735,-0.03979,-0.02502,-0.02917,0.0119,0.02734,0.01202,0.0498,0.04224,-0.0437,0.02698,-0.0238,-0.00754,0.00885,0.02527,0.06104,0.07471,0.02234,0.01611,-0.01929,0.06592,0.05664,0.07422,-0.00182,0.00946,-0.00589,0.0376,0.0282,-0.02368,-0.03931,0.04248,0.06641,0.00243,0.06543,0.00934,-0.00916,0.05396,-0.00684,-0.01953,-0.03149,0.01733,0.01282,0.02979,-0.03003,0.0083,0.0304,0.00243,0.07227,0.0188,-0.03662,0.03113,0.00375,-0.02283,0.03369,0.04419,-0.00778,-0.03003,0.02612,0.0415,0.02881,0.02917,0.00342,-0.00127,0.10449,0.04443,-0.00446,0.02539,0.02222,-0.01508,-0.02771,0.0199,-0.01471,0.03345,-0.00365,0.0188,0.05273,-0.01965,0.05078,0.0282,-0.03149,-0.01477,-0.00818,-0.03589,0.03662,0.052,0.0097,0.00285,-0.05469,-0.00256,0.00909,0.00604,0.01208,0.02051,0.01624,0.01349,-0.00211,0.02332,-0.04053,-0.01855,0.12793,-0.04053,0.0072,0.00218,0.07471,-0.00623,-0.00934,0.0108,0.02136,0.0498,-0.03662,0.03198,0.00061,-0.01489,0.00665,-0.0108,0.02832,-0.00221,-0.03198,0.04565,-0.02979,0.00497,0.03467,0.03613,0.04712,-0.00842,-0.01361,0.026,0.01501,0.01471,-0.01538,-0.0304,0.03149,-0.03247,-0.02148,-0.05518,0.02295,-0.00885,0.03101,0.05664,0.01965,0.00787,-0.03015,0.00464,0.01538,0.01556,-0.03711,0.03809,-0.01471,0.00653,-0.01428,-0.07617,0.0141,0.06445,0.0603,-0.00149,-0.00958,0.01019,0.04565,-0.02771,0.02637,0.02417,0.0004,0.04272,-0.03589,0.0022,0.03735,0.04932,-0.02051,0.0097,-0.03271,-0.02283,0.01416,0.03589,0.0354,-0.00287,0.03589,-0.00583,0.03174,-0.02295,-0.01367,0.03271,0.02661,0.03589,-0.02551,0.04663,0.02734,-0.01709,0.03809,0.06396,0.01489,0.00464,0.03955,0.02234,0.0752,0.0062,0.01385,0.00571,0.0031,0.03906,-0.021,0.03491,0.00681,-0.008,0.02576,0.04761,-0.02673,0.02393,-0.04028,0.013,-0.00497,0.01599,0.08984,0.026,-0.0603,-0.0144,-0.02979,0.05322,0.06592,0.08643,0.07422,0.00705,0.01379,0.00452,-0.02832,0.00589,0.0498,0.05542,0.03198,0.04199,-0.01007,-0.02063,0.05884,0.04639,0.0332,0.0188,0.00169,0.03149,0.00818,0.0459,-0.05664,0.0047,-0.0177,-0.01508,0.03052,0.0564,-0.0015,0.01508,0.01831,0.00019,0.02576,0.00699,0.02295,0.0282,-0.04077,-0.00443,-0.01489,-0.02502,0.02112,-0.03418,-0.13672,-0.01276,0.02051,-0.00543,-0.00522,-0.04688,-0.00909,0.00525,-0.05225,0.03198,0.02148,-0.00368,0.03296,-0.09619,0.00305,0.01196,-0.0459,0.02942,0.02612,-0.00235,-0.01398,0.01978,0.05591,0.05396,0.01025,-0.04395,0.0083,0.03247,0.052,-0.02551,0.005,0.02661,-0.01398,0.02527,-0.0376,0.01501,0.026,0.00491,0.09277,0.00455,0.02771,0.05103,-0.04053,0.00592,-0.01263,-0.00909,0.02698,1e-05,-0.01599,0.06396,0.04761,-0.02576,-0.01245,0.00595,0.11426,-0.04004,0.02161,-0.00537,-0.01709,0.05713,-0.00121,0.01135,-0.04297,-0.01044,-0.02527,0.04907,0.07324,0.01929,0.00562,0.01929,0.021,0.03589,-0.02271,-0.00516,0.02551,0.03711,-0.00494,0.04883,-0.0155,0.04517,-0.00653,-0.02893,0.0354,0.03491,0.03638,-0.00934,0.03589,0.01105,-0.02527,-0.02417,-0.00806,-0.0376,-0.03345,-0.00873,0.04541,-0.03296,-0.04492,-0.06543,0.02563,-0.04004,0.00909,-0.04199,0.01459,-0.00311,-0.0177,-0.04248,0.03015,-0.05103,0.01385,-0.04639,-0.01416,0.02478,-0.0238,-0.02954,0.03174,-0.00244,-0.00644,-0.00256,0.03857,0.02283,-0.06006,-0.00885,-0.01349,0.01398,0.013,0.02197,-0.02612,-0.01166,0.00668,0.0304,0.01416,0.01074,0.05713,-0.00699,0.01202,-0.05713,0.03052,-0.02283,-0.01733,-0.02246,0.02161,-0.02136,-0.00552,-0.00436,0.00215,0.02673,0.03955,0.01575,0.00528,0.06201,0.0199,-0.05127,0.00182,0.01733,0.01978,0.05322,-0.0481,0.03662,-0.00769,-0.00824,0.00048,0.05273,-0.01379,-0.00861,-0.00249,0.00362,0.02234,0.01819,-0.00266,0.01013,0.05762,-0.0304,-0.05347,0.01367,0.04077,0.03149,0.06299,-0.01917,0.0069,0.02185,0.04199,0.0354,0.01965,-0.02478,-0.03418,-0.04834,0.00958,-0.00885,0.02124,0.03247,-0.0188,-0.02942,0.0354,0.0708,0.01794,0.00589,-0.01794,-0.00308,0.05518,-0.03015,0.00226,0.03516,-0.03613,-0.02393,0.04663,0.04443,-0.03223,-0.02881,-0.04712,0.05713,0.05469,0.0332,0.06348,-0.02173,-0.06226,0.01556,0.06885,0.02637,0.01428,-0.00071,0.05518,0.0708,0.0058,0.0282,-0.01459,0.01379,0.03784,-0.0083,-0.08252,-0.0354,-0.02209,0.06689,0.0603,-0.00674,-0.05103,0.00873,-0.00388,-0.14941,0.01385,0.00693,-0.00595,-0.01953,-0.00156,0.02429,-0.04614,0.02063,0.0152,0.02368,0.01709,0.02209,-0.03442,0.01953,-0.04199,0.06079,0.0065,0.00337,0.05127,0.0708,0.02405,-0.06348,0.01709,0.00568,-0.05078,0.04468,-0.01636,0.01282,0.02612,-0.0072,0.0177,0.02441,0.02283],[-0.03906,0.04712,-0.02917,-0.00665,0.02295,-0.06201,0.04785,-0.01129,0.05347,0.03174,0.01471,-0.01599,0.02112,-0.02673,-0.01453,-0.00665,0.03442,0.00647,-0.01086,0.00038,-0.0188,0.02795,-0.01459,0.01929,0.0481,0.00223,0.02454,-0.00714,-0.00243,0.00333,0.03125,0.00136,0.00903,0.02014,0.05127,-0.03955,-0.03735,0.02026,-0.05591,0.02148,0.02405,-0.0097,-0.01831,0.06641,-0.08643,0.06177,-0.01892,0.00473,0.01575,0.04614,0.00276,0.03345,0.0141,-0.02979,-0.02002,0.03638,0.0332,0.02466,-0.03345,0.00226,-0.07471,-0.04053,0.02197,-0.02356,0.04199,0.03564,0.07031,0.00278,0.02844,-0.03857,0.01337,-0.00629,0.04053,0.05957,-0.01831,0.0199,-0.04175,0.04639,-0.06396,-0.02026,-0.01562,0.00446,0.03491,-0.00952,-0.01086,0.06079,-0.00415,-0.03516,0.02405,0.00705,0.01868,-0.01367,0.008,0.04102,0.03931,-0.0332,-0.04199,0.05078,0.01575,-0.00745,0.02734,0.00191,0.0199,0.03101,0.0043,-0.00958,0.05713,-0.02185,0.02197,-0.00301,0.03516,0.00537,-0.02234,-0.00577,-0.04077,0.03369,-0.02759,-0.02588,0.03711,0.03687,0.01294,0.0199,0.04761,-0.04395,0.04492,0.01331,0.05054,0.0108,0.03638,-0.0332,-0.06641,-0.03979,-0.03809,-0.00446,-0.00616,-0.0069,-0.01056,0.04614,0.00928,0.02429,0.03369,-0.02429,-0.04468,0.0127,0.02551,-0.00595,0.07373,0.02783,0.04297,0.07617,0.02795,-0.00513,-0.01672,-0.0105,-0.08057,-0.0033,-0.00033,0.0481,-0.03662,0.04932,0.03906,-0.06885,0.05029,0.0105,-0.01135,0.06299,0.05273,0.02441,-0.01392,0.026,0.04858,0.06006,0.02734,-0.09521,0.03149,0.01294,0.02942,0.07031,-0.0199,-0.02112,0.01855,0.01532,-0.03589,0.09326,0.06299,0.09131,0.02881,-0.01025,0.02161,0.0097,0.0304,0.05176,-0.01172,-0.00751,-0.01483,0.00659,-0.04248,0.02466,-0.026,-0.05054,0.01648,-0.04199,0.01453,-0.03088,0.02722,0.01544,-0.0155,0.03149,-0.00296,0.04321,-0.00169,0.01398,-0.00668,0.0354,-0.01379,0.01746,0.07275,0.01721,0.06738,-0.0752,0.01257,0.03418,0.02161,0.01746,-0.03003,-0.02551,-0.0006,-0.01685,0.0013,-0.02905,0.02844,0.05444,-0.01196,-0.04736,0.07617,-0.01013,0.02515,0.01068,-0.02271,0.00507,-0.06885,-0.02527,-0.00482,-0.01917,0.04663,0.02441,-0.03064,-0.05444,0.03613,-0.05273,0.0332,0.02759,0.08594,0.0034,0.04297,-0.09863,0.00256,0.02014,0.03223,-0.00253,0.03662,0.03516,0.03247,-0.02222,-0.02478,-0.0072,-0.01831,0.01697,-0.0238,0.08398,-0.00415,-0.02734,-0.03125,0.00836,0.06982,0.0332,0.01172,0.00525,0.00577,0.00861,0.02539,0.00473,-0.03369,0.0354,0.00626,0.02856,0.06201,0.00549,0.05029,0.00726,0.05835,0.04297,-0.02356,-0.02905,-0.0007,-0.00537,-0.01141,-0.00632,0.02759,-0.00375,0.0282,0.04224,-0.03809,-0.05273,0.04883,0.04199,-0.01843,-0.00121,0.02942,0.01483,0.00952,0.06836,0.00293,0.04224,-0.00903,0.00699,-0.04053,-0.02466,-0.05054,0.00793,-0.00342,-0.02612,0.04224,-0.01868,-0.03088,-0.01892,-0.00089,0.05103,0.04224,-0.08398,-0.0083,0.05469,-0.02417,0.03979,-0.03027,-0.04517,-0.01965,0.05127,-0.07031,-0.00146,-0.01056,-0.00035,0.00989,0.02161,-0.0199,0.0459,-0.04077,0.05444,0.03125,0.0481,0.06445,0.03149,-0.01007,0.02795,-0.00455,-0.03955,0.02576,-0.02698,-0.05054,-0.03442,-0.00946,0.04102,-0.01929,0.08105,0.04932,0.05444,0.02515,-0.04004,-0.0238,0.0144,0.03662,0.05151,-0.01624,0.01599,0.04907,-0.03088,0.01758,0.01221,0.0835,-0.0437,0.03149,0.00983,0.01917,-0.02979,-0.02722,0.01672,0.02783,-0.02051,0.0238,-0.0564,0.01306,0.06079,0.05566,0.0481,0.05273,0.05762,-0.00705,-0.00072,0.00275,-0.00308,0.02051,0.04077,0.07861,0.02698,-0.01562,-0.06055,0.04932,-0.03064,-0.0437,0.07617,0.02942,0.03882,-0.00415,-0.02856,-0.05713,0.03516,-0.02148,-0.0009,0.01428,0.0029,0.05127,0.02258,-0.01575,-0.02161,-0.00604,0.02539,0.01459,-0.00111,0.01471,0.03931,-0.02722,0.04297,0.0459,0.01361,0.02295,0.04932,-0.00632,0.0437,-0.00284,0.01361,0.06885,0.00751,0.00766,0.01172,0.05396,-0.02002,-0.03345,-0.02039,0.02271,-0.00446,-0.00647,-0.08594,0.01483,0.01367,0.06787,0.00604,0.07227,-0.01331,0.01398,-0.05713,-0.01483,0.026,-0.00246,0.03491,-0.01904,-0.002,-0.00182,-0.0166,0.06885,-0.00861,0.0152,0.021,0.02734,-0.01331,-0.05957,0.06299,0.01599,0.00699,-0.04541,-0.03174,-0.06445,0.02332,-0.01978,-0.01294,0.05444,0.01367,0.01062,0.0001,0.02197,0.01965,0.02441,-0.05054,0.04468,-0.01331,-0.02759,0.03491,-0.01636,-0.08887,-0.03442,-0.00375,0.04761,-0.00604,0.0105,-0.01697,0.02466,-0.01068,0.03223,0.03088,0.00147,-0.03101,-0.04248,-0.02637,0.05396,0.01758,0.02551,0.00074,-0.01379,-0.04199,0.01733,0.04639,0.02612,-0.00028,0.00635,-0.04321,0.00014,0.03088,0.06543,0.0437,-0.02271,-0.03662,0.04102,0.0076,0.06592,0.00172,-0.00873,0.11035,0.01038,0.02539,0.05054,0.02795,0.008,-0.02405,0.00372,0.02966,-0.04688,-0.06396,0.00488,0.06006,-0.01855,0.00313,-0.00299,0.03613,-0.03687,-0.01544,-0.00977,0.03003,0.01263,0.03271,0.03955,0.01831,0.00494,-0.02515,0.04346,0.02942,0.01807,0.05566,-0.04883,0.01007,0.03027,-0.04126,0.07275,0.01843,-0.03442,0.06885,-0.00989,-0.04175,0.04736,0.02612,-4e-05,-0.0119,0.06006,0.01184,0.04175,0.00555,0.02722,0.00155,-0.08398,0.01379,0.00046,-0.01904,0.12109,0.03247,-0.06055,0.00745,-0.01007,0.07617,-0.03223,-0.04468,-0.0332,0.03271,-0.00087,0.03882,-0.02637,0.05957,0.008,-0.00638,-0.01392,-0.05566,-0.03125,-0.01685,-0.03003,0.00357,0.0249,0.03394,0.02942,-0.00854,0.05005,0.03198,0.03442,0.0177,0.06128,-0.02173,0.09277,-0.0415,-0.0625,-0.04858,-0.01782,0.01611,0.01245,-0.03711,0.03442,-0.01794,-0.01782,0.01501,-0.00879,0.04443,-0.00013,0.0332,-0.07764,0.03149,0.02002,0.00684,0.03833,0.01233,0.05396,0.02405,0.04956,0.01965,0.00772,0.05811,-0.02466,-0.04443,0.04956,-0.02332,-0.05566,0.00012,-0.02039,-0.01501,0.04443,-0.0249,0.05566,0.01965,0.05957,-0.0282,-0.00836,0.00653,0.0304,0.05591,-0.0144,0.0001,0.03687,0.01471,0.03564,0.00552,0.02356,0.04492,0.03394,-0.00043,-0.01501,0.05396,0.0304,0.04321,0.01166,0.01483,0.05566,0.02185,0.02014,0.0199,-0.01746,0.0166,0.04419,0.00221,0.02405,-0.00696,0.04419,-0.01965,-0.02161,0.01471,-0.00238,0.00218,0.03589,0.10791,0.05469,-0.06177,0.04565,-0.02283,-0.01001,0.02026,0.06641,0.04443,-0.0304,-0.01807,0.03687,0.02588,0.02637,0.0791,0.05835,0.02234,0.04419,-0.00995,0.0007,-0.01483,-0.04053,-0.00427,0.02917,-0.04077,-0.02673,-0.02966,-0.02917,0.06592,0.04639,-0.02405,0.04468,0.00818,-0.03101,0.05811,0.02271,0.02222,0.01398,-0.01062,0.0054,-0.0127,0.03198,-0.05078,-0.01208,0.06641,0.01599,0.00714,0.00476,0.01031,0.10791,-0.02222,0.01721,0.03931,-0.00266,0.00516,-0.06201,0.0625,-0.01294,-0.01782,0.04736,-0.00127,0.06348,0.02856,0.0332,0.03882,0.02124,0.00854]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."],"revision":"a9cb5338491faf32b73dd17b714a31821c021bbf"},"intfloat/multilingual-e5-large":{"embeddings":[[0.00615,0.00167,-0.01884,-0.03221,0.02153,-0.02805,-0.03668,0.03462,0.05085,-0.03967,0.02994,0.02673,-0.01631,-0.03023,-0.00784,-0.02785,-0.01527,0.01416,-0.0175,-0.0024,0.01564,-0.01382,-0.03142,-0.05291,-0.01336,-0.00906,-0.05366,-0.03229,0.00018,-0.02371,0.02445,0.0198,-0.057,-0.02765,-0.01388,0.04315,0.01792,0.04386,-0.04145,0.02259,-0.04027,0.06549,-0.00615,-0.04685,-0.02545,0.01882,-7e-05,0.00995,-0.00658,0.0094,0.01294,0.03544,-0.01205,-0.00679,-0.05014,0.02546,-0.03369,-0.00695,-0.06792,0.02104,-0.00758,-0.01068,0.02602,-0.00884,-0.05043,0.04765,0.04374,0.01041,-0.0627,0.01121,-0.01663,0.02877,-0.00805,-0.01111,-0.0205,-0.00844,0.01198,-0.00926,0.02134,-0.04331,0.06926,-0.00373,0.02077,-0.00268,0.04092,0.05955,0.05645,0.03653,0.04056,-0.0068,-0.02024,0.04116,0.04302,-0.03086,-0.06146,-0.0102,0.02745,0.01578,-0.0087,0.02171,-0.02511,0.01118,0.04607,-0.04068,-0.01473,0.02825,0.04167,0.03878,0.01572,0.01375,0.02937,0.06091,0.01171,-0.03897,-0.02686,-0.0017,-0.01511,-0.0206,-0.01555,0.02105,0.03059,0.05445,-0.00398,-0.02821,0.03217,-0.0198,0.00498,0.01906,-0.03535,-0.00161,0.01966,0.0339,-0.04975,-0.00636,-0.03221,-0.01732,-0.03673,0.01371,-0.0158,0.00554,0.00017,0.03149,0.03001,-0.03474,-0.00475,-0.03925,-0.06055,0.02426,0.0156,-0.02372,0.01525,-0.03527,-0.01349,0.01126,0.04015,-0.02439,-0.00961,0.02856,0.01881,-0.0098,-0.03581,-0.04898,-0.02589,-0.02458,0.00259,-0.01206,0.02079,0.03067,0.00878,0.01893,-0.0057,-0.01867,-0.00036,-0.06085,0.0105,0.03999,0.03616,0.01615,0.04167,-0.02559,0.00959,0.04495,0.03505,-0.04797,-0.0236,0.04523,0.01781,-0.00457,0.04794,0.03589,0.04838,-0.03042,-0.01687,0.00771,-0.02514,0.02846,0.01811,0.02622,-0.04418,-0.04573,-0.00331,0.01715,-0.04608,0.03864,-0.00672,-0.018,-0.03178,-0.03277,0.04316,-0.021,-0.02684,0.00573,0.01766,0.01733,-0.03753,-0.04431,-0.0218,-0.04106,0.02515,-0.02909,0.02671,0.04492,0.00073,0.03602,0.02872,0.02141,0.04402,0.02366,0.07388,0.04,0.01923,0.01851,-0.02748,-0.01402,0.00598,-0.02162,-0.02918,0.00522,0.03035,0.06531,-0.03152,0.02151,0.00102,-0.0669,-0.00033,-0.01244,0.03099,-0.02126,0.06486,-0.00309,0.01367,0.00166,-0.03883,-0.0096,0.02686,-0.01297,0.03665,-0.02066,-0.00011,-0.01652,0.01446,0.01598,-0.00921,0.05124,0.03684,0.02166,-0.00765,-0.04292,-0.0186,-0.03449,-0.02408,-0.04944,-0.06127,-0.01436,-0.02312,-0.00964,-0.04073,-0.02219,-0.01258,0.01835,-0.01879,-0.02844,0.01862,-0.03649,-0.022,-0.00534,-0.01025,-0.01194,0.04166,0.0023,0.01103,-0.01967,0.09679,0.00777,0.03473,0.03102,-0.05535,-0.08699,-0.03839,0.04657,-0.03918,-0.04668,0.02044,0.02931,-0.03045,-0.05196,0.01273,0.03193,-0.06546,-0.0224,0.0303,0.04325,0.01745,-0.0319,-0.05916,-0.00678,-0.0362,-0.01359,-0.03348,0.04909,-0.04637,-0.00442,-0.03869,0.00256,-0.0091,0.0323,0.00699,-0.03583,-0.00141,-0.03772,0.0364,-0.03694,-0.00036,0.05587,-0.03022,0.01914,-0.04939,0.03388,0.02687,-0.00381,0.02428,-0.00716,-0.05017,0.01245,0.01753,-0.01649,0.01085,-0.03297,0.02048,0.01497,0.00411,-0.03754,0.04071,0.00336,-0.02853,0.06207,-0.00445,-0.01898,0.02212,-0.01418,0.00869,0.02196,0.00822,-0.02099,-0.04342,-0.00806,-0.01612,0.02104,-0.04276,-0.04005,0.00753,-0.00583,-0.01166,0.00018,-0.00525,-0.01323,-0.08784,-0.01248,0.0087,-0.06877,-0.04399,-0.03193,0.01033,-0.03153,0.0083,-0.03041,0.15939,0.00369,0.01998,-0.05045,-0.01189,0.04456,0.0318,0.02134,0.04552,0.01934,-0.03649,-0.00998,0.02344,-0.0149,0.04619,0.05614,0.00407,0.03745,0.0283,0.00314,0.02897,-0.01966,0.05961,0.0096,-0.05195,-0.03497,0.00896,0.02183,-0.04654,0.0342,-0.01183,0.0266,-0.02103,-0.00451,0.02802,0.01168,0.02302,-0.06055,0.03326,0.0066,-0.03371,-0.00198,-0.03486,0.06223,-0.00114,0.03318,0.035,-0.01025,-0.033,0.03541,0.00551,0.01503,-0.02654,-0.03598,-0.02727,-0.02694,-0.0154,-0.06813,0.0187,0.01968,0.06567,0.02885,0.01509,-0.04656,0.02185,0.01838,0.00224,-0.05228,-0.03437,0.00359,-0.03251,-0.00873,-0.03066,0.02017,0.0496,-0.00101,-0.02414,0.04115,0.03096,0.0125,0.01225,-0.01952,-0.00629,-0.0021,0.05267,-0.03803,0.00979,0.00656,-0.02996,0.01079,0.02984,0.02911,-0.0254,0.03443,-0.00182,-0.0528,-0.00454,-0.01863,-0.02927,-0.01361,-0.02177,0.02903,-0.04105,-0.01436,-0.02122,-0.01872,0.00244,0.03449,0.02727,0.00095,0.02517,5e-05,0.04947,-0.00208,0.03659,0.0357,0.01634,-0.04529,0.0405,0.01845,-0.01805,0.02499,-0.02753,0.06908,-0.01798,0.0303,-0.02546,0.00353,0.04697,0.03999,0.02565,-0.03321,-0.02968,0.03114,0.0114,-0.02264,0.05427,-0.02309,-0.01277,0.04318,-0.01662,0.0375,-0.03253,0.03411,0.01434,0.02483,-0.04909,-0.04621,-0.02879,-0.03238,0.0281,-0.04326,0.02168,0.00135,-0.01895,-0.02044,-0.00199,0.00882,0.0467,-0.038,-0.01491,0.00713,0.02949,0.08934,0.03923,0.03082,-0.0556,0.03442,0.04656,0.01178,-0.01369,-0.0359,-0.00317,0.0165,0.03202,-0.02372,-0.02483,0.04296,-0.05164,0.0388,-0.00114,0.03858,0.01901,-0.05343,-0.02259,0.00472,0.01412,-0.03766,0.00163,0.0276,-0.01545,0.06877,-0.0334,-0.00281,-0.03326,-0.00625,0.03139,0.0168,-0.01824,-0.03339,0.00804,-0.03694,-0.00283,-0.0191,-0.03522,-0.01633,-0.00378,0.03659,0.00572,0.06713,0.01236,-0.06875,0.00218,0.03009,0.031,0.01589,0.03229,0.03717,0.04021,-0.02505,0.02784,0.02861,0.02398,-0.03976,-0.04332,-0.03079,0.07925,0.01692,-0.01335,-0.02087,0.0103,-0.02611,-0.00049,-0.0713,-0.02071,-0.01673,0.04088,-0.0074,0.01223,-0.02923,0.04267,-0.00196,0.02846,-0.01718,-0.0108,0.0252,-0.03355,-0.01167,0.02645,0.01847,-0.00764,-0.02294,0.03671,0.04711,-0.04833,-0.00658,0.01129,-0.01064,-0.04095,-0.05825,0.02147,0.02263,-0.03088,0.00999,-0.02612,-0.0002,0.02627,-0.05376,-0.00558,-0.04673,0.0439,-0.027,-0.00552,-0.05778,0.03202,0.04894,-0.02891,-0.03764,-0.02056,-0.02949,-0.03207,0.01705,0.00691,0.02757,0.01405,0.03422,-0.04516,0.05298,-0.0214,-0.03076,-0.02831,0.0001,-0.03295,-0.00082,-0.02642,-0.04646,-0.02201,-0.01685,-0.01091,-0.04717,0.02315,-0.01265,-0.04558,-0.06804,-0.01355,0.03169,-0.03092,-0.00136,0.03555,0.02648,-0.00366,-0.03205,0.0166,0.02467,0.03169,0.00783,0.03284,-0.02549,-0.00702,0.04325,-0.04913,-0.00615,-0.01033,-0.01693,0.06183,-0.00409,0.02603,0.00841,0.01077,0.00269,0.01264,0.00167,-0.02049,0.00447,-0.05125,-0.05859,-0.02217,-0.0244,-0.03446,-0.04205,-0.00539,0.03457,0.02077,-0.02666,0.03425,0.02147,-0.02431,0.01696,-0.02137,-0.01403,-0.0261,0.02934,-0.01227,-0.01623,-0.00687,0.03435,-0.03354,0.00529,0.01695,-0.04586,-0.03223,-0.02635,0.01901,-0.01786,-0.00221,0.03267,-0.01975,0.02599,-0.00909,-0.00334,-0.01975,0.04265,-0.0342,0.00266,-0.02881,0.02719,-0.01439,0.01482,0.05184,0.01778,0.03,-0.07998,-0.05054,-0.04743,0.05319,-0.02988,-0.02204,0.02126,-0.00235,0.03775,-0.04423,0.04813,-0.01801,0.03014,-0.08231,-0.01351,-0.00466,-0.04255,-0.00617,-0.00554,-0.02872,0.01829,-0.00609,-0.0465,0.01261,0.03835,0.02638,-0.01093,-0.00171,0.03271,0.03421,-0.01843,0.00153,0.00913,0.02476,-0.03444,0.00221,0.01112,0.03416,0.03274,-0.00648,0.02359,-0.03598,-0.05186,-0.01576,-0.04442,0.0109,0.03642,-0.03093,0.04372,-0.00046,0.01294,-0.00861,-0.00414,-0.0453,0.0214,-0.05431,-0.00383,-0.0212,0.0344,-0.0017,-0.03341,-0.00681,-0.03029,-0.02644,-0.03305,0.02823,0.05523,0.02451,0.03289,-0.00016,-0.04674,0.0203,0.01865,0.00989,-0.03898,-0.01892,-0.03093,-0.03522,-0.01432,-0.0524,-0.03018,-0.02161,0.02863,0.03719,-0.02986,0.02648,-0.00214,0.04029,-0.01852,-0.00938,-0.00501,0.03258,-0.00685,-0.01942,-0.0135,0.01719,0.02572,-0.00758,0.0167,0.00859,-0.01627,-0.05348,0.0355,-0.03085,-0.00022,0.03865,-0.05134,-0.03863,-0.01656,-0.02716,0.00337,-0.02018,0.04672,-0.03461,-0.00563,0.01592,-0.00138,0.02164,0.01698,0.0256,0.02619,0.02859,-0.01706,-0.03702,-0.02659,-0.01661,-0.0227,-0.00344,0.02523,-0.02006,0.0081,-0.01022,0.00937,-0.02498,-0.07168,0.03724,-0.0122,0.04004,-0.00867,0.00592,-0.04052,0.00315,-0.0341,0.01146,-0.07161,0.04147,0.01907,-0.03241,-0.01396,0.02179,0.06182,-0.04631,0.04859,-0.00359,-0.04783,-0.02793,-0.0342,-0.03299,0.03834,-0.02407,-0.02525,-0.00041,-0.0372,0.03359,0.00067,-0.03844,0.0479,0.01996,-0.00293,0.01322,-0.00658,0.02476,-0.00072,-0.00677,0.01804,-0.0576,0.03539,-0.01186,-0.01977,0.01781,0.00802,-0.03949,-0.01356,0.02769,0.03701,-0.04249,0.01506,0.0521,-0.01934,0.05178,0.03863,0.02595,0.03125,0.02879,0.01489,-0.00325,0.0311,0.0201,0.05376,0.0145,0.01407,-0.03218,0.01808,-0.02966,0.03742,-0.04484,0.04111,0.06778,-0.02128,-0.02059,0.00531,-0.02297,-0.00817,0.00135,-0.00885,0.01699,-0.03618,-0.00299,-0.02229,-0.05462,0.00601,-0.02324,0.03343,0.04094,-0.00466,0.01443,0.02148,0.03247,0.02243,0.03655,-0.0471,-0.01349,0.04415,0.0376,0.03211,-0.00811,0.01018,-0.04028,0.0433,-0.05766,0.02474,-0.04351,-0.02233,0.00075,0.01573,0.00695,0.01393,-0.01471,-0.05068,0.04218,0.02377,0.02173,0.01943,-0.02215,0.00109,-0.02419,0.02293],[0.03846,-0.0082,-0.00921,-0.02049,0.03584,-0.00922,0.00239,0.05344,0.05032,-0.03811,0.0011,0.03783,-0.06407,-0.04176,0.01019,-0.00751,-0.00255,-0.00436,0.00156,-0.03799,0.02109,-0.00684,0.01454,-0.01424,-0.03091,-0.04024,0.02576,-0.01973,-0.01787,-0.03813,-0.01958,-0.00714,-0.03606,-0.04513,-0.00805,0.02994,0.03516,-0.01577,-0.03248,0.01195,-0.02364,0.02526,-0.00402,-0.01722,0.00034,0.03871,0.04856,0.00153,-0.03923,0.00818,-0.0246,-0.00379,-0.00817,-0.0081,-0.01533,-0.00799,-0.00553,-0.0174,-0.028,-0.01267,-0.03008,-0.02038,-0.00394,-0.0224,-0.05102,0.03389,0.03188,-0.0121,-0.0522,0.03167,0.0167,0.02915,-0.0119,-0.03779,0.0002,-0.02102,0.02669,-0.01683,0.04762,0.01064,0.07822,-0.00896,-0.00347,-0.02252,0.03228,0.04546,0.01981,0.00847,0.04402,0.01891,-0.00772,0.04674,0.03865,-0.00523,-0.03102,0.00949,0.02012,0.02886,0.01917,-0.00809,-0.00408,0.02538,0.01319,-0.04884,-0.03445,0.06332,0.01959,0.03309,0.01632,0.02214,0.06101,0.03052,-0.01289,-0.01785,-0.00822,-0.01567,-0.0133,-0.01967,-0.00718,0.01587,0.05309,0.0174,0.01217,-0.02263,0.03331,-0.00693,0.01562,0.03049,0.0047,0.06342,0.03621,0.02228,-0.01601,-0.05516,-0.07028,-0.05228,0.00454,0.01021,-0.01607,0.03895,0.01881,0.01285,0.02272,-0.03872,-0.0324,-0.07229,-0.03093,0.02382,-0.01966,-0.03185,0.01974,-0.02843,-0.04072,0.01126,0.02838,-0.02311,0.02693,0.0554,0.01074,0.0093,-0.04094,-0.02215,-0.00925,-0.02206,-0.01555,-0.03603,0.03611,0.03322,0.01989,-0.03424,-0.03098,-0.095,0.0094,-0.04766,0.03406,0.01623,0.04803,0.01081,0.02131,-0.01541,0.02439,0.02984,0.00327,-0.02181,-0.01283,-0.00599,0.06374,0.03159,0.02941,0.00732,-0.00017,-0.03511,-0.00779,-0.00279,-0.00334,0.01702,0.01239,-0.00758,-0.00403,-0.05993,0.00675,0.02629,-0.02605,-0.01759,0.00797,-0.01698,-0.03316,-0.06832,0.00377,-0.04204,-0.03728,0.00784,0.06657,0.01635,-0.03653,-0.03448,-0.0042,-0.00504,0.04644,0.0107,0.02565,0.05564,0.00885,0.02644,0.05212,0.00242,0.03536,0.0192,0.02383,0.01901,-0.01006,0.02705,-0.04279,-0.00606,0.01002,-0.00333,-0.05587,-0.00189,0.01561,0.0552,-0.00953,0.04897,-0.01361,-0.00384,0.04547,-0.02567,0.02247,-0.00893,0.02218,-0.01858,0.00663,0.03462,-0.03407,-0.02493,-0.01628,0.02987,0.04452,0.02077,-0.00532,-0.02901,0.00884,0.05931,-0.01644,0.02545,0.00471,0.03595,0.01567,-0.02338,-0.00839,-0.05281,-0.00585,-0.00971,-0.04897,-0.01035,-0.0088,0.03115,-0.02032,0.01065,0.00115,-0.00542,-0.01686,-0.02556,0.01987,-0.01151,-0.01397,0.00202,0.01282,-0.01807,0.00456,0.00372,0.01989,-0.06425,0.05562,-0.0227,0.03642,0.01535,-0.03691,-0.06583,-0.02022,0.02946,-0.05371,-0.02705,0.00887,0.03171,-0.06397,-0.01168,-0.02988,0.00561,-0.06922,-0.01226,0.01551,0.06984,0.05458,-0.02257,-0.04874,-0.02622,-0.01035,-0.02683,0.00419,0.0367,-0.03065,-0.0451,-0.04322,0.00292,0.00097,0.01853,0.05196,-0.0196,0.01295,-0.01488,0.01254,-0.01084,0.00515,0.00257,-0.04237,0.01804,-0.03397,0.03957,0.02916,-0.00373,0.02016,-0.03168,-0.04602,-0.03368,0.05328,-0.05221,0.02208,-0.01633,0.044,0.05692,0.02358,-0.01209,0.05275,0.0004,-0.04706,0.05451,0.03488,0.01114,0.01685,0.02183,0.0415,0.00107,0.02898,-0.04218,-0.04362,-0.01915,0.01537,-0.02492,-0.05637,-0.04821,0.02953,0.00898,-0.01376,0.00898,-0.00765,-0.06303,-0.09321,0.01205,-0.00143,-0.04368,-0.03241,-0.04265,0.00778,-0.03445,0.01385,-0.00417,0.17389,0.02144,0.01811,-0.04282,-0.02938,0.02029,0.05084,0.03387,0.02313,0.02912,-0.02514,0.00368,0.02531,-0.03619,0.03223,0.01939,0.02452,0.04334,0.04004,-0.03382,0.0725,-0.00347,0.05657,0.01201,-0.05239,-0.05421,0.06037,0.06205,-0.02382,0.01883,-0.006,0.00015,-0.0547,0.02826,0.01621,-0.00555,0.00126,-0.00604,0.04162,-0.0451,-0.0413,0.00824,-0.06537,0.01002,-0.00597,0.03765,0.02738,-0.02459,-0.01068,-0.00148,0.02199,0.07132,-0.02875,-0.01683,0.00299,-0.00196,-0.04945,-0.04756,-0.00539,0.00111,-0.00861,0.02563,-0.02761,-0.0386,0.02013,0.04481,-0.02944,-0.02613,-0.03505,-0.03158,-0.01601,-0.00712,-7e-05,0.00155,0.02369,-0.0324,-0.01394,0.03892,0.04372,-0.00193,0.04418,-0.03928,-0.04163,-0.00398,0.06629,0.0264,0.05103,0.00748,-0.0492,0.02804,0.03052,-0.00105,-0.03316,0.02228,0.00037,-0.01572,0.0066,0.01587,-0.01735,-0.00703,-0.01271,0.01407,-0.00865,-0.0341,0.01685,-0.03788,-0.00733,0.04954,0.01142,-0.01359,0.04169,-0.01214,0.0261,-0.04434,0.02192,0.03339,0.03089,-0.06181,0.02282,0.01409,-0.00954,0.04259,-0.05463,0.02098,-0.01203,0.01701,-0.00795,0.00068,0.00697,-0.00506,0.00558,-0.0235,-0.04607,0.02162,0.02121,-0.01804,0.06836,-0.02559,-0.0312,0.0399,-0.00942,0.04542,-0.03989,0.04967,0.00898,0.03946,-0.02289,-0.04089,0.0011,-0.01858,0.00709,-0.02093,0.01581,0.00608,-0.0314,0.03741,-0.01064,0.005,0.00783,-0.0476,-0.04019,-0.02989,-0.00365,0.08605,0.04807,0.03505,-0.03834,-0.01646,0.03454,0.03367,-0.02465,-0.02021,0.02289,0.03244,-0.01561,-0.00029,-0.02201,0.03637,-0.0198,0.02235,-0.02434,0.05452,0.02209,-0.04978,-0.02817,0.02683,0.00599,-0.04426,-0.04146,-6e-05,-0.04405,0.08753,-0.01529,-0.00884,-0.04071,0.00535,0.01481,0.02575,-0.0072,-0.0473,0.008,-0.02536,-0.04046,-0.00286,-0.0297,-0.02313,0.01337,0.02312,0.02692,0.00824,-0.02022,-0.04903,-0.01026,0.00556,0.03029,0.05101,0.01232,0.00249,0.03748,-0.03579,0.04941,-0.0029,0.04438,-0.01277,-0.00476,0.03104,0.01169,0.03661,-0.02662,-0.04389,-0.00172,-0.01408,0.00323,-0.02027,-0.03216,-0.02045,0.02354,-0.00234,0.02916,-0.01754,0.00328,0.00314,0.02663,0.00437,0.00045,0.00535,-0.01126,-0.012,0.00719,0.04475,-0.02448,0.01864,0.02765,0.03468,-0.01558,-0.03756,0.03619,-0.00548,-0.04546,-0.03343,-0.00926,-0.00686,-0.03879,0.05985,-0.06223,0.0305,0.02729,-0.03702,-0.01569,-0.02114,-0.00262,-0.01627,-0.03333,-0.04311,0.07162,0.04769,0.00949,-0.0426,-0.01389,-0.00998,-0.04531,0.03624,-0.00778,-0.02647,0.02361,0.02894,-0.03759,0.03943,-0.03981,-0.03441,-0.03085,0.02417,-0.01954,0.00281,-0.04189,-0.0699,0.00481,-0.04181,-0.0118,-0.01496,0.0203,-0.04305,-0.02346,-0.0034,-0.02306,0.00302,0.0012,0.00477,0.04665,0.01654,0.0164,-0.01789,0.03566,0.00419,0.01069,-0.0121,0.00968,-0.02772,-0.01241,0.0375,-0.02738,0.00448,-0.00186,-0.00761,-0.03128,0.02369,0.03182,0.05314,0.01423,-0.0056,0.03644,0.00363,0.00432,0.0198,-0.02121,-0.034,-0.01198,-0.0029,-0.01168,-0.05005,-0.01002,0.05525,0.05908,-0.04571,0.03257,-0.02228,-0.04558,0.00035,0.01208,-0.03366,-0.02763,0.03849,-0.01452,-0.01949,-0.04626,0.00885,0.00317,-0.01688,0.01821,-0.05876,-0.0209,-0.04364,0.03995,-0.0058,-0.0252,0.02031,-0.02688,0.02564,-0.0177,0.04474,-0.03797,0.00526,-0.02617,-0.02875,-0.0162,0.01981,-0.04636,-0.01443,0.00673,0.03683,-0.03811,-0.06851,-0.04211,-0.01595,0.02201,-0.01535,-0.02497,0.00274,0.00713,0.04303,-0.0389,0.03208,-0.00082,-0.00909,-0.05695,0.02611,0.02654,0.01378,-0.01442,-0.00709,-0.07004,0.02546,0.04437,-0.02081,0.02367,-0.01867,0.02155,-0.00377,-0.02557,0.04564,0.05545,-0.00715,0.01833,-0.00652,0.00443,-0.01437,-0.03383,0.01849,0.00272,0.03644,0.00863,0.01915,-0.03701,-0.07782,-0.01782,-0.04097,0.03454,0.00914,-0.05547,0.06169,-0.04774,-0.00566,-0.00554,-0.04663,-0.00346,0.03982,-0.01269,0.01321,-0.0036,0.0255,-0.00477,0.0054,0.03494,-0.04744,-0.04603,-0.03342,0.02808,0.06497,0.03598,0.04581,0.02919,-0.04386,0.02736,0.09067,0.00201,-0.05136,-0.04285,-0.02123,-0.02951,-0.03727,-0.03569,-0.00046,-0.01406,0.05342,-0.00236,-0.0192,-0.00725,0.00241,0.0394,-0.02234,-0.01723,0.01885,0.00991,-0.02232,-0.03134,-0.01029,0.04766,-0.01141,-0.02619,0.04905,-0.01742,-0.0156,-0.06988,0.02335,-0.03614,0.01286,0.0327,-0.02571,-0.06076,-0.01087,-0.02113,0.01315,-0.03591,0.02044,-0.01217,-0.03432,-0.00891,-0.00375,0.01765,0.00592,0.02714,-0.01618,-0.00579,0.00692,-0.02262,-0.04021,0.00358,0.01012,-0.03977,0.01806,0.00885,0.0413,-0.04,-0.02114,-0.00548,-0.0637,0.03599,-0.01827,0.05482,-0.01294,0.02388,-0.01964,-0.01468,-0.0318,0.0453,-0.07325,0.00374,0.04224,-0.01943,-0.01983,-0.00196,0.05563,-0.00797,0.01338,-0.00046,-0.03666,-0.00474,-0.0165,-0.00462,-0.03112,0.00709,-0.06224,-0.03842,-0.04052,0.02456,-0.034,-0.0335,0.03863,0.02719,0.00488,-0.00227,-0.01322,0.02791,-0.01461,0.00433,0.04981,-0.04388,0.03438,-0.05131,-0.03741,-0.01007,0.01641,-0.02147,-0.02365,-0.00543,0.04933,-0.01421,0.00839,0.01756,0.00882,0.03674,-0.01625,0.05141,0.03255,0.02371,0.00419,-0.04884,0.0134,0.02299,0.03823,0.02117,0.01411,-0.04217,0.04015,0.00781,0.0088,-0.03898,0.03052,0.01402,-0.02349,-0.01019,0.05048,-0.03606,-0.00628,0.02164,-0.02057,0.04572,-0.02141,0.00474,-0.02469,-0.0389,0.00954,-0.03281,0.04934,0.01655,-0.03643,0.01387,0.04636,-0.00695,0.00747,0.02263,-0.03546,0.00307,0.0339,0.02223,0.03965,-0.0283,-0.01036,-0.00708,-0.01042,-0.04451,0.0087,-0.01809,-0.04373,0.021,0.01272,0.01337,0.00774,0.00897,0.02052,0.0461,-0.00381,0.03194,-0.00241,0.03502,-0.02532,-0.04624,0.00254]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."],"revision":"3d7cfbdacd47fdda877c5cd8a79fbcc4f2a574f3"},"sentence-transformers/all-roberta-large-v1":{"embeddings":[[-0.00491,0.0044,-0.01892,-0.00247,-0.07814,-0.06722,0.00618,0.04673,-0.02171,0.02503,-0.00043,0.0242,0.01465,0.03645,-0.03645,-0.0303,0.0148,-0.02533,-0.05983,-0.05134,0.01746,0.07743,0.04035,0.06142,-0.03255,-0.02727,-0.01221,0.00277,-0.00847,-0.02716,0.01255,0.00519,0.02993,-0.01681,-0.00143,-0.03417,-0.0274,-0.00213,-0.01146,-0.00556,0.01481,0.0074,-0.03304,0.03185,-0.01974,0.00986,-0.01792,0.04463,0.05752,-0.03997,0.07135,-0.00506,-0.02994,0.00729,-0.02176,-0.05994,0.0436,-0.06935,0.05496,0.06513,0.00618,0.03998,0.00477,0.00872,-0.01157,0.02082,0.00239,0.01787,-0.02335,0.00173,0.02505,-0.0253,-0.0048,-0.05145,-0.01483,-0.02045,0.02061,0.03904,-0.0242,0.03605,0.00746,0.05673,-0.02943,-0.03818,0.0057,-0.00487,-0.01773,0.03736,0.04141,-0.01267,0.01949,-0.09329,-0.0586,-0.01511,0.01865,0.00241,0.00344,-0.02159,-0.02566,0.03557,0.004,0.04771,0.03518,0.02876,-0.01231,-0.02624,0.00157,0.02732,-0.02482,0.00629,-0.03364,-0.02953,-0.01454,0.00136,-0.02454,0.02143,-0.02548,0.00873,0.00365,0.04238,0.003,0.03508,-0.00857,-0.01997,-0.00211,-0.03049,-0.0091,0.04757,0.00166,0.0182,0.0371,0.01116,0.02716,-0.02996,0.03532,-0.00068,0.01254,0.04435,0.01503,-0.01619,-0.0509,-0.04084,-0.02074,0.01607,0.01155,0.05921,0.02744,-0.01229,-0.00266,0.00486,-0.00701,0.01625,-0.04082,-0.00256,-0.00917,0.0095,0.00071,0.0086,0.01242,0.00685,-0.00454,-0.0298,-0.00142,-0.04007,-0.04089,-0.02946,-0.03479,0.01798,0.06451,0.04752,0.01171,0.02314,0.01749,-0.00481,0.01423,0.01982,0.0163,-0.01267,0.01271,0.03126,0.03253,-0.03753,-0.00359,-0.0126,-0.02904,0.00883,0.0046,0.01535,-0.01568,-0.00207,-0.03463,-0.02127,0.03125,-0.02658,0.00492,-0.04955,-0.00759,0.0269,-0.08531,0.01391,-0.01611,0.04433,0.01175,-0.00047,0.03434,0.00643,0.01327,-0.03346,0.01086,0.00132,0.00855,0.04931,-0.03122,-0.06766,-0.00312,0.01679,0.02908,0.00318,0.00156,-0.05245,-0.01896,0.01812,0.00441,0.00719,0.0047,-0.00169,0.01055,0.03316,-0.02826,-0.06428,0.00171,0.06791,-0.01168,0.02612,-0.03324,0.03362,-0.03644,0.02252,0.01491,0.04821,-0.02506,-0.04503,-0.04295,0.0499,0.0426,-0.05623,-0.0477,0.02001,-0.03126,-0.02243,0.00375,-0.0028,-0.03933,0.03446,0.00091,0.01792,-0.0067,0.00154,0.00762,0.02367,0.05458,-0.0094,-0.00325,0.03178,0.02171,0.04649,-0.01136,-0.00953,0.03607,-0.02213,-0.00294,0.01412,-0.00882,-0.01808,0.01598,-0.02923,0.02937,-0.04259,-0.05132,0.03444,-0.02586,-0.01684,-0.01645,-0.00298,-0.05119,-0.01414,-0.01932,0.05487,0.00152,-0.01417,0.03513,-0.03364,-0.04596,-0.01979,-0.00108,0.01705,0.03152,-0.01226,0.11015,-0.02259,-0.00777,-0.01998,-0.02853,0.02265,0.00672,0.05379,-0.03633,-0.04947,-0.04784,-0.06021,0.00587,-0.04636,0.00057,0.05334,0.01505,0.04036,-0.04575,-0.01661,-0.02811,0.02913,0.05085,-0.02248,0.02317,0.02474,-0.01465,0.03192,-0.05312,-0.00058,-0.01229,0.00217,0.02721,0.03574,0.03714,-0.01278,-0.01241,0.02278,-0.04196,-0.04708,-0.04374,-0.01287,-0.00865,-0.01168,0.0415,-0.04438,0.0311,0.02402,0.00413,-0.02244,-0.01726,0.06536,0.00535,0.01371,-0.02013,0.01311,-0.02689,0.00247,-0.02445,-0.03718,0.00307,0.00774,0.00495,-0.00145,-0.02505,-0.06807,0.05282,-0.01866,0.0034,0.06462,0.02268,-0.01834,0.00437,0.03333,-0.03028,-0.00205,-0.01992,0.00291,-0.00312,0.02033,0.01145,0.04449,0.07244,-0.01434,0.01164,-0.01788,-0.00801,-0.02117,-0.03054,0.0146,0.01765,-0.04504,0.00633,0.0427,-0.01145,0.01272,-0.04937,0.03976,-0.02522,-0.0062,-0.02807,0.02329,0.04338,-0.00469,0.03403,0.00379,0.02278,-0.01415,-0.0505,0.00756,-0.05988,-0.04672,0.01345,0.03528,0.02346,-0.02717,0.02367,0.0109,0.03678,-0.00212,0.00956,-0.0298,0.00592,-0.03096,-0.02238,0.02461,0.00253,0.01747,0.02629,-0.01784,0.01282,-0.00496,0.02501,0.01547,-0.02617,0.00983,-0.00499,-0.01497,0.01642,-0.02056,0.01205,0.06724,0.01422,0.01044,0.02634,0.01067,0.01288,0.01041,-0.00132,-0.0265,-0.03179,-0.01864,0.02567,-0.02476,-0.01453,-0.02537,0.01648,-0.03386,-0.00404,-0.01197,0.02813,0.01349,-0.00103,0.0362,0.03468,-0.01717,-0.00592,0.03509,0.03049,-0.05479,0.02247,0.00641,0.01327,0.05331,0.00286,-0.04148,0.02662,-0.02991,-0.01189,0.03,-0.01124,-0.01759,0.07745,0.02257,0.00103,0.0138,-0.05063,-0.03517,-0.03791,0.00638,0.0025,0.02377,0.00432,-0.05742,0.03785,0.00496,0.0258,0.00969,0.01271,0.02467,0.00955,-0.02789,0.0023,0.00495,0.03569,-0.05401,0.06956,-0.00585,-0.01595,0.01418,0.01695,0.03314,0.00375,-0.0264,0.02842,-0.00581,0.06678,0.07884,0.02801,-0.00835,0.04728,-0.04482,0.00496,-0.03334,-0.00353,-0.01707,-0.02384,0.01986,-0.02282,-0.01359,0.03213,-0.00551,-0.0134,0.06444,0.0102,0.01717,0.036,0.0449,-0.05578,0.02189,-0.05393,0.05831,-0.00837,0.04365,-0.03047,0.04218,0.04157,0.04729,-0.05191,0.04156,0.00472,-0.00231,0.04871,0.01498,-0.01352,-0.00074,-0.05809,0.00414,-0.01312,0.02515,0.04828,-0.029,0.00185,-0.02006,0.03079,-0.02273,-0.00054,0.00124,-0.04775,-0.03523,0.06123,0.02369,0.03288,-0.00942,-0.02377,0.03404,-0.01834,0.03162,-0.04765,0.00311,-0.04369,0.01712,-0.0046,-0.02907,-0.0196,0.01807,-0.02918,0.02673,-0.01424,0.06018,0.02108,0.05122,0.05261,-0.0844,0.01294,-0.00105,0.01756,0.06813,0.01816,-0.01834,-0.01305,0.04269,0.02993,-0.05113,-0.00208,0.01542,0.05871,-0.01004,-0.00253,0.02197,-0.00239,0.04582,-0.02667,-0.02511,-0.00493,-0.03509,-0.01359,-0.00577,-0.00793,-0.08161,0.01071,-0.02469,-0.00321,-0.02586,0.01736,0.04034,0.03213,-0.0275,0.01922,0.04221,-0.01884,0.0041,0.05143,-0.04363,-0.02603,0.01359,-0.04034,-0.02981,-0.0028,-0.00479,0.02249,0.02391,-0.03841,0.05699,0.00909,-0.05007,0.01069,0.01308,0.01425,0.01592,-0.01341,-0.03907,-0.05654,-0.0339,-0.02548,0.01985,0.00424,0.00145,-0.02947,0.0172,0.02241,0.02027,-0.01031,-0.01832,-0.10862,0.01336,0.00109,-0.09422,-0.01585,0.03974,-0.00537,-0.01127,-0.05054,0.04651,-0.03548,0.01287,-0.00876,0.08663,-0.00255,-0.00923,-0.01168,-0.00829,0.02034,-0.03456,-0.00957,0.02267,0.01669,0.01538,0.00694,0.02215,-0.01303,0.01227,0.01423,-0.01576,0.02942,-0.02075,0.05806,0.00468,-0.07045,-0.03456,0.07849,-0.02087,0.03461,0.01593,0.01949,0.00697,0.02198,0.02225,0.01448,-0.03578,0.03795,-0.02227,-0.0208,-0.01368,0.08223,-0.02417,-0.02438,0.00678,0.05668,-0.04544,0.01442,-0.02251,-0.0069,0.03637,-0.03406,-0.01678,-0.00843,-0.03108,0.04477,0.01902,-0.02718,-0.01966,-0.04527,-0.03346,0.0193,-0.03969,0.02866,-0.04977,-0.0432,-0.0015,-0.06911,-0.00447,-0.02955,0.02724,-0.0512,-0.00296,-0.03071,0.00101,-0.01681,0.03946,0.03542,-0.00468,-0.02902,0.0628,0.00848,-0.00765,-0.02137,-0.06543,-0.00419,0.00129,-0.02534,0.04327,0.03223,-0.00096,0.034,0.03474,-0.04742,0.08326,-0.02815,0.02141,-0.03308,0.00944,-0.04045,-0.02303,-0.0151,-0.01923,-0.03251,-0.01503,-0.01013,0.05736,0.00123,0.04242,0.04085,-0.08235,-0.00573,-0.00472,0.01922,8e-05,-0.00753,-0.03246,0.06445,0.02563,0.03713,0.02073,0.00281,-0.01773,-0.00507,-0.0161,-0.05207,0.0207,-0.01122,0.00498,0.05607,0.02406,-0.0077,0.00524,0.00656,0.02172,-0.00582,0.002,0.0163,0.00321,0.00288,-0.04978,0.03,-0.0309,0.0502,-0.00924,0.0218,-0.02395,0.00244,0.00688,0.01555,0.02774,-0.00523,0.00513,-0.05242,0.00226,-0.0107,0.01536,-0.06243,-0.04384,-0.02613,-0.0093,0.00854,-0.05516,-0.04737,-0.01029,0.0075,0.01335,0.00357,-0.0241,0.04834,-0.03318,0.00376,0.04555,0.01395,0.02176,0.00248,-0.00834,-0.0281,-0.01321,0.01123,0.00135,-0.0306,0.00553,-0.02512,-0.07398,-0.01681,0.02158,-0.01458,0.03779,-0.02546,0.01265,0.03814,0.00501,0.01211,-0.00943,0.02332,-0.00755,-0.01952,0.06056,-0.0167,0.02276,0.00015,0.03737,0.00537,0.02031,-0.03587,-0.002,-0.00012,0.02132,0.07326,-0.05212,-0.01719,0.02474,-0.01287,-0.05148,-0.02772,0.00794,0.01174,-0.0036,0.04666,-0.07608,-0.02778,0.03752,0.03669,0.03921,-0.04757,0.03959,0.05085,0.00134,-0.00052,0.008,-0.05218,-0.05433,-0.03486,0.00802,-0.06204,-0.03135,0.02423,0.00268,0.00185,-0.04034,-0.0165,-0.03715,-0.01869,0.00485,0.04019,0.03224,0.03855,0.04126,-0.02699,-0.01076,0.03594,-0.02552,0.00764,0.01708,0.00122,0.02882,0.06046,0.02527,0.0274,0.00398,0.00739,-0.02137,-1e-05,0.00693,0.03125,-0.01413,-0.00583,0.05256,-0.02371,0.0205,0.03769,0.048,0.02885,-0.06224,0.02187,-0.02163,0.04458,0.0028,0.04348,-0.03598,-0.0406,0.00882,0.04196,-0.05882,-0.02998,-0.03239,-0.02962,0.01559,0.00703,-0.06205,-0.0125,0.02488,-0.01178,0.0424,0.02942,0.00555,-0.0195,-0.01782,0.0299,0.01069,-0.05646,0.00072,-0.01938,-0.02975,-0.00091,-0.05657,0.0558,0.00578,-0.04121,-0.00179,0.04888,0.01047,0.00592,0.00888,0.02142,0.01432,0.01216,-0.01609,0.03247,-0.0326,0.03299,-0.03135,0.04498,0.04766,0.03267,0.06434,0.01058,0.02177,-0.03881,0.01728,-0.03258,-0.03123,-0.05876,-0.03556,-0.03094,0.03852,0.02517,-0.01034,-0.02895,-0.00488,0.02061,0.00195,-0.00882,-0.02741,0.02004,0.00786,-0.0326,0.00172,0.00652,0.01467,-0.02109,-0.01082],[-0.00206,0.04185,-0.04616,-0.03325,-0.01363,0.0082,0.01742,0.05534,0.01351,-0.01256,0.0137,-0.03019,0.00824,-0.03269,-0.01796,0.03263,-0.00355,0.0102,0.01736,0.00741,0.00189,-0.06432,-0.04325,0.04347,0.05882,-0.00604,0.02799,0.05457,-0.04737,-0.03622,0.02471,0.08941,-0.01274,0.02435,0.01284,0.03211,-0.01611,-0.03103,-0.01731,0.00477,0.01518,0.00549,-0.02322,0.00771,0.03693,-0.01392,-0.06142,0.03588,0.01451,0.00468,-0.01784,-0.01676,0.01256,-0.00215,0.00603,0.04483,-0.00245,0.01215,0.06297,-0.00953,0.05738,0.01576,0.04132,0.02367,0.00396,0.01046,-0.02265,-0.00082,0.01129,0.00746,0.06808,0.02221,-0.00141,-0.00015,-0.05808,0.0123,-0.00018,-0.04353,-0.04387,0.01658,0.0369,0.01736,-0.04124,-0.00343,-0.00814,-0.00035,0.01998,-0.0409,-0.01798,-0.07698,0.00107,0.01193,0.02141,-0.00631,-0.02103,0.02256,-0.00209,-0.03015,-0.00204,-0.05704,-0.05389,0.02812,0.07857,0.00886,-0.03011,0.0054,0.02466,-0.00949,0.05027,-0.0005,-0.0035,0.0802,-0.02775,-0.06988,-0.01802,0.00885,0.03015,-0.02436,-0.01613,0.01159,-0.0055,-0.03583,0.03152,0.02712,-0.02054,-0.01463,-0.10985,-0.02312,-0.00278,0.04664,0.04068,0.02423,0.01606,-0.03462,0.03663,0.00702,0.03483,-0.01558,0.01751,-0.04317,-0.00442,-0.0077,-0.01749,-0.07771,0.0298,-0.02019,0.00635,-0.00566,0.05242,0.00437,-0.04975,-0.01857,-0.02515,-0.03745,-0.01927,-0.01457,-0.01866,-0.03556,-0.00124,-0.05705,0.03255,0.03204,-0.0137,0.0852,0.03619,-0.00916,-0.02803,0.01214,-0.01861,-0.01464,0.0049,0.01991,0.06227,-0.01888,0.01824,-0.00231,0.02944,0.04604,-0.0331,0.02263,0.0051,-0.00735,0.00551,0.0072,0.03683,0.0038,0.02445,0.00724,-0.0013,0.04939,-0.02201,-0.00442,0.02748,-0.0205,-0.02536,0.01176,-0.06234,0.02924,-0.04934,0.08147,0.00588,-0.03138,0.03934,0.02914,0.04506,-0.00323,-0.04797,-0.03986,-0.00735,0.05942,-0.02488,-0.02243,-0.07256,0.02463,0.00706,0.00536,0.00423,0.04708,0.04336,-0.00569,-0.02316,0.02415,-0.02535,0.00019,0.01234,0.01061,-0.00099,0.09205,0.00833,0.00127,0.0048,-0.04443,-0.00441,0.03757,-0.02662,0.004,0.03018,-0.00481,0.02466,0.06229,0.00082,-0.00233,-0.03372,-0.00677,-0.01455,0.05469,0.02689,-0.0046,-0.05951,-0.00254,0.00873,0.06474,-0.00062,-0.00091,0.00322,-0.00839,-0.05954,-0.04145,-0.03907,0.00237,0.04903,0.03624,0.0301,-0.00821,0.0074,0.01214,-0.00045,-0.01308,0.02098,-0.03085,0.00782,0.04569,0.01247,-0.00435,0.01807,-0.01932,-0.00761,0.01233,-0.00909,-0.01043,-0.00562,-0.05074,0.01968,0.06434,0.00438,-0.0325,-0.01599,-0.00363,0.00043,0.00717,-0.00278,-0.04382,-0.03562,-0.03171,-0.08286,0.00014,0.01623,-0.02824,-0.03717,0.00498,-0.04826,-0.00263,-0.05056,-0.04194,-0.06332,0.06747,0.00319,-0.0241,0.08014,0.04745,0.03,-0.02278,0.01141,0.00233,-0.05887,-0.0393,0.00773,-0.04164,0.05626,-0.03491,0.03835,-0.05679,-0.0455,0.0033,0.05183,-0.02316,0.00121,-0.01228,0.03183,0.00488,0.01489,0.0265,0.03702,0.03868,-0.00911,0.06631,-0.01747,-0.03507,0.00137,-0.00545,-0.04762,0.01688,-0.01308,-0.00574,-0.03176,0.0231,-0.03408,-0.02792,-0.04425,0.05368,0.02461,-0.00494,-0.03715,-0.02371,-0.00814,-0.00642,-0.03314,0.01627,-0.00674,0.0027,-0.00055,0.04965,-0.01034,-0.0039,0.0244,-0.01923,0.02178,0.03745,-0.0033,-0.00019,-0.01738,-0.01694,-0.01221,-0.00066,-0.03128,-0.0155,0.03564,0.02341,-0.02074,-0.00488,0.017,0.01026,-0.08508,-0.01792,-0.03493,-0.01157,0.03419,-0.00872,-0.01336,-0.0178,0.0033,0.02473,-0.01584,-0.00609,0.07753,-0.05776,-0.09227,0.00851,0.00028,0.02119,0.09262,0.02519,0.01116,0.01204,0.00134,0.00788,0.00674,-0.00923,-0.02457,-0.04842,-0.01902,0.01435,0.00651,0.04692,0.02628,-0.02251,-0.04578,0.03277,0.03743,-0.0183,-0.0186,-0.03579,-0.04275,-0.00343,0.03249,0.01309,0.00876,-0.04225,-0.01847,-0.01134,0.00493,-0.02431,0.00356,0.00173,-0.0063,-0.00407,-0.05433,-0.0562,0.04584,0.02586,-0.04907,0.01275,-0.01099,-0.01046,0.02472,0.00286,-0.0039,-0.03374,0.00101,0.01051,0.01628,-0.00488,-0.07494,-0.02079,0.0633,-0.01488,-0.00093,-0.02143,-0.02613,0.00891,-0.0145,0.0108,0.01447,0.01659,-0.01807,-0.02359,-0.02848,-0.02412,0.05532,0.07516,-0.02723,0.02305,-0.04249,-0.03747,-0.03154,-0.0061,0.03429,-0.00176,-0.02662,0.02845,-0.01058,0.07094,-0.0081,-0.02039,-0.04784,0.04751,-0.01331,-0.02918,-0.02033,0.07103,0.0086,0.01157,0.0155,-0.02949,0.03427,0.00323,0.03064,0.03099,-0.02994,-0.02622,0.05816,0.07138,-0.01573,-0.03923,0.02165,-0.03942,-0.04769,0.02503,-0.06287,0.00134,-0.035,-0.01707,0.00082,0.00742,-0.00899,0.02446,-0.03779,-0.04808,0.01033,0.01803,-0.02838,-0.00682,-0.01022,-0.03513,-0.00151,0.03024,-0.01573,0.01315,-0.00141,0.01937,0.01113,0.03308,0.00821,0.03881,0.05997,0.02937,0.00052,0.03524,0.01438,-0.00951,0.0479,0.01645,0.02437,0.04108,-0.04179,0.01595,0.00915,0.01474,-0.01479,-0.01138,-0.00546,-0.00542,-0.02009,0.01217,-0.00839,-0.00528,0.03692,-0.02462,0.04018,0.01027,-0.02305,0.01413,-0.03221,-0.01619,-0.00366,0.00257,-0.01686,-0.01958,0.04608,-0.04776,-0.0189,-0.01264,-0.01905,0.00327,-0.00213,0.01189,-0.01534,0.04735,-0.02226,0.02636,0.00891,-0.02736,-0.0681,0.01168,-0.03668,0.04577,0.0606,-0.00887,0.02025,0.00978,-0.01444,0.01215,-0.03604,0.02607,-0.01037,0.00013,0.01262,0.00477,-0.01335,0.00707,-0.00663,-0.03453,-0.01628,0.01052,0.00832,-0.06043,0.03126,0.03244,0.00244,0.0503,-0.00291,0.0403,0.0198,-0.02373,0.03449,0.04352,0.02152,-0.04059,0.01888,-0.01627,-0.03998,0.02578,-0.0159,-0.00028,-0.0416,0.00671,0.00205,0.02315,0.00656,-0.00318,-0.01772,-0.02864,0.02008,-0.0263,0.01811,-0.0233,0.01575,-0.04911,-0.01261,0.02936,0.00213,0.02135,0.03473,-0.04622,0.04333,-0.00737,-0.03358,0.0227,0.01581,-0.00803,0.04156,0.01076,-0.01992,0.04343,-0.01923,-0.04269,-0.01079,0.01713,-0.03031,0.00322,0.02112,0.00213,-0.02852,0.04839,0.04973,0.05696,0.05122,-0.02071,0.00917,0.0374,0.00752,0.05741,0.01297,-0.0085,0.05919,-0.09915,-0.00748,-0.02673,-0.05797,-0.04767,0.05608,0.00457,0.03995,0.00935,-0.0533,0.02858,-0.00054,0.06385,0.00376,0.01214,0.05129,-0.01525,0.02155,0.01229,0.01694,-0.02441,-0.05561,0.02383,0.05626,-0.01144,-0.00459,0.00665,-0.04026,-0.02969,-0.00016,-0.03639,-0.0002,0.01423,-0.01365,0.0454,0.02831,0.04642,0.00878,0.10063,-0.04216,0.04849,-0.01032,-0.04384,-0.01958,0.03792,-0.0294,0.04493,-0.01516,-0.02431,0.01484,-0.05908,-0.0253,-0.00734,-0.0294,-0.0166,-0.01069,-0.01722,-0.01005,0.03595,0.00291,-0.05385,-0.02243,0.04624,-0.02963,0.0159,0.03085,-0.03689,-0.01771,0.01029,-0.01832,-0.02373,-0.04424,0.03274,-0.00124,-0.029,0.00258,0.00913,-0.00039,0.01209,0.03608,0.00296,0.02905,-0.01307,0.0118,-0.01356,0.00902,-0.04151,0.01593,-0.06927,0.02683,-0.02332,0.00528,0.07809,0.04598,0.00329,-0.01174,-0.02161,0.0144,0.03986,-0.00577,-0.00054,0.01055,0.10328,-0.00854,0.04123,-0.0281,-0.01627,-0.00809,0.00506,-0.01148,-0.00705,-0.02379,-0.00815,0.01779,-0.02028,0.01839,0.00314,0.00357,-0.00945,0.0606,-0.02172,-0.00588,-0.00081,-0.00645,-0.0295,0.0403,0.02714,0.03324,-0.02197,0.02591,0.07449,-0.02155,-0.03062,-0.01114,-0.02688,-0.00346,-0.03228,-0.03849,0.00818,-0.02564,0.01415,-0.01518,0.00811,-0.0192,-0.03064,-0.0161,-0.01336,-0.01244,-0.03178,0.03565,0.01717,0.02071,-0.03996,-0.04877,-0.04514,-0.00076,0.03818,-0.0116,-0.0291,-0.02425,-0.02174,0.02149,0.00177,0.02188,-0.01879,0.04391,-0.00093,0.01344,0.05639,-0.03074,-0.01048,0.02899,-0.01762,0.01265,-0.04332,-0.00886,0.04306,-0.02101,-0.04151,-0.02396,0.03238,0.00911,0.02717,0.01738,-0.01189,-0.03267,0.00315,-0.03517,-0.05413,0.0594,0.00687,0.02194,0.05496,-0.05165,-0.00918,0.05983,-0.01215,-0.01962,-0.03941,-0.00014,-0.01077,0.00628,0.02216,-0.01699,-0.04789,0.01132,-0.03225,0.00092,-0.01141,-0.01038,-0.05065,0.03999,-0.00573,0.06075,0.03056,0.0105,0.01771,-0.03294,-0.00462,0.02377,0.0391,-0.04897,0.0438,0.02617,8e-05,0.00045,-0.03178,-0.03441,-0.05191,-0.02425,-0.01991,-0.02728,0.03142,0.01688,0.00766,0.0004,-0.01117,-0.03277,-0.01741,-0.00079,-0.01226,0.06248,0.03794,-0.03267,0.01789,0.00371,0.01055,-0.04564,-0.03566,-0.01644,0.01897,-0.00794,-0.03457,-0.00889,-0.03042,-0.01716,-0.0378,-0.05054,-0.02853,0.02834,-0.00845,0.03893,0.02269,0.05895,0.04078,0.01726,-0.01072,-0.00909,0.05516,0.00596,0.02647,-0.00682,0.01746,0.03387,0.00924,0.06054,0.01094,-0.0156,-0.05887,-0.01205,0.04112,0.00109,0.05043,-0.03037,-0.01829,-0.00996,-0.00122,0.00906,-0.01136,-0.00464,0.01854,-0.06368,-0.01129,-0.0293,0.002,0.00344,0.00424,-0.00762,-0.02214,-0.01772,0.04501,-0.04189,-0.00136,-0.02585,0.00197,0.00762,-0.0319,0.01055,0.0245,-0.00664,-0.03415,0.02254,0.03979,-0.03625,-0.02966,0.04111,0.01457,0.02157,0.02104,-0.00403,0.00085,-0.00641,0.04651,-0.0203,0.01873,0.02191,0.02907,-0.03252,-0.03089,0.09486,0.02133,0.04798,-0.01348,-0.00205,-0.0566,-0.02041,0.03671,-0.01997,-0.01283,-0.00655,0.0332,0.04795,-0.01107,-0.01457,0.01461,0.01305,0.05409,-0.01398,0.01082]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."],"revision":"cf74d8acd4f198de950bf004b262e6accfed5d2c"}} \ No newline at end of file diff --git a/tests/data/generate_encoder_embed_refs.py b/tests/data/generate_encoder_embed_refs.py index 75452eb53..06672b923 100644 --- a/tests/data/generate_encoder_embed_refs.py +++ b/tests/data/generate_encoder_embed_refs.py @@ -31,6 +31,19 @@ "intfloat/multilingual-e5-large", "sentence-transformers/all-roberta-large-v1", ] + +# Pinned so a reference keeps meaning one specific set of weights. Unpinned, an upstream +# re-upload silently redefines what the cosine gate compares against, and the test cannot +# tell that from a regression on the Spyre side. Written into the JSON per model and read +# back by the test, so the gate loads the same revision this file measured -- do not pin +# here without regenerating. +MODEL_REVISIONS = { + "ibm-granite/granite-embedding-125m-english": "4ab61ffd423be45cd932b21a7c696063d82bf45f", + "ibm-granite/granite-embedding-278m-multilingual": "a9cb5338491faf32b73dd17b714a31821c021bbf", + "intfloat/multilingual-e5-large": "3d7cfbdacd47fdda877c5cd8a79fbcc4f2a574f3", + "sentence-transformers/all-roberta-large-v1": "cf74d8acd4f198de950bf004b262e6accfed5d2c", +} + EMBEDDING_PROMPTS = [ "Hello world.", "The quick brown fox jumps over the lazy dog.", @@ -44,10 +57,12 @@ def main() -> None: data: dict[str, dict] = {} for model in EMBEDDING_MODELS: - print(f"Encoding {model} ...") - st = SentenceTransformer(model, device="cpu") + revision = MODEL_REVISIONS[model] + print(f"Encoding {model} @ {revision} ...") + st = SentenceTransformer(model, revision=revision, device="cpu") embeddings = st.encode(prompts, normalize_embeddings=True) data[model] = { + "revision": revision, "prompts": prompts, "embeddings": [[round(float(x), _ROUND) for x in row] for row in embeddings], } diff --git a/tests/data/generate_rerank_score_refs.py b/tests/data/generate_rerank_score_refs.py index 6099e574e..5379d2922 100644 --- a/tests/data/generate_rerank_score_refs.py +++ b/tests/data/generate_rerank_score_refs.py @@ -44,7 +44,13 @@ # One query against documents a reranker should separate widely, listed most relevant # first. The spread is what lets the test gate the *ranking* and not only the individual # scores, so neighbouring documents have to stay far enough apart that fp16 drift cannot -# swap them: every adjacent pair below differs by at least a factor of 8 in odds. +# swap them. What has to clear that is the gap against the drift the two scores can absorb +# between them -- min(7e-3, 13% * p) each, the worst measured in test_encoder_models.py -- +# and not a ratio of scores or of odds: odds explode near p=1, where drift is absolute, and +# collapse onto the score ratio near p=0, where it is relative, so either ratio rates the +# wrong pair as the risky one. The tightest pair below clears its combined drift by 2.8x +# (the top two, 0.99997 vs 0.96147: a gap of 0.039 against 0.014 of drift); every other +# pair clears it by 6x or more. QUERY = "What is the capital of France?" DOCUMENTS = [ "The capital of France is Paris.", @@ -57,8 +63,11 @@ # bge-reranker-large saturates: it scores every Paris-adjacent document above 0.9994, so on # the list above its top five land within 5e-4 of each other and their order is decided by -# fp16 noise rather than by relevance. Give it documents spread across the range it -# actually resolves. Drop this entry only for a model that separates the shared list. +# fp16 noise rather than by relevance -- the same drift bound, failed. Give it documents +# spread across the range it actually resolves; the list below clears that bound by 4.4x at +# its tightest (0.00187 vs 0.00052, a gap of 1.4e-3 against 3.1e-4 of drift), so it is the +# safer of the two despite sitting lower. Drop this entry only for a model that separates +# the shared list. MODEL_DOCUMENTS = { "BAAI/bge-reranker-large": [ "The capital of France is Paris.", diff --git a/tests/e2e/test_encoder_models.py b/tests/e2e/test_encoder_models.py index d8cc3eb84..836555b86 100644 --- a/tests/e2e/test_encoder_models.py +++ b/tests/e2e/test_encoder_models.py @@ -100,12 +100,12 @@ def _cosine(a: list[float], b: list[float]) -> float: ).item() -def _hf_last_token_embeddings(model: str, prompts: list[str]) -> list[list[float]]: +def _hf_last_token_embeddings(model: str, revision: str, prompts: list[str]) -> list[list[float]]: """CPU HF last-nonpad-token + L2 (matches vLLM LastPool + normalize).""" from transformers import AutoModel, AutoTokenizer - tok = AutoTokenizer.from_pretrained(model) - hf = AutoModel.from_pretrained(model) + tok = AutoTokenizer.from_pretrained(model, revision=revision) + hf = AutoModel.from_pretrained(model, revision=revision) hf.eval() with torch.inference_mode(): enc = tok( @@ -145,6 +145,8 @@ def _assert_embeddings_match_refs(model: str, enforce_eager: bool) -> None: prompts = ref["prompts"] llm = LLM( model=model, + revision=ref["revision"], + tokenizer_revision=ref["revision"], runner="pooling", max_model_len=64, max_num_seqs=1, @@ -180,6 +182,8 @@ def test_encoder_embed_mean_multi_seq(model: str) -> None: prompts = ref["prompts"] llm = LLM( model=model, + revision=ref["revision"], + tokenizer_revision=ref["revision"], runner="pooling", max_model_len=64, max_num_seqs=2, @@ -209,11 +213,17 @@ def test_encoder_embed_last_pooling() -> None: override exercises the LAST gather + normalize path that ``configure_pooling_for_spyre`` patches to ``SpyreLastPool``. """ + # Taken from the embed reference entry rather than hardcoded, so this case cannot drift + # onto a different revision than the embed gates measure. Both sides are computed in + # the same run here, so the pin buys reproducibility, not a valid comparison. + revision = _REFERENCES[LAST_POOLING_MODEL]["revision"] prompts = LAST_POOLING_PROMPTS - ref_embs = _hf_last_token_embeddings(LAST_POOLING_MODEL, prompts) + ref_embs = _hf_last_token_embeddings(LAST_POOLING_MODEL, revision, prompts) llm = LLM( model=LAST_POOLING_MODEL, + revision=revision, + tokenizer_revision=revision, runner="pooling", max_model_len=64, max_num_seqs=1, @@ -252,6 +262,16 @@ def test_encoder_rerank_models_compiled(model: str) -> None: def _assert_rerank_scores_match_refs(model: str, enforce_eager: bool) -> None: + """What runs on Spyre here is the encoder body, not the score itself. + + A reranker's classifier head stays float32, and torch-spyre has no FP32 batchmatmul + (torch-spyre#1794), so ``configure_pooling_for_spyre`` sends the whole pooling tail + through ``run_pooling_tail_on_cpu`` -- logged as "FP32 classifier/head unsupported on + Spyre ... running pooler on CPU" at load, in the compiled case as much as the eager + one. So the bounds below gate the transformer blocks and the CLS gather, and a + regression confined to the head or the sigmoid would pass both. Extending the gate to + it needs the head on device, not another tolerance. + """ ref = _RERANK_REFERENCES.get(model) if ref is None: pytest.skip(f"No HF ref for {model}; run tests/data/generate_rerank_score_refs.py") From 6d412e4ebfa7bd70cdb4a3c4aa9ab2fd23cd6c1e Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Mon, 7 Sep 2026 13:49:24 +0200 Subject: [PATCH 13/18] cleanup Signed-off-by: Yannick Schnider --- .github/workflows/_test_matrix.yaml | 6 +- .pre-commit-config.yaml | 5 +- Makefile | 24 +++----- docs/contributing/README.md | 59 ++++++++----------- tests/data/generate_decoder_output_refs.py | 9 +-- tests/data/generate_encoder_embed_refs.py | 7 +-- tests/data/generate_rerank_score_refs.py | 23 ++------ tests/e2e/test_encoder_models.py | 32 ++++------- tests/e2e/test_model_quality.py | 67 +++++++--------------- 9 files changed, 77 insertions(+), 155 deletions(-) diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml index d7b4214b6..75b2b43c7 100644 --- a/.github/workflows/_test_matrix.yaml +++ b/.github/workflows/_test_matrix.yaml @@ -282,10 +282,8 @@ jobs: - linux image_label: image_torch_spyre test_target: test-smoke-shard-7 - # Product-model output-quality gate. Carved out of the smoke shards - # above and deliberately NOT in `integration`: every case loads and - # compiles a product model, from the 125M encoders up to the 31B - # decoders, too slow for the tier the multi-repo integration hook runs. + # Product-model output-quality gate: its own job and not in + # `integration`, since every case compiles a product model. - cfg: Spyre model quality tests test_types: regression trunk runs_on: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ebed7266..299414fc5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,9 +16,8 @@ repos: rev: v1.41.0 hooks: - id: typos - # Reference data holds tokenizer fragments -- "Gather" arrives as " G"+"ather", - # "Machines" as " M"+"achines" -- which the hook's --write-changes silently rewrites - # into real words. + # Reference data holds tokenizer fragments, which split words mid-token and which + # this hook's --write-changes silently rewrites into real words. exclude: '^tests/data/.*\.json$' - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.45.0 diff --git a/Makefile b/Makefile index 0bcf8ddda..3c5c829d1 100644 --- a/Makefile +++ b/Makefile @@ -96,19 +96,15 @@ else ifeq ($(TEST_TYPE),perf) MARK_EXPR := else ifeq ($(TEST_TYPE),integration) # Single-invocation integration = the CI smoke suite, which now also carries the -# compiled (enforce_eager=False) tests/e2e/test_compile.py cases. Probes and the -# model-quality gate are excluded here just as the sharded smoke jobs exclude -# them, each running in its own CI job: probes must not gate integration on -# strict-xfail flips, and the model-quality gate compiles a product model per case -# -- encoders from 125M up to the 31B decoders -- far too slow for the integration -# tier. +# compiled (enforce_eager=False) tests/e2e/test_compile.py cases. Probes are +# excluded here just as the sharded smoke jobs exclude them (they run in their +# own test-probes job and must not gate integration on strict-xfail flips), and +# the model-quality gate likewise has its own job: every case compiles a product +# model, up to the 31B decoders. MARK_EXPR := -m "not (distributed or upstream or attention or probe or model_quality)" else ifeq ($(TEST_TYPE),unit) -# Mirrors the CI jobs the `unit` tier schedules (_test_matrix.yaml test_types): -# the smoke shards (which now carry the compiled e2e cases), the attention shards, -# encoder-attention, the distributed shards and the probe shards -- so probes stay -# in here, unlike the integration tier above. model_quality is regression/trunk -# only there, so it stays out here too. +# model_quality is scheduled regression/trunk only (_test_matrix.yaml), so it stays +# out of the unit tier as well. MARK_EXPR := -m "not (upstream or model_quality)" else # The validation above already rejected any type outside VALID_TEST_TYPES, so @@ -196,10 +192,8 @@ test-smoke-shard: ## Run one smoke shard (SMOKE_SHARDS=N SMOKE_SHARD_ID=i). test-smoke-shard-%: $(MAKE) test-smoke-shard SMOKE_SHARD_ID=$* JUNIT_XML=$(JUNIT_XML) -# Carved out of smoke (and so out of integration): every case loads and compiles a -# product model -- a 125M-580M encoder or an 8B-31B decoder -- and the decoders alone -# are more than the smoke shards' runtime budget can absorb. -test-model-quality: ## Run the product-model output-quality gates (its own job; compiles 125M encoders up to 31B decoders). +# Carved out of smoke: every case compiles a product model, up to the 31B decoders. +test-model-quality: ## Run the product-model output-quality gates against the cached HF references. $(MAKE) run-one MARK_OVERRIDE='model_quality and not (distributed or upstream)' JUNIT_XML=$(JUNIT_XML) test-probes: ## Run the torch-spyre backend probes (excluded from integration), unsharded (local full run). diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 6691a28d8..36229d670 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -85,18 +85,17 @@ pytest --upstream -m "attention" #### Model Output Quality Gate The `model_quality` marker gates the product models on their output: each is loaded -**compiled** (the platform default) and compared against a CPU HF reference ā€” greedy -token ids plus per-token probabilities for the decoders -(`tests/e2e/test_model_quality.py`), cosine similarity for the embedding models and -sigmoid scores plus document ranking for the cross-encoder rerankers -(`tests/e2e/test_encoder_models.py`). +**compiled** (the platform default) and compared against a CPU HF reference ā€” greedy token +ids and per-token probabilities for the decoders (`tests/e2e/test_model_quality.py`), +cosine similarity for the embedding models and sigmoid scores plus document ranking for +the cross-encoder rerankers (`tests/e2e/test_encoder_models.py`). ```bash make test-model-quality # the whole gate, one card ``` -The models are too large to run through transformers in CI, so the references live in -`tests/data/` and are regenerated by hand where the weights are cached: +The models are too large to run through transformers in CI, so the references are checked +into `tests/data/` and regenerated by hand where the weights are cached: ```bash python tests/data/generate_decoder_output_refs.py --models ibm-granite/granite-4.1-8b @@ -104,36 +103,26 @@ python tests/data/generate_encoder_embed_refs.py python tests/data/generate_rerank_score_refs.py ``` -Regenerate only when the *expected* output changes (new model or prompt), never to make a -failing test pass ā€” that is the regression the gate exists to catch. Prompts are per -model: `MODEL_PROMPTS` (decoders) and `MODEL_DOCUMENTS` (rerankers) in the generators say -which models get their own inputs and why. - -Every gated model is pinned to a revision, in the generator's `MODEL_REVISIONS` and again -in `.github/cache_config/hf_models_and_datasets.yaml`. Each generator writes the revision -it measured into its JSON, and the test loads that revision back ā€” so a reference always -names the weights it was taken from, and an upstream re-upload cannot silently redefine -what the gate compares against. Bumping a pin means regenerating that model's reference. -`SPYRE_TEST_ABS_TOL` (default `0.08`) and `SPYRE_TEST_REL_TOL` (default `0.5`) set the -decoder probability tolerance: the stricter of the two applies, so a low-confidence -reference token is held to a fraction rather than to the same absolute margin. -`SPYRE_TEST_SCORE_ABS_TOL` (default `0.03`) and `SPYRE_TEST_SCORE_REL_TOL` (default -`0.5`) bound a reranker score the same stricter-of-the-two way, which matters more here -than for the decoders because the scores are sigmoids sitting near the rails. The -document ranking is checked separately: all scores can drift the same direction without -reordering anything, and two documents can reorder while both stay inside the bound. - -The FP8 decoder checkpoints (`FP8_DECODER_MODELS`) are load-and-decode cases with no -reference of their own ā€” writing one means dequantizing a compressed-tensors checkpoint on -CPU, which the generator does not do ā€” so they gate the FP8 weight load and the Spyre -`aten._scaled_mm` kernel against breaking outright, while the unquantized siblings gate the -numerics. Each borrows its prompts from the sibling's reference entry. +Regenerate only when the *expected* output changes (a new model or prompt), never to make a +failing test pass ā€” that is the regression the gate exists to catch. Prompt sets are per +model: see `MODEL_PROMPTS` and `MODEL_DOCUMENTS` in the generators. + +Every gated model is pinned to a revision, in the generator's `MODEL_REVISIONS` and in +`.github/cache_config/hf_models_and_datasets.yaml`. Each generator records the revision it +measured in its JSON and the test loads that one back, so bumping a pin means regenerating +that model's reference. + +`SPYRE_TEST_ABS_TOL` / `SPYRE_TEST_REL_TOL` (decoder probabilities) and +`SPYRE_TEST_SCORE_ABS_TOL` / `SPYRE_TEST_SCORE_REL_TOL` (reranker scores) set the +tolerances; the stricter of each pair applies, so a low-confidence reference is held to a +fraction rather than to the same absolute margin. Reranker ranking is checked separately +from the per-score bound. The FP8 decoder checkpoints are load-and-decode cases with no +reference of their own ā€” their unquantized siblings gate the numerics. A greedy path that diverges from HF on a near-tie cannot be compared past the split, so -each decoder case prints how many reference steps it actually matched -(`matched 32/48 reference steps (16/16, 0/16, 16/16 per prompt)`). Read that line rather -than the pass/fail alone: a prompt that stops after a step or two gates very little, and -the fix is a prompt whose greedy path is confident, not a looser tolerance. +each decoder case prints how many reference steps it matched. Read that line rather than +the pass/fail alone: a prompt that stops after a step or two gates very little, and the fix +is a more confident prompt, not a looser tolerance. #### Upstream Test Integration diff --git a/tests/data/generate_decoder_output_refs.py b/tests/data/generate_decoder_output_refs.py index a555fe6b4..23329627e 100644 --- a/tests/data/generate_decoder_output_refs.py +++ b/tests/data/generate_decoder_output_refs.py @@ -58,11 +58,8 @@ _TEMPLATE.format("Convert char to string in Java."), ] -# gemma-4 drifts from HF as the prompt grows, because torch-spyre runs RMSNorm in fp16: -# on the prompts above its first-token probability is 0.65 against HF's 0.84 and the -# continuation diverges, while short prompts match token for token. Neither the reference -# dtype (fp16 CPU HF agrees with fp32 to <0.002) nor torch.compile (eager deviates just -# as far) is involved, so drop this entry once torch-spyre normalises in fp32. +# gemma-4 diverges from HF on the prompts above because torch-spyre runs RMSNorm in fp16; +# short prompts match token for token. Drop this entry once it normalises in fp32. MODEL_PROMPTS = { "google/gemma-4-31B": [ "What are IBMs main businesses?", @@ -133,7 +130,7 @@ def main() -> None: data[model_id] = generate_reference( model_id, MODEL_REVISIONS[model_id], getattr(torch, args.dtype) ) - # Written per model: each one takes minutes and is easy to interrupt. + # Written per model so an interrupted run keeps what it already generated. args.out.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") print(f"Wrote {args.out}", flush=True) diff --git a/tests/data/generate_encoder_embed_refs.py b/tests/data/generate_encoder_embed_refs.py index 06672b923..da90e65d3 100644 --- a/tests/data/generate_encoder_embed_refs.py +++ b/tests/data/generate_encoder_embed_refs.py @@ -32,11 +32,8 @@ "sentence-transformers/all-roberta-large-v1", ] -# Pinned so a reference keeps meaning one specific set of weights. Unpinned, an upstream -# re-upload silently redefines what the cosine gate compares against, and the test cannot -# tell that from a regression on the Spyre side. Written into the JSON per model and read -# back by the test, so the gate loads the same revision this file measured -- do not pin -# here without regenerating. +# Written into the JSON per model and read back by the test, so the gate loads the same +# weights measured here and an upstream re-upload cannot redefine what it compares against. MODEL_REVISIONS = { "ibm-granite/granite-embedding-125m-english": "4ab61ffd423be45cd932b21a7c696063d82bf45f", "ibm-granite/granite-embedding-278m-multilingual": "a9cb5338491faf32b73dd17b714a31821c021bbf", diff --git a/tests/data/generate_rerank_score_refs.py b/tests/data/generate_rerank_score_refs.py index 5379d2922..a3c233d23 100644 --- a/tests/data/generate_rerank_score_refs.py +++ b/tests/data/generate_rerank_score_refs.py @@ -41,16 +41,8 @@ "BAAI/bge-reranker-large": "55611d7bca2a7133960a6d3b71e083071bbfc312", } -# One query against documents a reranker should separate widely, listed most relevant -# first. The spread is what lets the test gate the *ranking* and not only the individual -# scores, so neighbouring documents have to stay far enough apart that fp16 drift cannot -# swap them. What has to clear that is the gap against the drift the two scores can absorb -# between them -- min(7e-3, 13% * p) each, the worst measured in test_encoder_models.py -- -# and not a ratio of scores or of odds: odds explode near p=1, where drift is absolute, and -# collapse onto the score ratio near p=0, where it is relative, so either ratio rates the -# wrong pair as the risky one. The tightest pair below clears its combined drift by 2.8x -# (the top two, 0.99997 vs 0.96147: a gap of 0.039 against 0.014 of drift); every other -# pair clears it by 6x or more. +# One query against documents a reranker should separate widely, most relevant first: the +# ranking gate needs neighbouring scores farther apart than the tolerance each can drift. QUERY = "What is the capital of France?" DOCUMENTS = [ "The capital of France is Paris.", @@ -61,13 +53,8 @@ "The IBM Spyre accelerator runs AI inference workloads.", ] -# bge-reranker-large saturates: it scores every Paris-adjacent document above 0.9994, so on -# the list above its top five land within 5e-4 of each other and their order is decided by -# fp16 noise rather than by relevance -- the same drift bound, failed. Give it documents -# spread across the range it actually resolves; the list below clears that bound by 4.4x at -# its tightest (0.00187 vs 0.00052, a gap of 1.4e-3 against 3.1e-4 of drift), so it is the -# safer of the two despite sitting lower. Drop this entry only for a model that separates -# the shared list. +# bge-reranker-large scores every Paris-adjacent document above 0.9994, so on the shared +# list its top five land within fp16 noise of each other and their order is arbitrary. MODEL_DOCUMENTS = { "BAAI/bge-reranker-large": [ "The capital of France is Paris.", @@ -102,7 +89,7 @@ def generate_reference(model_id: str, revision: str) -> dict[str, Any]: logit = model(**inputs).logits.reshape(-1) assert logit.numel() == 1, f"{model_id}: expected num_labels=1, got {logit.numel()}" # vLLM's PoolerClassify sigmoids a single-label head, so the reference is a - # probability and the test's tolerance is an absolute bound on [0, 1]. + # probability on [0, 1]. scores.append(round(float(torch.sigmoid(logit)[0]), _ROUND)) print(f" {document!r}\n -> {scores[-1]:.6f}", flush=True) diff --git a/tests/e2e/test_encoder_models.py b/tests/e2e/test_encoder_models.py index 836555b86..c5690ba35 100644 --- a/tests/e2e/test_encoder_models.py +++ b/tests/e2e/test_encoder_models.py @@ -54,8 +54,7 @@ ] # Cross-encoder rerankers (classify / score path). The two BGE variants share -# XLMRobertaForSequenceClassification but not their weights or position table (514 vs -# 8194 slots), so each is gated on its own cached scores. +# XLMRobertaForSequenceClassification but not their weights or position table. RERANKER_MODELS = [ "BAAI/bge-reranker-v2-m3", "BAAI/bge-reranker-large", @@ -73,13 +72,8 @@ # Match upstream check_embeddings_close(tol=1e-2). COSINE_MIN = 0.99 -# Reranker references are sigmoid probabilities, and a bare absolute bound is a poor gate -# for one: most of them sit just above zero, where 0.03 permits an arbitrary relative -# error -- 400x on the smallest. Paired with a relative bound (stricter of the two wins, -# as in test_model_quality.py) a near-zero reference is held to a fraction instead, while -# the near-one end stays on the absolute bound, which is already tight relatively there. -# Worst measured drift on the reference documents is 7e-3 absolute and 13% relative, both -# on bge-reranker-large, so each bound keeps ~4x margin. +# Reranker references are sigmoid probabilities and most sit just above zero, where an +# absolute bound permits an arbitrary relative error, so the stricter of the two applies. SCORE_ABS_TOL = float(os.environ.get("SPYRE_TEST_SCORE_ABS_TOL", "0.03")) SCORE_REL_TOL = float(os.environ.get("SPYRE_TEST_SCORE_REL_TOL", "0.5")) @@ -213,9 +207,8 @@ def test_encoder_embed_last_pooling() -> None: override exercises the LAST gather + normalize path that ``configure_pooling_for_spyre`` patches to ``SpyreLastPool``. """ - # Taken from the embed reference entry rather than hardcoded, so this case cannot drift - # onto a different revision than the embed gates measure. Both sides are computed in - # the same run here, so the pin buys reproducibility, not a valid comparison. + # Both sides are computed in this run, so the pin buys reproducibility rather than a + # valid comparison; read from the embed refs so it cannot drift off the gated weights. revision = _REFERENCES[LAST_POOLING_MODEL]["revision"] prompts = LAST_POOLING_PROMPTS ref_embs = _hf_last_token_embeddings(LAST_POOLING_MODEL, revision, prompts) @@ -264,13 +257,9 @@ def test_encoder_rerank_models_compiled(model: str) -> None: def _assert_rerank_scores_match_refs(model: str, enforce_eager: bool) -> None: """What runs on Spyre here is the encoder body, not the score itself. - A reranker's classifier head stays float32, and torch-spyre has no FP32 batchmatmul - (torch-spyre#1794), so ``configure_pooling_for_spyre`` sends the whole pooling tail - through ``run_pooling_tail_on_cpu`` -- logged as "FP32 classifier/head unsupported on - Spyre ... running pooler on CPU" at load, in the compiled case as much as the eager - one. So the bounds below gate the transformer blocks and the CLS gather, and a - regression confined to the head or the sigmoid would pass both. Extending the gate to - it needs the head on device, not another tolerance. + The classifier head stays float32 and torch-spyre has no FP32 batchmatmul + (torch-spyre#1794), so ``configure_pooling_for_spyre`` runs the pooling tail on CPU, + compiled case included. """ ref = _RERANK_REFERENCES.get(model) if ref is None: @@ -293,9 +282,8 @@ def _assert_rerank_scores_match_refs(model: str, enforce_eager: bool) -> None: scores = [out.outputs.score for out in outputs] assert all(math.isfinite(s) for s in scores), f"{model}: non-finite score in {scores}" - # A reranker is used for its ordering, and every score can drift the same direction - # without disturbing that -- so the ranking is checked apart from the per-score bound, - # which conversely passes on a pair that has swapped inside the tolerance. + # Ranking is checked apart from the per-score bound: all scores can drift the same + # direction without reordering, and a pair can swap while both stay inside tolerance. order = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True) ref_order = sorted(range(len(ref_scores)), key=lambda i: ref_scores[i], reverse=True) assert order == ref_order, ( diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index b57018638..69a39ec69 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -38,10 +38,8 @@ "meta-llama/Llama-3.1-8B-Instruct", ] -# Weight-only FP8 (compressed-tensors) checkpoints, each mapped to the unquantized sibling -# whose reference entry it borrows prompts from -- the smoke test below compares no -# output, it only needs prompts that fit a compiled prefill bucket. Both run end to end on -# Spyre today, so the sibling entries are what gate the numerics. +# Each FP8 checkpoint borrows prompts from its unquantized sibling; the smoke test below +# compares no output, it only needs prompts that fit a compiled prefill bucket. FP8_DECODER_MODELS = { "ibm-granite/granite-3.3-8b-instruct-FP8": "ibm-granite/granite-3.3-8b-instruct", "ibm-granite/granite-4.1-8b-fp8": "ibm-granite/granite-4.1-8b", @@ -50,29 +48,24 @@ "ibm-granite/granite-3.3-8b-instruct-FP8": "4b5990b8d402a75febe0086abbf1e490af494e3d", "ibm-granite/granite-4.1-8b-fp8": "070021b3608433b6107a00733d561c9779b9937e", } -# Short: nothing is compared, so the run only has to prove decode advances at all. +# Nothing is compared, so the run only has to prove decode advances. FP8_MAX_TOKENS = 8 # fp16 on device reorders accumulation against the fp32 reference, so probabilities are # compared with a tolerance. Same default as sendnn-inference's TEST_ABS_TOL. ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.08")) -# ABS_TOL alone is not a uniform bound: it holds p=0.999 to 8% but lets p=0.08 land -# anywhere in [0, 0.16], a 2x relative error, so the gate is loosest exactly where the -# reference is least certain. Below the ABS_TOL/REL_TOL crossover the bound goes -# relative, holding a low-confidence token to the same *fraction* instead of the same -# margin. At the defaults the crossover is p=0.16, so nothing above it changes. +# Below the crossover with ABS_TOL (p=0.16 at the defaults) `_prob_tol` bounds relatively: +# a flat 0.08 on a reference of 0.08 would permit a 2x error. REL_TOL = float(os.environ.get("SPYRE_TEST_REL_TOL", "0.5")) -# Enough of the distribution that HF's greedy token is present even when Spyre picks a -# different one -- `_compare_against_hf` needs p(HF token) under *Spyre* to tell a -# near-tie from two distributions that disagree. 20 is vLLM's default `max_logprobs`. +# `_compare_against_hf` needs p(HF token) under Spyre, so HF's greedy token has to be in +# the returned distribution even when Spyre picks another. 20 is vLLM's `max_logprobs`. NUM_LOGPROBS = 20 MAX_MODEL_LEN = 256 MAX_NUM_SEQS = 3 -# Top of COMPILE_SIZES below, which is what caps warmup: passing compile_sizes -# explicitly skips the default buckets platform.py would derive, and platform.py then -# clamps max_num_batched_tokens down to the largest bucket. Every prompt fits this one. +# Passing compile_sizes skips the buckets platform.py would derive, and platform.py clamps +# max_num_batched_tokens down to the largest one, so this is the top of COMPILE_SIZES. MAX_NUM_BATCHED_TOKENS = 64 COMPILE_SIZES = [MAX_NUM_SEQS, MAX_NUM_BATCHED_TOKENS] @@ -113,7 +106,7 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No SamplingParams( temperature=0.0, max_tokens=max_tokens, - logprobs=NUM_LOGPROBS, # sampled token plus enough to locate HF's + logprobs=NUM_LOGPROBS, ignore_eos=True, # the reference is a fixed-length run with EOS disabled ), use_tqdm=False, @@ -125,8 +118,7 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No for hf_result, output in zip(ref["results"], outputs) ] # A prompt that diverges early verifies only the steps before the split, so a green - # case is not automatically a well-covered one. Printed (PYTEST_ARGS carries -s) so - # the coverage a run actually achieved is visible without having to fail first. + # case is not automatically a well-covered one. per_prompt = ", ".join(f"{n}/{max_tokens}" for n in matched) print( f"\n{model}: matched {sum(matched)}/{len(prompts) * max_tokens} reference steps " @@ -140,11 +132,8 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No def test_fp8_decoder_model_smoke(model: str, monkeypatch: pytest.MonkeyPatch) -> None: """A compiled FP8 checkpoint loads and decodes. - Load-and-decode only, with no reference comparison: a reference for a - compressed-tensors checkpoint means dequantizing it on CPU first, which the generator - does not do. So this holds the FP8 weight load and the ``aten._scaled_mm`` kernel - (``custom_ops/fp8_linear_kernel.py``) to running at all rather than to a numerical - bound -- `test_decoder_model_output` gates the unquantized siblings' output. + No reference comparison: writing one means dequantizing a compressed-tensors + checkpoint on CPU, which the generator does not do. """ base = FP8_DECODER_MODELS[model] base_ref = _REFERENCES.get(base) @@ -190,12 +179,9 @@ def test_fp8_decoder_model_smoke(model: str, monkeypatch: pytest.MonkeyPatch) -> def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[str]) -> None: """Fail loudly if a prompt outgrew the largest compiled prefill bucket. - Nothing else does: past the largest bucket `SpyreShapeBucketer.find_bucket` returns - None (spyre_shape_bucketer.py) and execute_model runs the shape unpadded, so an - over-long prompt is not an error but a Dynamo recompile inside generate() -- and the - raised VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS lets that grind for hours instead of - failing. Run before the engine is built so an edited prompt costs seconds, not a - warmup. + Past the largest bucket `SpyreShapeBucketer.find_bucket` returns None and the shape + runs unpadded, so an over-long prompt is a silent Dynamo recompile inside generate() + rather than an error. """ from transformers import AutoTokenizer @@ -210,11 +196,6 @@ def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[ def _prob_tol(reference_prob: float) -> float: - """Tolerance for one probability comparison, tightening as the reference gets small. - - ``min`` and not ``max``: this only ever tightens ABS_TOL, never loosens it, so the - bound is the stricter of "within ABS_TOL" and "within REL_TOL of the reference". - """ return min(ABS_TOL, REL_TOL * reference_prob) @@ -242,12 +223,8 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu ) if hf_id != token_id: - # Greedy paths only diverge legitimately on a near-tie. Judge that on the HF - # token in *both* distributions, never on the two sampled tokens' own - # probabilities: those agree whenever the models are equally confident, so - # HF at p=0.9 on one token and Spyre at p=0.9 on another -- a total - # disagreement -- would read as a tie. Past this step the prefixes differ, - # so no later token is comparable either way. + # The sampled tokens' own probabilities agree whenever the models are equally + # confident, however far apart they picked, so judge the tie on HF's token. spyre_hf = completion.logprobs[step].get(hf_id) assert spyre_hf is not None, ( f"{model}: wrong token and HF's token is outside Spyre's top " @@ -258,12 +235,8 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu f"{model}: wrong token and p(HF token) differs by more than {tol:.4f} " f"(Spyre {spyre_hf_prob:.4f} vs HF {hf_prob:.4f}), {detail}" ) - # A tie also means Spyre itself ranks the two level. Without this, a flat HF - # distribution (its own argmax at p=0.1) would excuse Spyre being confidently - # elsewhere at p=0.85, since p(HF token) still matches at 0.1 in both. - # Bound is doubled: HF picked its token, so it led there - # (p_hf(spyre token) <= hf_prob), and each of the two may drift by `tol` in - # the opposite direction, which is what flipped the argmax in the first place. + # A tie also means Spyre ranks the two level: a flat HF distribution must not + # excuse Spyre being confident elsewhere. Doubled: both may drift by `tol`. tie_tol = 2 * tol assert abs(prob - spyre_hf_prob) <= tie_tol, ( f"{model}: wrong token, and Spyre puts it {prob - spyre_hf_prob:.4f} > " From a6ebef84ef08e1785a47c91b50847883deb80dde Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Wed, 9 Sep 2026 14:01:28 +0200 Subject: [PATCH 14/18] add gemma4 MoE Signed-off-by: Yannick Schnider --- .../cache_config/hf_models_and_datasets.yaml | 2 + tests/data/decoder_output_refs.json | 181 ++++++++++++++++++ tests/data/generate_decoder_output_refs.py | 16 +- tests/e2e/test_model_quality.py | 1 + 4 files changed, 194 insertions(+), 6 deletions(-) diff --git a/.github/cache_config/hf_models_and_datasets.yaml b/.github/cache_config/hf_models_and_datasets.yaml index 817adfa58..928dd0ff3 100644 --- a/.github/cache_config/hf_models_and_datasets.yaml +++ b/.github/cache_config/hf_models_and_datasets.yaml @@ -41,4 +41,6 @@ public_models: revision: 070021b3608433b6107a00733d561c9779b9937e - repo: google/gemma-4-31B revision: 5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89 + - repo: google/gemma-4-26B-A4B + revision: 24548b62aa021d562695c04aaf7758a1ea47990b - qwrt/Swedish0.1M diff --git a/tests/data/decoder_output_refs.json b/tests/data/decoder_output_refs.json index f50a1a019..2791ccb00 100644 --- a/tests/data/decoder_output_refs.json +++ b/tests/data/decoder_output_refs.json @@ -1,4 +1,185 @@ { + "google/gemma-4-26B-A4B": { + "dtype": "float32", + "max_tokens": 16, + "results": [ + { + "logprobs": [ + -0.608086, + -1.174431, + -1.03372, + -1.044913, + -2.672324, + -0.033213, + -1.656933, + -2.380829, + -0.590173, + -0.130631, + -0.20754, + -0.13099, + -0.105282, + -0.604167, + -0.835592, + -0.751586 + ], + "prompt": "What are IBMs main businesses?", + "text": "\n\nWhat is the difference between a product and a service?\n\nWhat is the", + "token_ids": [ + 108, + 3689, + 563, + 506, + 4954, + 1534, + 496, + 1698, + 532, + 496, + 2509, + 236881, + 108, + 3689, + 563, + 506 + ], + "tokens": [ + "\n\n", + "What", + " is", + " the", + " difference", + " between", + " a", + " product", + " and", + " a", + " service", + "?", + "\n\n", + "What", + " is", + " the" + ] + }, + { + "logprobs": [ + -1.890419, + -1.724759, + -1.272666, + -2.045985, + -0.647059, + -2.163997, + -0.357102, + -1.038351, + -1.267442, + -0.444188, + -1.555656, + -0.700945, + -1.350318, + -0.928785, + -0.511243, + -0.759674 + ], + "prompt": "The capital of France is", + "text": " a city of romance, art, and culture. It is also a city of", + "token_ids": [ + 496, + 3207, + 529, + 30875, + 236764, + 1610, + 236764, + 532, + 6540, + 236761, + 1030, + 563, + 992, + 496, + 3207, + 529 + ], + "tokens": [ + " a", + " city", + " of", + " romance", + ",", + " art", + ",", + " and", + " culture", + ".", + " It", + " is", + " also", + " a", + " city", + " of" + ] + }, + { + "logprobs": [ + -0.132, + -0.319847, + -0.004313, + -1e-06, + -0.724456, + -0.466033, + -0.188152, + -1.336368, + -0.037957, + -0.064367, + -0.043371, + -0.016508, + -0.004109, + -0.033714, + -0.003028, + -0.000332 + ], + "prompt": "Q: What is the largest planet in our solar system?\nA:", + "text": " Jupiter\n\nQ: What is the smallest planet in our solar system?\nA", + "token_ids": [ + 52895, + 108, + 236935, + 236787, + 2900, + 563, + 506, + 21548, + 13401, + 528, + 1023, + 10321, + 1458, + 236881, + 107, + 236776 + ], + "tokens": [ + " Jupiter", + "\n\n", + "Q", + ":", + " What", + " is", + " the", + " smallest", + " planet", + " in", + " our", + " solar", + " system", + "?", + "\n", + "A" + ] + } + ], + "revision": "24548b62aa021d562695c04aaf7758a1ea47990b" + }, "google/gemma-4-31B": { "dtype": "float32", "max_tokens": 16, diff --git a/tests/data/generate_decoder_output_refs.py b/tests/data/generate_decoder_output_refs.py index 23329627e..ce70d7704 100644 --- a/tests/data/generate_decoder_output_refs.py +++ b/tests/data/generate_decoder_output_refs.py @@ -36,6 +36,7 @@ "ibm-granite/granite-3.3-8b-instruct", "ibm-granite/granite-4.1-8b", "google/gemma-4-31B", + "google/gemma-4-26B-A4B", "meta-llama/Llama-3.1-8B-Instruct", ] @@ -43,6 +44,7 @@ "ibm-granite/granite-3.3-8b-instruct": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b", "ibm-granite/granite-4.1-8b": "1504002f650e656a0a3789d99574df12e3e94ed0", "google/gemma-4-31B": "5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89", + "google/gemma-4-26B-A4B": "24548b62aa021d562695c04aaf7758a1ea47990b", "meta-llama/Llama-3.1-8B-Instruct": "0e9e39f249a16976918f6564b8830bc894c89659", } @@ -59,13 +61,15 @@ ] # gemma-4 diverges from HF on the prompts above because torch-spyre runs RMSNorm in fp16; -# short prompts match token for token. Drop this entry once it normalises in fp32. +# short prompts match token for token. Drop these entries once it normalises in fp32. +_GEMMA4_PROMPTS = [ + "What are IBMs main businesses?", + "The capital of France is", + "Q: What is the largest planet in our solar system?\nA:", +] MODEL_PROMPTS = { - "google/gemma-4-31B": [ - "What are IBMs main businesses?", - "The capital of France is", - "Q: What is the largest planet in our solar system?\nA:", - ], + "google/gemma-4-31B": _GEMMA4_PROMPTS, + "google/gemma-4-26B-A4B": _GEMMA4_PROMPTS, } MAX_TOKENS = 16 diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index 69a39ec69..ab183e3f0 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -35,6 +35,7 @@ "ibm-granite/granite-3.3-8b-instruct", "ibm-granite/granite-4.1-8b", "google/gemma-4-31B", + "google/gemma-4-26B-A4B", "meta-llama/Llama-3.1-8B-Instruct", ] From 789449af5b16624229522e80c09f0d9f51eddde9 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Wed, 9 Sep 2026 17:53:14 +0200 Subject: [PATCH 15/18] address feedback Signed-off-by: Yannick Schnider --- .github/scripts/rebalance_test_shards.py | 6 ++ .github/workflows/_test_matrix.yaml | 53 +++++++++++++++-- Makefile | 27 +++++++-- docs/contributing/README.md | 14 +++-- tests/e2e/test_encoder_models.py | 9 +-- tests/e2e/test_model_quality.py | 57 +++++++++++-------- tests/plugin/spyre_testing_plugin/sharding.py | 33 ++++++++++- tests/test_sharding.py | 3 +- 8 files changed, 157 insertions(+), 45 deletions(-) diff --git a/.github/scripts/rebalance_test_shards.py b/.github/scripts/rebalance_test_shards.py index 26878bb66..7be044ef3 100644 --- a/.github/scripts/rebalance_test_shards.py +++ b/.github/scripts/rebalance_test_shards.py @@ -68,6 +68,12 @@ "collect_upstream": False, }, {"key": "probe", "makefile_name": "probes", "var": "PROBE_SHARDS", "collect_upstream": False}, + { + "key": "quality", + "makefile_name": "model-quality", + "var": "QUALITY_SHARDS", + "collect_upstream": False, + }, ] diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml index 75b2b43c7..db8052c04 100644 --- a/.github/workflows/_test_matrix.yaml +++ b/.github/workflows/_test_matrix.yaml @@ -282,16 +282,60 @@ jobs: - linux image_label: image_torch_spyre test_target: test-smoke-shard-7 - # Product-model output-quality gate: its own job and not in - # `integration`, since every case compiles a product model. - - cfg: Spyre model quality tests + # Product-model output-quality gate: its own jobs and not in + # `integration`, since every case compiles a product model. Sharded + # across parallel 1-card jobs like smoke/attention, bounded by the + # slowest single case. The plugin owns the weighted partition; + # QUALITY_SHARDS (Makefile default) is the count. Adding/removing a + # shard = add/remove an entry here and bump that default. + - cfg: Spyre model quality tests (shard 1/6) test_types: regression trunk runs_on: - x86_64 - spyre_pf_x1 - linux image_label: image_torch_spyre - test_target: test-model-quality + test_target: test-model-quality-shard-0 + - cfg: Spyre model quality tests (shard 2/6) + test_types: regression trunk + runs_on: + - x86_64 + - spyre_pf_x1 + - linux + image_label: image_torch_spyre + test_target: test-model-quality-shard-1 + - cfg: Spyre model quality tests (shard 3/6) + test_types: regression trunk + runs_on: + - x86_64 + - spyre_pf_x1 + - linux + image_label: image_torch_spyre + test_target: test-model-quality-shard-2 + - cfg: Spyre model quality tests (shard 4/6) + test_types: regression trunk + runs_on: + - x86_64 + - spyre_pf_x1 + - linux + image_label: image_torch_spyre + test_target: test-model-quality-shard-3 + - cfg: Spyre model quality tests (shard 5/6) + test_types: regression trunk + runs_on: + - x86_64 + - spyre_pf_x1 + - linux + image_label: image_torch_spyre + test_target: test-model-quality-shard-4 + - cfg: Spyre model quality tests (shard 6/6) + test_types: regression trunk + runs_on: + - x86_64 + - spyre_pf_x1 + - linux + image_label: image_torch_spyre + test_target: test-model-quality-shard-5 # Decoder attention, sharded across parallel jobs. The compiled # (STOCK on device) cases dominate runtime and grow HBM within a # process, so each shard is its own job on its own card: wall-clock @@ -552,6 +596,7 @@ jobs: spyre-rpms.lock tests/**/*.py tests/**/*.yaml + tests/data/** spyre_inference/**/*.py spyre_inference/**/*.yaml diff --git a/Makefile b/Makefile index 3c5c829d1..de95385ac 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,8 @@ endif RESULTS_DIR ?= . .PHONY: help test tests run-one aiu-setup perf-tests coverage print-test-type \ - test-smoke test-smoke-shard test-model-quality test-probes test-probes-shard \ + test-smoke test-smoke-shard test-model-quality test-model-quality-shard \ + test-probes test-probes-shard \ test-attention test-attention-shard \ test-distributed test-distributed-shard test-upstream test-upstream-shard \ test-upstream-distributed tests-single-card tests-multi-card @@ -193,9 +194,25 @@ test-smoke-shard-%: $(MAKE) test-smoke-shard SMOKE_SHARD_ID=$* JUNIT_XML=$(JUNIT_XML) # Carved out of smoke: every case compiles a product model, up to the 31B decoders. -test-model-quality: ## Run the product-model output-quality gates against the cached HF references. +test-model-quality: ## Run the product-model output-quality gates against the cached HF references. Unsharded (local full run). $(MAKE) run-one MARK_OVERRIDE='model_quality and not (distributed or upstream)' JUNIT_XML=$(JUNIT_XML) +# CI fans the gate out across parallel shard jobs like smoke/attention; the plugin's +# weighted partition (--quality-shards) balances by recorded runtime when a durations file +# is present, else by decoder-vs-encoder path weight. No partition beats the slowest single +# case, so that bounds the useful count. QUALITY_SHARDS is the single source of truth. +QUALITY_SHARDS ?= 6 +QUALITY_SHARD_ID ?= 0 +test-model-quality-shard: ## Run one model-quality shard (QUALITY_SHARDS=N QUALITY_SHARD_ID=i). + $(MAKE) run-one MARK_OVERRIDE='model_quality and not (distributed or upstream)' \ + PYTEST_ARGS='$(PYTEST_ARGS) --quality-shards=$(QUALITY_SHARDS) --quality-shard-id=$(QUALITY_SHARD_ID)' \ + JUNIT_XML=$(JUNIT_XML) + +# CI runs one matrix job per shard as `test-model-quality-shard-` so each JUnit +# artifact name is unique; the pattern maps to QUALITY_SHARD_ID. +test-model-quality-shard-%: + $(MAKE) test-model-quality-shard QUALITY_SHARD_ID=$* JUNIT_XML=$(JUNIT_XML) + test-probes: ## Run the torch-spyre backend probes (excluded from integration), unsharded (local full run). $(MAKE) run-one MARK_OVERRIDE='probe and not upstream' JUNIT_XML=$(JUNIT_XML) @@ -285,13 +302,15 @@ test-upstream-distributed: ## Run the upstream+distributed marker combo. # Single-card / multi-card split, grouping the 6 marker combos above by how many cards they need. # Each suite gets its own junit-/junit-.xml subdir, matching GHA's artifact-name/file-name layout (_test_matrix.yaml) so a Jenkins run's JUnit paths line up 1:1 with a GHA run's. -tests-single-card: ## Run the 1-card marker combos (smoke shards / model-quality / attention shards / encoder-attention / upstream shards). Needs 1 card. +tests-single-card: ## Run the 1-card marker combos (smoke shards / model-quality shards / attention shards / encoder-attention / upstream shards). Needs 1 card. mkdir -p "$(RESULTS_DIR)"; \ rc=0; \ for i in $$(seq 0 $$(( $(SMOKE_SHARDS) - 1 ))); do \ mkdir -p "$(RESULTS_DIR)/junit-test-smoke-shard-$$i" && $(MAKE) test-smoke-shard SMOKE_SHARD_ID=$$i JUNIT_XML="$(RESULTS_DIR)/junit-test-smoke-shard-$$i/junit-test-smoke-shard-$$i.xml" || rc=1; \ done; \ - mkdir -p "$(RESULTS_DIR)/junit-test-model-quality" && $(MAKE) test-model-quality JUNIT_XML="$(RESULTS_DIR)/junit-test-model-quality/junit-test-model-quality.xml" || rc=1; \ + for i in $$(seq 0 $$(( $(QUALITY_SHARDS) - 1 ))); do \ + mkdir -p "$(RESULTS_DIR)/junit-test-model-quality-shard-$$i" && $(MAKE) test-model-quality-shard QUALITY_SHARD_ID=$$i JUNIT_XML="$(RESULTS_DIR)/junit-test-model-quality-shard-$$i/junit-test-model-quality-shard-$$i.xml" || rc=1; \ + done; \ for i in $$(seq 0 $$(( $(ATTN_SHARDS) - 1 ))); do \ mkdir -p "$(RESULTS_DIR)/junit-test-attention-shard-$$i" && $(MAKE) test-attention-shard ATTN_SHARD_ID=$$i JUNIT_XML="$(RESULTS_DIR)/junit-test-attention-shard-$$i/junit-test-attention-shard-$$i.xml" || rc=1; \ done; \ diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 36229d670..52126bcbd 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -91,9 +91,14 @@ cosine similarity for the embedding models and sigmoid scores plus document rank the cross-encoder rerankers (`tests/e2e/test_encoder_models.py`). ```bash -make test-model-quality # the whole gate, one card +make test-model-quality # the whole gate, one card +make test-model-quality-shard-0 # one CI shard (QUALITY_SHARDS=6) ``` +CI runs the gate as `QUALITY_SHARDS` parallel 1-card jobs, weighted by recorded runtime like +the smoke and attention suites. The slowest single case bounds the useful shard count, so +resize with the `rebalance-test-shards` skill rather than by raising it on a hunch. + The models are too large to run through transformers in CI, so the references are checked into `tests/data/` and regenerated by hand where the weights are cached: @@ -120,9 +125,10 @@ from the per-score bound. The FP8 decoder checkpoints are load-and-decode cases reference of their own ā€” their unquantized siblings gate the numerics. A greedy path that diverges from HF on a near-tie cannot be compared past the split, so -each decoder case prints how many reference steps it matched. Read that line rather than -the pass/fail alone: a prompt that stops after a step or two gates very little, and the fix -is a more confident prompt, not a looser tolerance. +each decoder case prints how many reference steps it matched and fails below +`MIN_MATCHED_FRACTION` of them. Read the printed line rather than the pass/fail alone: a +prompt that stops early gates very little, and the fix is a more confident prompt, not a +looser tolerance or a lower floor. #### Upstream Test Integration diff --git a/tests/e2e/test_encoder_models.py b/tests/e2e/test_encoder_models.py index c5690ba35..36b07b6ad 100644 --- a/tests/e2e/test_encoder_models.py +++ b/tests/e2e/test_encoder_models.py @@ -255,12 +255,9 @@ def test_encoder_rerank_models_compiled(model: str) -> None: def _assert_rerank_scores_match_refs(model: str, enforce_eager: bool) -> None: - """What runs on Spyre here is the encoder body, not the score itself. - - The classifier head stays float32 and torch-spyre has no FP32 batchmatmul - (torch-spyre#1794), so ``configure_pooling_for_spyre`` runs the pooling tail on CPU, - compiled case included. - """ + """What runs on Spyre is the encoder body, not the score: the classifier head stays + float32 and torch-spyre has no FP32 batchmatmul (torch-spyre#1794), so the pooling tail + runs on CPU even in the compiled case.""" ref = _RERANK_REFERENCES.get(model) if ref is None: pytest.skip(f"No HF ref for {model}; run tests/data/generate_rerank_score_refs.py") diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index ab183e3f0..a78fe6adf 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -39,8 +39,7 @@ "meta-llama/Llama-3.1-8B-Instruct", ] -# Each FP8 checkpoint borrows prompts from its unquantized sibling; the smoke test below -# compares no output, it only needs prompts that fit a compiled prefill bucket. +# Prompts come from the unquantized sibling; the smoke case only needs ones that fit a bucket. FP8_DECODER_MODELS = { "ibm-granite/granite-3.3-8b-instruct-FP8": "ibm-granite/granite-3.3-8b-instruct", "ibm-granite/granite-4.1-8b-fp8": "ibm-granite/granite-4.1-8b", @@ -55,20 +54,26 @@ # fp16 on device reorders accumulation against the fp32 reference, so probabilities are # compared with a tolerance. Same default as sendnn-inference's TEST_ABS_TOL. ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.08")) -# Below the crossover with ABS_TOL (p=0.16 at the defaults) `_prob_tol` bounds relatively: -# a flat 0.08 on a reference of 0.08 would permit a 2x error. +# Below the ABS_TOL crossover (p=0.16) a flat 0.08 on a 0.08 reference would permit a 2x error. REL_TOL = float(os.environ.get("SPYRE_TEST_REL_TOL", "0.5")) -# `_compare_against_hf` needs p(HF token) under Spyre, so HF's greedy token has to be in -# the returned distribution even when Spyre picks another. 20 is vLLM's `max_logprobs`. +# HF's greedy token must be in Spyre's distribution even when Spyre picks another; 20 is +# vLLM's `max_logprobs`. NUM_LOGPROBS = 20 +# A near-tie split ends the comparison, so without a floor a case that mispredicts at step 0 +# on every prompt would pass having compared nothing. +MIN_MATCHED_FRACTION = 0.5 + MAX_MODEL_LEN = 256 MAX_NUM_SEQS = 3 # Passing compile_sizes skips the buckets platform.py would derive, and platform.py clamps # max_num_batched_tokens down to the largest one, so this is the top of COMPILE_SIZES. MAX_NUM_BATCHED_TOKENS = 64 COMPILE_SIZES = [MAX_NUM_SEQS, MAX_NUM_BATCHED_TOKENS] +# Slack for the prompt-fit guard below: it counts with a bare `tokenizer(prompt)` while the +# engine tokenizes through vLLM, which can differ by a special token or two. +PROMPT_TOKEN_MARGIN = 8 _REF_PATH = Path(__file__).parent.parent / "data" / "decoder_output_refs.json" _REFERENCES: dict = json.loads(_REF_PATH.read_text()) if _REF_PATH.exists() else {} @@ -118,8 +123,6 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No _compare_against_hf(model, hf_result, output) for hf_result, output in zip(ref["results"], outputs) ] - # A prompt that diverges early verifies only the steps before the split, so a green - # case is not automatically a well-covered one. per_prompt = ", ".join(f"{n}/{max_tokens}" for n in matched) print( f"\n{model}: matched {sum(matched)}/{len(prompts) * max_tokens} reference steps " @@ -127,15 +130,19 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No f"near-tie and gate little -- see MODEL_PROMPTS in " f"tests/data/generate_decoder_output_refs.py." ) + min_matched = math.ceil(MIN_MATCHED_FRACTION * max_tokens) + assert all(n >= min_matched for n in matched), ( + f"{model}: matched {per_prompt} reference steps per prompt, under the " + f"{min_matched}/{max_tokens} floor -- the near-tie split came too early to gate " + f"anything. Every prompt matched all {max_tokens} steps when the reference was " + f"taken, so treat this as a regression, not as a floor to lower." + ) @pytest.mark.parametrize("model", FP8_DECODER_MODELS) def test_fp8_decoder_model_smoke(model: str, monkeypatch: pytest.MonkeyPatch) -> None: - """A compiled FP8 checkpoint loads and decodes. - - No reference comparison: writing one means dequantizing a compressed-tensors - checkpoint on CPU, which the generator does not do. - """ + """A compiled FP8 checkpoint loads and decodes; no reference, the generator does not + dequantize compressed-tensors on CPU.""" base = FP8_DECODER_MODELS[model] base_ref = _REFERENCES.get(base) assert base_ref is not None, ( @@ -171,28 +178,30 @@ def test_fp8_decoder_model_smoke(model: str, monkeypatch: pytest.MonkeyPatch) -> for output in outputs: completion = output.outputs[0] print(f"\n{model} prompt: {output.prompt!r}\n Spyre: {completion.text!r}") + # Token count only: this case must not assume the tokens decode to non-empty text. assert len(completion.token_ids) == FP8_MAX_TOKENS, ( f"{model}: generated {len(completion.token_ids)} of {FP8_MAX_TOKENS} tokens" ) - assert completion.text.strip(), f"{model}: empty completion for {output.prompt!r}" def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[str]) -> None: """Fail loudly if a prompt outgrew the largest compiled prefill bucket. - Past the largest bucket `SpyreShapeBucketer.find_bucket` returns None and the shape - runs unpadded, so an over-long prompt is a silent Dynamo recompile inside generate() - rather than an error. + Past the largest bucket `SpyreShapeBucketer.find_bucket` returns None and the shape runs + unpadded, so an over-long prompt is a silent Dynamo recompile inside generate(). """ from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(model, revision=revision) + limit = MAX_NUM_BATCHED_TOKENS - PROMPT_TOKEN_MARGIN for prompt in prompts: num_tokens = len(tokenizer(prompt).input_ids) - assert num_tokens <= MAX_NUM_BATCHED_TOKENS, ( - f"{model}: prompt is {num_tokens} tokens, past the largest compiled bucket " - f"({MAX_NUM_BATCHED_TOKENS}) -- it would recompile at generate() time. Shorten " - f"it, or raise MAX_NUM_BATCHED_TOKENS here and in the generator: {prompt!r}" + assert num_tokens <= limit, ( + f"{model}: prompt is {num_tokens} tokens, over the {limit}-token bound this " + f"guard holds ({PROMPT_TOKEN_MARGIN} below the largest compiled bucket, " + f"{MAX_NUM_BATCHED_TOKENS}) -- past the bucket it would recompile at generate() " + f"time. Shorten it, or raise MAX_NUM_BATCHED_TOKENS here and in the generator: " + f"{prompt!r}" ) @@ -224,8 +233,8 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu ) if hf_id != token_id: - # The sampled tokens' own probabilities agree whenever the models are equally - # confident, however far apart they picked, so judge the tie on HF's token. + # Two equally confident models agree on p(sampled) however far apart they + # picked, so judge the tie on HF's token. spyre_hf = completion.logprobs[step].get(hf_id) assert spyre_hf is not None, ( f"{model}: wrong token and HF's token is outside Spyre's top " @@ -236,7 +245,7 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu f"{model}: wrong token and p(HF token) differs by more than {tol:.4f} " f"(Spyre {spyre_hf_prob:.4f} vs HF {hf_prob:.4f}), {detail}" ) - # A tie also means Spyre ranks the two level: a flat HF distribution must not + # A tie also means Spyre ranks the two level, so a flat HF distribution cannot # excuse Spyre being confident elsewhere. Doubled: both may drift by `tol`. tie_tol = 2 * tol assert abs(prob - spyre_hf_prob) <= tie_tol, ( diff --git a/tests/plugin/spyre_testing_plugin/sharding.py b/tests/plugin/spyre_testing_plugin/sharding.py index d85bbbe8b..9e8536ed1 100644 --- a/tests/plugin/spyre_testing_plugin/sharding.py +++ b/tests/plugin/spyre_testing_plugin/sharding.py @@ -14,7 +14,7 @@ """Duration-weighted test sharding for CI fan-out. -Each suite (attn/smoke/upstream/dist/probe) is split across N parallel jobs; a job +Each suite (attn/smoke/upstream/dist/probe/quality) is split across N parallel jobs; a job keeps only its shard's slice. Every shard job computes the same weighted greedy longest-processing-time partition, so no cross-job coordination is needed and the union of all shards is the full selection exactly once (guarded by @@ -60,7 +60,7 @@ def _will_skip(item: pytest.Item) -> bool: def add_shard_options(parser) -> None: """Register one ---shards / ---shard-id pair per CI suite.""" group = parser.getgroup("spyre-test-sharding") - for suite in ("attn", "smoke", "upstream", "dist", "probe"): + for suite in ("attn", "smoke", "upstream", "dist", "probe", "quality"): group.addoption( f"--{suite}-shards", type=int, @@ -93,6 +93,7 @@ def apply_shards(config: pytest.Config, items: list[pytest.Item]) -> None: _apply_upstream_shard(config, items) _apply_distributed_shard(config, items) _apply_probe_shard(config, items) + _apply_model_quality_shard(config, items) def _load_durations(config: pytest.Config) -> dict[str, float]: @@ -352,6 +353,34 @@ def select(item: pytest.Item) -> bool: ) +def _apply_model_quality_shard(config: pytest.Config, items: list[pytest.Item]) -> None: + """The product-model output-quality gate, split across parallel 1-card jobs. + + Every case compiles a product model, so the spread that matters is decoders (load, + compile, then 16 greedy steps) against the much smaller encoder cases. + """ + + def select(item: pytest.Item) -> bool: + return bool(item.get_closest_marker("model_quality")) and not item.get_closest_marker( + "upstream" + ) + + # Heavy = a decoder case; the encoder gates share a much smaller compile. + def weight(item: pytest.Item) -> int: + return 8 if "test_model_quality" in item.nodeid else 1 + + _apply_shard( + config, + items, + num_shards=config.getoption("--quality-shards"), + shard_id=config.getoption("--quality-shard-id"), + select=select, + weight=weight, + label="quality", + durations=_load_durations(config), + ) + + # Per-nodeid wall time this session, written out when SPYRE_TEST_DURATIONS_OUT is # set (CI). A later run pins the merged file and feeds it back as # SPYRE_TEST_DURATIONS so _apply_shard balances shards by measured runtime. Keyed diff --git a/tests/test_sharding.py b/tests/test_sharding.py index 0a8da53e1..90843178c 100644 --- a/tests/test_sharding.py +++ b/tests/test_sharding.py @@ -461,6 +461,7 @@ def _makefile_shard_counts() -> dict[str, int]: ("upstream", "UPSTREAM_SHARDS"), ("distributed", "DIST_SHARDS"), ("probes", "PROBE_SHARDS"), + ("model-quality", "QUALITY_SHARDS"), ): m = re.search(rf"^{var}\s*\?=\s*(\d+)", text, re.MULTILINE) assert m, f"{var} not found in Makefile" @@ -472,7 +473,7 @@ def _matrix_shard_ids() -> dict[str, list[int]]: text = (_REPO_ROOT / ".github/workflows/_test_matrix.yaml").read_text() return { suite: sorted({int(n) for n in re.findall(rf"test-{suite}-shard-(\d+)\b", text)}) - for suite in ("smoke", "attention", "upstream", "distributed", "probes") + for suite in ("smoke", "attention", "upstream", "distributed", "probes", "model-quality") } From 98ec0dd3e6a6e4ff91309399b7e116de8127729a Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Thu, 10 Sep 2026 09:51:45 +0200 Subject: [PATCH 16/18] feedback, fix tolerances Signed-off-by: Yannick Schnider --- docs/contributing/README.md | 22 ++++++++++---- tests/e2e/test_encoder_models.py | 8 ++++- tests/e2e/test_model_quality.py | 50 +++++++++++++++++++++++++++----- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 52126bcbd..96d033815 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -117,12 +117,22 @@ Every gated model is pinned to a revision, in the generator's `MODEL_REVISIONS` measured in its JSON and the test loads that one back, so bumping a pin means regenerating that model's reference. -`SPYRE_TEST_ABS_TOL` / `SPYRE_TEST_REL_TOL` (decoder probabilities) and -`SPYRE_TEST_SCORE_ABS_TOL` / `SPYRE_TEST_SCORE_REL_TOL` (reranker scores) set the -tolerances; the stricter of each pair applies, so a low-confidence reference is held to a -fraction rather than to the same absolute margin. Reranker ranking is checked separately -from the per-score bound. The FP8 decoder checkpoints are load-and-decode cases with no -reference of their own ā€” their unquantized siblings gate the numerics. +`SPYRE_TEST_MEAN_ABS_TOL` / `SPYRE_TEST_ABS_TOL` / `SPYRE_TEST_REL_TOL` (decoder +probabilities) and `SPYRE_TEST_SCORE_ABS_TOL` / `SPYRE_TEST_SCORE_REL_TOL` (reranker scores) +set the tolerances. For a low-confidence reference the stricter of the absolute and relative +bound applies, so it is held to a fraction rather than to the same absolute margin. Reranker +ranking is checked separately from the per-score bound. + +The decoder gate is aggregate-first: `SPYRE_TEST_MEAN_ABS_TOL` bounds each prompt's *mean* +error and is what holds quality, while `SPYRE_TEST_ABS_TOL` only caps a single step against +gross breakage. A reference near p=0.5 is maximally ill-conditioned (`dp/dlogit` peaks at +`p(1-p)`), and one compiled graph has measured 0.115 apart on such a step between two CI pods +with every token still exact ā€” a tight per-step bound buys flakiness, not coverage. Each case +prints `mean=`/`max=` per prompt, so a failure is readable without a rerun. +`SPYRE_TEST_TIE_ABS_TOL` holds token disagreements to a tighter bound, since picking a +different token is a stronger signal than drift. The FP8 decoder checkpoints are +load-and-decode cases with no reference of their own ā€” their unquantized siblings gate the +numerics. A greedy path that diverges from HF on a near-tie cannot be compared past the split, so each decoder case prints how many reference steps it matched and fails below diff --git a/tests/e2e/test_encoder_models.py b/tests/e2e/test_encoder_models.py index 36b07b6ad..176576465 100644 --- a/tests/e2e/test_encoder_models.py +++ b/tests/e2e/test_encoder_models.py @@ -209,7 +209,13 @@ def test_encoder_embed_last_pooling() -> None: """ # Both sides are computed in this run, so the pin buys reproducibility rather than a # valid comparison; read from the embed refs so it cannot drift off the gated weights. - revision = _REFERENCES[LAST_POOLING_MODEL]["revision"] + ref = _REFERENCES.get(LAST_POOLING_MODEL) + if ref is None: + pytest.skip( + f"No HF ref for {LAST_POOLING_MODEL}; run tests/data/generate_encoder_embed_refs.py" + ) + + revision = ref["revision"] prompts = LAST_POOLING_PROMPTS ref_embs = _hf_last_token_embeddings(LAST_POOLING_MODEL, revision, prompts) diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index a78fe6adf..f059af107 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -52,10 +52,21 @@ FP8_MAX_TOKENS = 8 # fp16 on device reorders accumulation against the fp32 reference, so probabilities are -# compared with a tolerance. Same default as sendnn-inference's TEST_ABS_TOL. -ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.08")) -# Below the ABS_TOL crossover (p=0.16) a flat 0.08 on a 0.08 reference would permit a 2x error. +# compared with a tolerance. Two bounds, because the failure modes are opposites. +# +# The mean is the sensitive bound: drift spread over a prompt shows up here while no single +# step looks unusual. Worst measured over the five gated decoders is 0.006 (granite-3.3). +MEAN_ABS_TOL = float(os.environ.get("SPYRE_TEST_MEAN_ABS_TOL", "0.03")) +# The per-step cap only has to catch gross breakage, so it is deliberately loose: a reference +# near p=0.5 is maximally ill-conditioned (dp/dlogit peaks at p(1-p)), and one graph measured +# 0.115 apart there across two CI pods with every token still exact (PR #723). Worst measured +# step is 0.027 -- 3x under the 0.08 this replaces, so that bound was thin for every model. +ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.20")) +# Low-probability steps keep a relative bound; a flat one would permit an arbitrary ratio. REL_TOL = float(os.environ.get("SPYRE_TEST_REL_TOL", "0.5")) +# A token disagreement is a stronger signal than drift, so judging one as a near-tie keeps the +# original tight bound rather than inheriting ABS_TOL. +TIE_ABS_TOL = float(os.environ.get("SPYRE_TEST_TIE_ABS_TOL", "0.08")) # HF's greedy token must be in Spyre's distribution even when Spyre picks another; 20 is # vLLM's `max_logprobs`. @@ -206,9 +217,29 @@ def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[ def _prob_tol(reference_prob: float) -> float: + """Per-step cap when Spyre and HF picked the same token.""" return min(ABS_TOL, REL_TOL * reference_prob) +def _tie_tol(reference_prob: float) -> float: + """Bound for accepting a token disagreement as a near-tie rather than a regression.""" + return min(TIE_ABS_TOL, REL_TOL * reference_prob) + + +def _assert_mean_prob_error(model: str, prompt: str, diffs: list[float]) -> None: + """The bound that holds quality: drift spread over a prompt fails here well before any + single step reaches ``ABS_TOL``.""" + if not diffs: + return + mean = sum(diffs) / len(diffs) + print(f" prob error over {len(diffs)} compared steps: mean={mean:.4f} max={max(diffs):.4f}") + assert mean <= MEAN_ABS_TOL, ( + f"{model}: mean probability error {mean:.4f} over {len(diffs)} steps exceeds " + f"{MEAN_ABS_TOL:.4f} for prompt {prompt!r} -- the distribution drifted as a whole, " + f"which no single-step bound catches. A regression, not a tolerance to raise." + ) + + def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOutput) -> int: completion = output.outputs[0] token_ids = list(completion.token_ids) @@ -222,6 +253,7 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu f"{model}: generated {len(token_ids)} tokens, reference has {len(hf_result['token_ids'])}" ) + diffs: list[float] = [] for step, (hf_id, hf_logprob, token_id, logprob) in enumerate( zip(hf_result["token_ids"], hf_result["logprobs"], token_ids, logprobs, strict=True) ): @@ -241,13 +273,14 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu f"{NUM_LOGPROBS}, so the distributions disagree outright, {detail}" ) spyre_hf_prob = math.exp(spyre_hf.logprob) - assert abs(spyre_hf_prob - hf_prob) <= tol, ( - f"{model}: wrong token and p(HF token) differs by more than {tol:.4f} " + ref_tol = _tie_tol(hf_prob) + assert abs(spyre_hf_prob - hf_prob) <= ref_tol, ( + f"{model}: wrong token and p(HF token) differs by more than {ref_tol:.4f} " f"(Spyre {spyre_hf_prob:.4f} vs HF {hf_prob:.4f}), {detail}" ) # A tie also means Spyre ranks the two level, so a flat HF distribution cannot - # excuse Spyre being confident elsewhere. Doubled: both may drift by `tol`. - tie_tol = 2 * tol + # excuse Spyre being confident elsewhere. Doubled: both may drift by `ref_tol`. + tie_tol = 2 * ref_tol assert abs(prob - spyre_hf_prob) <= tie_tol, ( f"{model}: wrong token, and Spyre puts it {prob - spyre_hf_prob:.4f} > " f"{tie_tol:.4f} above HF's token (p={spyre_hf_prob:.4f}), so this is not " @@ -257,10 +290,13 @@ def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOu f" diverged on a near-tie at {detail}; p(HF token) on Spyre " f"{spyre_hf_prob:.4f}; not comparing further" ) + _assert_mean_prob_error(model, hf_result["prompt"], diffs) return step assert abs(hf_prob - prob) <= tol, ( f"{model}: probability differs by more than {tol:.4f}, {detail}" ) + diffs.append(abs(hf_prob - prob)) + _assert_mean_prob_error(model, hf_result["prompt"], diffs) return len(token_ids) From 28051e09b31ead1f231a6fd87d50587ee3bfa141 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Thu, 10 Sep 2026 10:57:54 +0200 Subject: [PATCH 17/18] count the matched-step floor per case, not per prompt Signed-off-by: Yannick Schnider --- tests/e2e/test_model_quality.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index f059af107..64f8d6a1e 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -73,7 +73,10 @@ NUM_LOGPROBS = 20 # A near-tie split ends the comparison, so without a floor a case that mispredicts at step 0 -# on every prompt would pass having compared nothing. +# on every prompt would pass having compared nothing. Counted over the case, not per prompt: +# where the reference itself is a coin flip (granite-4.1's second prompt opens on p=0.4961) +# fp16 drift alone decides the argmax, and one prompt truncating to zero says nothing about +# output quality -- but every prompt truncating still fails. MIN_MATCHED_FRACTION = 0.5 MAX_MODEL_LEN = 256 @@ -141,12 +144,13 @@ def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> No f"near-tie and gate little -- see MODEL_PROMPTS in " f"tests/data/generate_decoder_output_refs.py." ) - min_matched = math.ceil(MIN_MATCHED_FRACTION * max_tokens) - assert all(n >= min_matched for n in matched), ( - f"{model}: matched {per_prompt} reference steps per prompt, under the " - f"{min_matched}/{max_tokens} floor -- the near-tie split came too early to gate " - f"anything. Every prompt matched all {max_tokens} steps when the reference was " - f"taken, so treat this as a regression, not as a floor to lower." + total_steps = len(prompts) * max_tokens + min_matched = math.ceil(MIN_MATCHED_FRACTION * total_steps) + assert sum(matched) >= min_matched, ( + f"{model}: matched {sum(matched)}/{total_steps} reference steps ({per_prompt} per " + f"prompt), under the {min_matched}/{total_steps} floor -- the near-tie splits came too " + f"early to gate anything. Every prompt matched all {max_tokens} steps when the " + f"reference was taken, so treat this as a regression, not as a floor to lower." ) From adad7cb04b53d0fe087b7fd164916cb7b66c2ab5 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Fri, 11 Sep 2026 01:19:15 +0200 Subject: [PATCH 18/18] test(e2e): compare the model-quality gate against live HF, not cached refs Drop the checked-in reference JSONs and the scripts that generated them. Each gate now computes its HF side in-run through upstream vLLM's HfRunner and judges it with upstream's own check_logprobs_close / check_embeddings_close, reached through the pinned tests/ tree the upstream suite already clones. That clone helper moves out of pytest_plugin.py into spyre_testing_plugin/ upstream.py. Since tests.* now resolves to upstream's package rather than this repo's directory, sibling test modules import each other as top-level modules. Signed-off-by: Yannick Schnider --- .github/workflows/_test_matrix.yaml | 1 - .pre-commit-config.yaml | 3 - Makefile | 2 +- docs/contributing/README.md | 81 +- pyproject.toml | 2 +- tests/attention/test_spyre_attn_recorder.py | 10 +- tests/conftest.py | 79 ++ tests/data/decoder_output_refs.json | 907 ------------------ tests/data/encoder_embed_refs.json | 1 - tests/data/generate_decoder_output_refs.py | 143 --- tests/data/generate_encoder_embed_refs.py | 74 -- tests/data/generate_rerank_score_refs.py | 119 --- tests/data/rerank_score_refs.json | 40 - tests/e2e/test_compile.py | 34 +- tests/e2e/test_encoder_models.py | 228 ++--- tests/e2e/test_model_quality.py | 321 +++---- .../spyre_testing_plugin/pytest_plugin.py | 211 +--- tests/plugin/spyre_testing_plugin/upstream.py | 269 ++++++ 18 files changed, 620 insertions(+), 1905 deletions(-) delete mode 100644 tests/data/decoder_output_refs.json delete mode 100644 tests/data/encoder_embed_refs.json delete mode 100644 tests/data/generate_decoder_output_refs.py delete mode 100644 tests/data/generate_encoder_embed_refs.py delete mode 100644 tests/data/generate_rerank_score_refs.py delete mode 100644 tests/data/rerank_score_refs.json create mode 100644 tests/plugin/spyre_testing_plugin/upstream.py diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml index f1b24411b..6efe95b48 100644 --- a/.github/workflows/_test_matrix.yaml +++ b/.github/workflows/_test_matrix.yaml @@ -596,7 +596,6 @@ jobs: spyre-rpms.lock tests/**/*.py tests/**/*.yaml - tests/data/** spyre_inference/**/*.py spyre_inference/**/*.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 299414fc5..71a7fcacd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,9 +16,6 @@ repos: rev: v1.41.0 hooks: - id: typos - # Reference data holds tokenizer fragments, which split words mid-token and which - # this hook's --write-changes silently rewrites into real words. - exclude: '^tests/data/.*\.json$' - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.45.0 hooks: diff --git a/Makefile b/Makefile index d025afe1d..a704a5ba8 100644 --- a/Makefile +++ b/Makefile @@ -199,7 +199,7 @@ test-smoke-shard-%: $(MAKE) test-smoke-shard SMOKE_SHARD_ID=$* JUNIT_XML=$(JUNIT_XML) # Carved out of smoke: every case compiles a product model, up to the 31B decoders. -test-model-quality: ## Run the product-model output-quality gates against the cached HF references. Unsharded (local full run). +test-model-quality: ## Run the product-model output-quality gates against live CPU HF. Unsharded (local full run). $(MAKE) run-one MARK_OVERRIDE='model_quality and not (distributed or upstream)' JUNIT_XML=$(JUNIT_XML) # CI fans the gate out across parallel shard jobs like smoke/attention; the plugin's diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 96d033815..c6eae4f84 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -86,9 +86,17 @@ pytest --upstream -m "attention" The `model_quality` marker gates the product models on their output: each is loaded **compiled** (the platform default) and compared against a CPU HF reference ā€” greedy token -ids and per-token probabilities for the decoders (`tests/e2e/test_model_quality.py`), -cosine similarity for the embedding models and sigmoid scores plus document ranking for -the cross-encoder rerankers (`tests/e2e/test_encoder_models.py`). +ids for the decoders (`tests/e2e/test_model_quality.py`), cosine similarity for the embedding +models and scores for the cross-encoder rerankers (`tests/e2e/test_encoder_models.py`). + +The decoder gate follows upstream's hybrid-model test directly: the reference comes live from +`HfRunner.generate_greedy_logprobs_limit`, Spyre output from +`VllmRunner.generate_greedy_logprobs`, and `check_logprobs_close` compares them. Embeddings +likewise use upstream's `check_embeddings_close`. These helpers come from the pinned vLLM +`tests/` tree via `spyre_testing_plugin.upstream.ensure_upstream_tests_importable` (the same +clone the upstream suite uses, resolved on first use so a deselected job never pays for it). +The `hf_runner` / `vllm_runner` / `example_prompts` fixtures in `tests/conftest.py` mirror +upstream's, so each gate reads like the model test it is modelled on. ```bash make test-model-quality # the whole gate, one card @@ -97,52 +105,33 @@ make test-model-quality-shard-0 # one CI shard (QUALITY_SHARDS=6) CI runs the gate as `QUALITY_SHARDS` parallel 1-card jobs, weighted by recorded runtime like the smoke and attention suites. The slowest single case bounds the useful shard count, so -resize with the `rebalance-test-shards` skill rather than by raising it on a hunch. - -The models are too large to run through transformers in CI, so the references are checked -into `tests/data/` and regenerated by hand where the weights are cached: - -```bash -python tests/data/generate_decoder_output_refs.py --models ibm-granite/granite-4.1-8b -python tests/data/generate_encoder_embed_refs.py -python tests/data/generate_rerank_score_refs.py -``` - -Regenerate only when the *expected* output changes (a new model or prompt), never to make a -failing test pass ā€” that is the regression the gate exists to catch. Prompt sets are per -model: see `MODEL_PROMPTS` and `MODEL_DOCUMENTS` in the generators. - -Every gated model is pinned to a revision, in the generator's `MODEL_REVISIONS` and in -`.github/cache_config/hf_models_and_datasets.yaml`. Each generator records the revision it -measured in its JSON and the test loads that one back, so bumping a pin means regenerating -that model's reference. - -`SPYRE_TEST_MEAN_ABS_TOL` / `SPYRE_TEST_ABS_TOL` / `SPYRE_TEST_REL_TOL` (decoder -probabilities) and `SPYRE_TEST_SCORE_ABS_TOL` / `SPYRE_TEST_SCORE_REL_TOL` (reranker scores) -set the tolerances. For a low-confidence reference the stricter of the absolute and relative -bound applies, so it is held to a fraction rather than to the same absolute margin. Reranker -ranking is checked separately from the per-score bound. - -The decoder gate is aggregate-first: `SPYRE_TEST_MEAN_ABS_TOL` bounds each prompt's *mean* -error and is what holds quality, while `SPYRE_TEST_ABS_TOL` only caps a single step against -gross breakage. A reference near p=0.5 is maximally ill-conditioned (`dp/dlogit` peaks at -`p(1-p)`), and one compiled graph has measured 0.115 apart on such a step between two CI pods -with every token still exact ā€” a tight per-step bound buys flakiness, not coverage. Each case -prints `mean=`/`max=` per prompt, so a failure is readable without a rerun. -`SPYRE_TEST_TIE_ABS_TOL` holds token disagreements to a tighter bound, since picking a -different token is a stronger signal than drift. The FP8 decoder checkpoints are -load-and-decode cases with no reference of their own ā€” their unquantized siblings gate the -numerics. - -A greedy path that diverges from HF on a near-tie cannot be compared past the split, so -each decoder case prints how many reference steps it matched and fails below -`MIN_MATCHED_FRACTION` of them. Read the printed line rather than the pass/fail alone: a -prompt that stops early gates very little, and the fix is a more confident prompt, not a -looser tolerance or a lower floor. +resize with the `rebalance-test-shards` skill. + +No reference data is checked in: every gate computes its HF side in-run, so there is nothing +to regenerate and nothing to go stale. Each gated model is still pinned to a revision, in the +test's `MODEL_REVISIONS` and in `.github/cache_config/hf_models_and_datasets.yaml`. + +The decoders carry no tolerance of their own: `check_logprobs_close` compares greedy token +ids and, where they disagree, requires each side's token to be in the other's top-N. +Embeddings use upstream's `check_embeddings_close(tol=1e-2)`. The rerankers keep local bounds +(`SPYRE_TEST_SCORE_ABS_TOL` / `SPYRE_TEST_SCORE_REL_TOL`, the stricter of the two applying), +since upstream has no reranker helper and its cross-encoder test's inlined tolerances only +hold with both sides at the same precision; their ranking is checked apart from the per-score +bound. The FP8 decoder checkpoints are load-and-decode cases with no comparison at all ā€” +transformers does not dequantize compressed-tensors on CPU, so their unquantized siblings +gate the numerics. #### Upstream Test Integration -Upstream tests are cloned from the vLLM repository at the commit pinned in `pyproject.toml`, fetching only the `tests/` directory. The clone happens on demand, the first time a run asks for upstream tests (see the marker gate above). Cloned tests are cached in `~/.cache/vllm-upstream-tests` (or `$XDG_CACHE_HOME/vllm-upstream-tests`) with separate worktrees per commit, allowing multiple vLLM versions to be tested simultaneously. All upstream tests run with `VLLM_PLUGINS=spyre_inference,spyre_inference_ops` set automatically. Pointing the plugin at a vLLM checkout instead of the cache is the one case that still needs the flag by hand: `pytest -p spyre_testing_plugin.pytest_plugin -m upstream` from the checkout root. See `tests/plugin/spyre_testing_plugin/pytest_plugin.py` for implementation details. +Upstream tests are cloned from the vLLM repository at the commit pinned in `pyproject.toml`, fetching only the `tests/` directory. The clone happens on demand, the first time a run asks for upstream tests (see the marker gate above). Cloned tests are cached in `~/.cache/vllm-upstream-tests` (or `$XDG_CACHE_HOME/vllm-upstream-tests`) with separate worktrees per commit, allowing multiple vLLM versions to be tested simultaneously. All upstream tests run with `VLLM_PLUGINS=spyre_inference,spyre_inference_ops` set automatically. Pointing the plugin at a vLLM checkout instead of the cache is the one case that still needs the flag by hand: `pytest -p spyre_testing_plugin.pytest_plugin -m upstream` from the checkout root. See `tests/plugin/spyre_testing_plugin/upstream.py` (the clone) and `pytest_plugin.py` (collection and filtering) for implementation details. + +Local tests can import upstream test helpers from that same clone via +`spyre_testing_plugin.upstream.ensure_upstream_tests_importable` (see the model-quality gate +above). One constraint comes with it: upstream's `tests/` is a real package and this repo's is +an `__init__.py`-less directory, and a real package wins that name whatever the `sys.path` +order ā€” so **`tests.*` always means upstream's tree**, and may only be used to reach an +upstream helper. Sibling test modules here import each other as top-level modules +(`from test_spyre_attn import ...`), which is what pytest already registers them as. !!! tip To force a re-clone, remove `~/.cache/vllm-upstream-tests`. diff --git a/pyproject.toml b/pyproject.toml index 6b2c05727..3f81d6ae9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -278,7 +278,7 @@ markers = [ "encoder_attention: encoder-attention subset (tests/attention/test_spyre_encoder_attn.py); also marked attention, split into its own CI job", "probe: torch-spyre backend probes (tests/probes/); run in their own CI job, excluded from integration", "uses_subprocess: tests that spawn a subprocess engine; the plugin runs them first, before the main process initializes the Spyre device", - "model_quality: product-model output gates vs cached HF references (tests/e2e/test_model_quality.py and the compiled encoder cases in test_encoder_models.py); compiles models from 125M encoders up to 31B decoders, runs in its own CI job", + "model_quality: product-model output gates vs live CPU HF, compared with upstream vLLM's check_logprobs_close / check_embeddings_close (tests/e2e/test_model_quality.py and the compiled encoder cases in test_encoder_models.py); compiles models from 125M encoders up to 31B decoders, runs in its own CI job", "rotary: Spyre RoPE tests (tests/custom_ops/test_rotary_embedding.py, tests/multimodal/test_pixtral.py)", "fp8: FP8 quantization tests (tests/custom_ops/test_fp8.py)", "conv: Spyre Conv2d custom-op tests (tests/custom_ops/test_conv.py)", diff --git a/tests/attention/test_spyre_attn_recorder.py b/tests/attention/test_spyre_attn_recorder.py index 270584bbf..ae972721e 100644 --- a/tests/attention/test_spyre_attn_recorder.py +++ b/tests/attention/test_spyre_attn_recorder.py @@ -164,7 +164,7 @@ def test_real_metadata_dispatch_compiles_nothing(self, impl, kv_cache): metadata for unbucketed kv_lens through ``SpyreAttentionMetadataBuilder`` and dispatches on the block counts ``build()`` actually produced. """ - from tests.attention.test_spyre_attn import _padded_mask_metadata + from test_spyre_attn import _padded_mask_metadata # Built from the live config, not make_bucketer's narrower stand-in, so # this bucketer and the builder's derive from the same config. @@ -192,7 +192,7 @@ def test_real_metadata_dispatch_compiles_nothing(self, impl, kv_cache): def test_mixed_batch_dispatch_compiles_nothing(self, impl, kv_cache): """A mixed batch dispatches two query widths; both must be recorded.""" - from tests.attention.test_spyre_attn import _padded_mask_metadata + from test_spyre_attn import _padded_mask_metadata bucketer = SpyreAttnBucketer(get_current_vllm_config()) impl.record_graphs(torch.device("cpu"), bucketer, kv_cache) @@ -227,7 +227,7 @@ def test_wide_chunk_beside_short_decode_stays_on_recorded_keys( three query buckets, and only bites when a chunk is wider than another sequence's padded KV, so the other recorder tests never reach it. """ - from tests.attention.test_spyre_attn import _padded_mask_metadata + from test_spyre_attn import _padded_mask_metadata cfg = get_current_vllm_config() monkeypatch.setattr(cfg.scheduler_config, "max_num_batched_tokens", 2048) @@ -265,7 +265,7 @@ def test_wide_chunk_beside_short_decode_stays_on_recorded_keys( def test_mixed_batch_row_tables_keep_their_own_width(self, impl, kv_cache): """The recorded key is not enough: the row table's width is a guard too.""" - from tests.attention.test_spyre_attn import _padded_mask_metadata + from test_spyre_attn import _padded_mask_metadata metadata = _padded_mask_metadata( [(32, 300), (1, 200), (1, 65)], @@ -292,7 +292,7 @@ def test_mixed_batch_real_row_tables_compile_nothing(self, impl, kv_cache): The other mixed-batch tests reach the kernel through ``_record_one``, which rebuilds the row table itself and so cannot see a dispatcher/recorder drift. """ - from tests.attention.test_spyre_attn import _padded_mask_metadata + from test_spyre_attn import _padded_mask_metadata bucketer = SpyreAttnBucketer(get_current_vllm_config()) impl.record_graphs(torch.device("cpu"), bucketer, kv_cache) diff --git a/tests/conftest.py b/tests/conftest.py index 156f14d4f..2cbc7b06f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import math + import pytest from spyre_testing_plugin.tags import result_tags @@ -35,3 +37,80 @@ def _emit_result_tags(request, record_property): params = getattr(getattr(request.node, "callspec", None), "params", {}) for name, value in result_tags(params): record_property(name, value) + + +@pytest.fixture(scope="session") +def hf_runner(): + """Upstream's ``HfRunner``, resolved lazily so the tree is only cloned when a test + that needs it is collected.""" + from spyre_testing_plugin.upstream import ensure_upstream_tests_importable + + ensure_upstream_tests_importable() + from tests.conftest import HfRunner + + class CpuHfRunner(HfRunner): + def get_default_device(self): + return "cpu" + + return CpuHfRunner + + +@pytest.fixture +def example_prompts() -> list[str]: + """Upstream's ``example_prompts``, read with ``readlines()`` as its ``_read_prompts`` + does -- trailing newlines included, since those are the strings its model tests send.""" + from spyre_testing_plugin.upstream import ensure_upstream_tests_importable + + tests_dir = ensure_upstream_tests_importable() + with open(tests_dir / "prompts" / "example.txt") as f: + return f.readlines() + + +@pytest.fixture +def hf_embeddings(hf_runner): + """Live CPU HF embeddings for `(model, revision, prompts)`. + + `is_sentence_transformer=True` applies the checkpoint's own pooling and normalization, + matching what the vLLM side runs when no `pooler_config` overrides it. Prompts must + arrive stripped: sentence-transformers strips its inputs, so otherwise the two sides + tokenize different text. + """ + + def _embed(model: str, revision: str, prompts: list[str]) -> list[list[float]]: + with hf_runner(model, revision=revision, is_sentence_transformer=True) as hf_model: + return hf_model.encode(prompts) + + return _embed + + +@pytest.fixture +def assert_embeddings_close(): + """Upstream's `check_embeddings_close`, plus a finiteness check it omits.""" + from spyre_testing_plugin.upstream import ensure_upstream_tests_importable + + ensure_upstream_tests_importable() + from tests.models.utils import check_embeddings_close + + def _assert(label: str, embeddings, refs, tol: float = 1e-2) -> None: + for embedding in embeddings: + assert all(math.isfinite(x) for x in embedding), f"{label}: non-finite embedding value" + check_embeddings_close( + embeddings_0_lst=refs, + embeddings_1_lst=embeddings, + name_0=f"hf ({label})", + name_1=f"spyre ({label})", + tol=tol, + ) + + return _assert + + +@pytest.fixture(scope="session") +def vllm_runner(): + """Upstream's ``VllmRunner``, from the pinned vLLM ``tests/`` tree.""" + from spyre_testing_plugin.upstream import ensure_upstream_tests_importable + + ensure_upstream_tests_importable() + from tests.conftest import VllmRunner + + return VllmRunner diff --git a/tests/data/decoder_output_refs.json b/tests/data/decoder_output_refs.json deleted file mode 100644 index 2791ccb00..000000000 --- a/tests/data/decoder_output_refs.json +++ /dev/null @@ -1,907 +0,0 @@ -{ - "google/gemma-4-26B-A4B": { - "dtype": "float32", - "max_tokens": 16, - "results": [ - { - "logprobs": [ - -0.608086, - -1.174431, - -1.03372, - -1.044913, - -2.672324, - -0.033213, - -1.656933, - -2.380829, - -0.590173, - -0.130631, - -0.20754, - -0.13099, - -0.105282, - -0.604167, - -0.835592, - -0.751586 - ], - "prompt": "What are IBMs main businesses?", - "text": "\n\nWhat is the difference between a product and a service?\n\nWhat is the", - "token_ids": [ - 108, - 3689, - 563, - 506, - 4954, - 1534, - 496, - 1698, - 532, - 496, - 2509, - 236881, - 108, - 3689, - 563, - 506 - ], - "tokens": [ - "\n\n", - "What", - " is", - " the", - " difference", - " between", - " a", - " product", - " and", - " a", - " service", - "?", - "\n\n", - "What", - " is", - " the" - ] - }, - { - "logprobs": [ - -1.890419, - -1.724759, - -1.272666, - -2.045985, - -0.647059, - -2.163997, - -0.357102, - -1.038351, - -1.267442, - -0.444188, - -1.555656, - -0.700945, - -1.350318, - -0.928785, - -0.511243, - -0.759674 - ], - "prompt": "The capital of France is", - "text": " a city of romance, art, and culture. It is also a city of", - "token_ids": [ - 496, - 3207, - 529, - 30875, - 236764, - 1610, - 236764, - 532, - 6540, - 236761, - 1030, - 563, - 992, - 496, - 3207, - 529 - ], - "tokens": [ - " a", - " city", - " of", - " romance", - ",", - " art", - ",", - " and", - " culture", - ".", - " It", - " is", - " also", - " a", - " city", - " of" - ] - }, - { - "logprobs": [ - -0.132, - -0.319847, - -0.004313, - -1e-06, - -0.724456, - -0.466033, - -0.188152, - -1.336368, - -0.037957, - -0.064367, - -0.043371, - -0.016508, - -0.004109, - -0.033714, - -0.003028, - -0.000332 - ], - "prompt": "Q: What is the largest planet in our solar system?\nA:", - "text": " Jupiter\n\nQ: What is the smallest planet in our solar system?\nA", - "token_ids": [ - 52895, - 108, - 236935, - 236787, - 2900, - 563, - 506, - 21548, - 13401, - 528, - 1023, - 10321, - 1458, - 236881, - 107, - 236776 - ], - "tokens": [ - " Jupiter", - "\n\n", - "Q", - ":", - " What", - " is", - " the", - " smallest", - " planet", - " in", - " our", - " solar", - " system", - "?", - "\n", - "A" - ] - } - ], - "revision": "24548b62aa021d562695c04aaf7758a1ea47990b" - }, - "google/gemma-4-31B": { - "dtype": "float32", - "max_tokens": 16, - "results": [ - { - "logprobs": [ - -1.186606, - -1.509682, - -1.086679, - -1.072163, - -2.122931, - -2.515233, - -0.972521, - -1.722148, - -0.336788, - -0.343377, - -1.098584, - -0.801154, - -0.545086, - -0.649821, - -0.777465, - -0.235408 - ], - "prompt": "What are IBMs main businesses?", - "text": "\n\nWhat are the main businesses of IBM?\n\nWhat are the main businesses of", - "token_ids": [ - 108, - 3689, - 659, - 506, - 1689, - 8634, - 529, - 31209, - 236881, - 108, - 3689, - 659, - 506, - 1689, - 8634, - 529 - ], - "tokens": [ - "\n\n", - "What", - " are", - " the", - " main", - " businesses", - " of", - " IBM", - "?", - "\n\n", - "What", - " are", - " the", - " main", - " businesses", - " of" - ] - }, - { - "logprobs": [ - -1.829299, - -1.579614, - -1.214549, - -2.250874, - -0.600629, - -2.059917, - -0.471437, - -0.932735, - -1.361293, - -0.381795, - -1.460487, - -0.791934, - -0.959909, - -0.507025, - -0.303709, - -0.436555 - ], - "prompt": "The capital of France is", - "text": " a city of romance, art, and culture. It is also a city of", - "token_ids": [ - 496, - 3207, - 529, - 30875, - 236764, - 1610, - 236764, - 532, - 6540, - 236761, - 1030, - 563, - 992, - 496, - 3207, - 529 - ], - "tokens": [ - " a", - " city", - " of", - " romance", - ",", - " art", - ",", - " and", - " culture", - ".", - " It", - " is", - " also", - " a", - " city", - " of" - ] - }, - { - "logprobs": [ - -0.236248, - -0.423639, - -0.031093, - -1.6e-05, - -0.70528, - -0.530978, - -0.247519, - -1.565231, - -0.161889, - -0.029527, - -0.112657, - -0.049126, - -0.016049, - -0.003028, - -0.043576, - -0.004413 - ], - "prompt": "Q: What is the largest planet in our solar system?\nA:", - "text": " Jupiter\n\nQ: What is the second largest planet in our solar system?\n", - "token_ids": [ - 52895, - 108, - 236935, - 236787, - 2900, - 563, - 506, - 1855, - 7488, - 13401, - 528, - 1023, - 10321, - 1458, - 236881, - 107 - ], - "tokens": [ - " Jupiter", - "\n\n", - "Q", - ":", - " What", - " is", - " the", - " second", - " largest", - " planet", - " in", - " our", - " solar", - " system", - "?", - "\n" - ] - } - ], - "revision": "5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89" - }, - "ibm-granite/granite-3.3-8b-instruct": { - "dtype": "float32", - "max_tokens": 16, - "results": [ - { - "logprobs": [ - -0.000607, - -0.749758, - -0.106717, - -3e-05, - -0.088363, - -0.000342, - -0.696137, - -0.000183, - -0.585119, - -0.475057, - -0.21888, - -0.00091, - -3e-06, - -0.41757, - -0.003762, - -0.305773 - ], - "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup.\n\n### Response:", - "text": "\n\n1. Gather ingredients: 1 whole chicken, 2", - "token_ids": [ - 203, - 203, - 35, - 32, - 570, - 3790, - 42568, - 44, - 225, - 35, - 9844, - 663, - 21217, - 30, - 225, - 36 - ], - "tokens": [ - "\n", - "\n", - "1", - ".", - " G", - "ather", - " ingredients", - ":", - " ", - "1", - " whole", - " ch", - "icken", - ",", - " ", - "2" - ] - }, - { - "logprobs": [ - -0.011008, - -0.0722, - -0.248327, - -0.110021, - -0.000184, - -2.7e-05, - -5e-06, - -0.000195, - -0.061382, - -1e-06, - -0.213029, - -0.396524, - -2e-06, - -0.394157, - -0.950517, - -0.376066 - ], - "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nWhat are the main businesses of IBM?\n\n### Response:", - "text": "\nIBM, or International Business Machines Corporation, has diversified its business portfolio", - "token_ids": [ - 203, - 36671, - 30, - 556, - 21554, - 17265, - 488, - 11386, - 13180, - 30, - 1401, - 34142, - 1639, - 2819, - 12590, - 31397 - ], - "tokens": [ - "\n", - "IBM", - ",", - " or", - " International", - " Business", - " M", - "achines", - " Corporation", - ",", - " has", - " divers", - "ified", - " its", - " business", - " portfolio" - ] - }, - { - "logprobs": [ - -0.001327, - -0.615451, - -0.000136, - -0.000182, - -0.879656, - -0.529216, - -0.394932, - -0.133598, - -0.267801, - -0.004337, - -0.013194, - -4.6e-05, - -2e-06, - -0.1755, - -0.0, - -0.254457 - ], - "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nConvert char to string in Java.\n\n### Response:", - "text": "\nIn Java, a character is represented by the `char` data type,", - "token_ids": [ - 203, - 383, - 4949, - 30, - 312, - 5341, - 438, - 17718, - 810, - 322, - 529, - 1594, - 82, - 706, - 827, - 30 - ], - "tokens": [ - "\n", - "In", - " Java", - ",", - " a", - " character", - " is", - " represented", - " by", - " the", - " `", - "char", - "`", - " data", - " type", - "," - ] - } - ], - "revision": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b" - }, - "ibm-granite/granite-4.1-8b": { - "dtype": "float32", - "max_tokens": 16, - "results": [ - { - "logprobs": [ - -0.302353, - -0.299105, - -6.4e-05, - -0.083195, - -0.014468, - -9e-06, - -0.006912, - -0.190234, - -0.263843, - -0.093166, - -0.036535, - -0.008035, - -0.346234, - -3e-06, - -0.001928, - -0.717883 - ], - "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup.\n\n### Response:", - "text": " \n1. **Gather Ingredients**: Collect all necessary ingredients such as chicken breasts", - "token_ids": [ - 720, - 16, - 13, - 3146, - 38, - 1894, - 52275, - 96618, - 21153, - 682, - 5995, - 14293, - 1778, - 439, - 16553, - 37449 - ], - "tokens": [ - " \n", - "1", - ".", - " **", - "G", - "ather", - " Ingredients", - "**:", - " Collect", - " all", - " necessary", - " ingredients", - " such", - " as", - " chicken", - " breasts" - ] - }, - { - "logprobs": [ - -0.701019, - -0.014904, - -0.251564, - -0.000126, - -0.0, - -1e-06, - -0.002019, - -0.002093, - -0.062779, - -0.073797, - -0.088115, - -0.000268, - -0.009194, - -0.085598, - -0.057024, - -0.298725 - ], - "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nWhat are the main businesses of IBM?\n\n### Response:", - "text": " \n\nIBM (International Business Machines Corporation) is a multinational technology company with a diverse", - "token_ids": [ - 4815, - 68838, - 320, - 34746, - 8184, - 45004, - 13332, - 8, - 374, - 264, - 69026, - 5557, - 2883, - 449, - 264, - 17226 - ], - "tokens": [ - " \n\n", - "IBM", - " (", - "International", - " Business", - " Machines", - " Corporation", - ")", - " is", - " a", - " multinational", - " technology", - " company", - " with", - " a", - " diverse" - ] - }, - { - "logprobs": [ - -0.083338, - -0.150565, - -1.2e-05, - -1e-05, - -1.02652, - -0.382205, - -6.1e-05, - -0.001031, - -2.5e-05, - -1e-06, - -1e-05, - -7.1e-05, - -0.006363, - -0.026268, - -0.281138, - -0.042108 - ], - "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nConvert char to string in Java.\n\n### Response:", - "text": " \n\nTo convert a character to a string in Java, you can use the `", - "token_ids": [ - 4815, - 1271, - 5625, - 264, - 3752, - 311, - 264, - 925, - 304, - 8102, - 11, - 499, - 649, - 1005, - 279, - 1595 - ], - "tokens": [ - " \n\n", - "To", - " convert", - " a", - " character", - " to", - " a", - " string", - " in", - " Java", - ",", - " you", - " can", - " use", - " the", - " `" - ] - } - ], - "revision": "1504002f650e656a0a3789d99574df12e3e94ed0" - }, - "meta-llama/Llama-3.1-8B-Instruct": { - "dtype": "float32", - "max_tokens": 16, - "results": [ - { - "logprobs": [ - -0.48447, - -0.932041, - -0.928222, - -0.034002, - -0.69502, - -0.002108, - -0.00067, - -0.004681, - -0.565966, - -0.349952, - -0.521094, - -0.55201, - -5.5e-05, - -0.076344, - -1.010146, - -0.003694 - ], - "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup.\n\n### Response:", - "text": " \n\nHere's a step-by-step guide to preparing delicious chicken soup:\n\n1.", - "token_ids": [ - 4815, - 8586, - 596, - 264, - 3094, - 14656, - 30308, - 8641, - 311, - 20646, - 18406, - 16553, - 19724, - 1473, - 16, - 13 - ], - "tokens": [ - " \n\n", - "Here", - "'s", - " a", - " step", - "-by", - "-step", - " guide", - " to", - " preparing", - " delicious", - " chicken", - " soup", - ":\n\n", - "1", - "." - ] - }, - { - "logprobs": [ - -0.489573, - -0.563746, - -1.127867, - -0.279811, - -0.700401, - -0.172913, - -0.405786, - -0.011894, - -0.274383, - -0.763821, - -1.057111, - -0.333751, - -1.217825, - -0.639492, - -0.759504, - -0.141694 - ], - "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nWhat are the main businesses of IBM?\n\n### Response:", - "text": " \nIBM is a multinational technology and consulting company that operates in various sectors, including", - "token_ids": [ - 720, - 68838, - 374, - 264, - 69026, - 5557, - 323, - 31831, - 2883, - 430, - 27149, - 304, - 5370, - 26593, - 11, - 2737 - ], - "tokens": [ - " \n", - "IBM", - " is", - " a", - " multinational", - " technology", - " and", - " consulting", - " company", - " that", - " operates", - " in", - " various", - " sectors", - ",", - " including" - ] - }, - { - "logprobs": [ - -0.337628, - -1.170689, - -0.002479, - -0.026265, - -0.631445, - -0.105513, - -0.064008, - -0.027309, - -0.001081, - -0.000404, - -0.003033, - -0.078346, - -0.043913, - -0.082205, - -0.045746, - -0.765835 - ], - "prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nConvert char to string in Java.\n\n### Response:", - "text": " \n\nTo convert a char to a string in Java, you can use the `", - "token_ids": [ - 4815, - 1271, - 5625, - 264, - 1181, - 311, - 264, - 925, - 304, - 8102, - 11, - 499, - 649, - 1005, - 279, - 1595 - ], - "tokens": [ - " \n\n", - "To", - " convert", - " a", - " char", - " to", - " a", - " string", - " in", - " Java", - ",", - " you", - " can", - " use", - " the", - " `" - ] - } - ], - "revision": "0e9e39f249a16976918f6564b8830bc894c89659" - } -} diff --git a/tests/data/encoder_embed_refs.json b/tests/data/encoder_embed_refs.json deleted file mode 100644 index 1af5337fa..000000000 --- a/tests/data/encoder_embed_refs.json +++ /dev/null @@ -1 +0,0 @@ -{"ibm-granite/granite-embedding-125m-english":{"embeddings":[[-0.01831,0.00958,-0.02759,-0.01465,-0.01917,-0.01819,-0.00061,-0.02112,-0.01337,0.04932,-0.0282,-0.02612,-0.01843,-0.02734,-0.00497,-0.0166,0.00455,0.00488,0.01215,-0.02673,0.0354,-0.05444,-0.02332,-0.052,-0.03247,-0.01807,0.02454,-0.01648,0.00598,0.0332,-0.01373,-0.02771,0.04199,0.01831,0.04785,-0.02441,-0.00806,-0.00922,0.01184,-0.03491,0.00105,-0.0047,-0.00671,-0.01611,0.01093,0.00084,0.01331,-0.00362,-0.04199,0.03394,0.03687,0.00352,0.00885,-0.00467,0.03687,-0.00371,0.03467,-0.026,-0.02002,-0.01733,-0.01495,0.03223,-0.01587,-0.00036,0.01276,0.04736,0.01941,-0.01953,-0.04565,0.0415,0.01447,-0.026,-0.04468,-0.02673,0.03394,-0.00928,-0.07471,-0.03516,-0.00415,0.02087,0.02112,0.01746,-0.01379,0.02026,-0.03394,-0.02258,0.00577,-0.02917,-0.02393,-0.01514,-0.00668,-0.00983,0.02429,-0.00552,-0.02502,0.00203,-0.02307,0.026,-0.01196,-0.00319,0.00787,-0.00903,0.02063,-0.013,0.04761,0.01337,-0.00255,0.04419,0.02588,0.00867,0.02661,-0.01215,-0.02625,0.04321,0.01483,-0.00167,-0.0152,0.00681,-0.0006,-0.00665,-0.00436,-0.00958,0.01337,-0.03931,0.01917,0.01257,0.00488,0.01495,0.00958,-0.01611,-0.00117,0.0072,0.05371,-0.00854,-0.0105,0.00433,-0.0054,0.05981,-0.02734,-0.02332,0.00171,0.02881,-0.04761,-0.01672,-0.021,-0.00824,0.02734,-0.03809,-0.02185,-0.0144,-0.04663,-0.04663,-0.01257,0.03052,-0.02039,-0.0043,0.05811,-0.00739,-0.02087,0.02283,0.02502,-0.02661,-0.01501,-0.02075,-0.01019,-0.01337,-0.01038,-0.01343,-0.00093,-0.02734,0.02612,-0.02844,-0.00519,0.03833,-0.00916,-0.03369,-0.0076,-0.0293,0.03931,0.00378,0.01196,0.03564,-0.01807,0.01697,-0.00055,-0.02356,-0.00371,0.00577,-0.02551,-0.01257,-0.03174,0.02039,0.02136,-0.01044,0.01105,-0.00934,-0.02319,-0.01929,-0.02502,-0.0238,-0.02612,-0.01294,-0.03687,-0.03906,0.01483,-0.008,-0.02185,0.00163,0.04297,-0.03687,-0.0062,0.00252,-0.04785,-0.33203,0.00879,0.01843,-0.33008,-0.00461,-0.01953,-0.01105,0.02209,-0.05078,-0.00714,-0.03589,0.02368,-0.01428,0.04053,0.03369,-0.01611,-0.00162,0.02576,-0.01709,-0.01123,0.04468,0.0144,-0.05493,0.00153,0.00656,-0.02625,0.00781,-0.01117,-0.03613,-0.02783,0.02771,-0.02954,-0.05835,-0.01514,-0.0199,-0.04565,-0.0354,-0.00198,-0.00421,-0.32617,0.02441,-0.00476,0.02881,-0.0105,-0.02356,0.01019,-0.01685,0.02209,-0.02148,-0.03149,0.01019,-0.00711,0.01385,0.03906,-0.0152,0.00157,-0.01489,-0.03247,0.02905,-0.01782,0.01831,-0.00063,-0.00063,0.02356,-0.01422,-0.04614,-0.04321,0.04028,-0.01624,0.00333,-0.00687,0.01019,-0.01422,-0.01624,-0.00842,-0.03296,0.04297,-0.01447,0.01782,0.00793,-0.04883,0.03101,0.03467,-0.00879,-0.00177,0.02112,-0.00763,-0.0282,0.01385,0.00775,-0.0152,-0.01941,-0.01459,0.02661,0.0293,-0.00824,0.03589,0.02551,0.0188,-0.02673,-0.03113,-0.00598,-0.00916,0.00141,-0.02283,0.01941,-0.00516,-0.04053,-0.00885,-0.02576,-0.06055,-0.01587,-0.00671,0.01868,0.01965,0.01904,0.02173,-0.03345,0.013,-0.02295,-0.00173,-0.02368,-0.03149,-0.02893,-0.00983,-0.03467,0.02893,0.02295,0.01483,0.01562,0.01373,0.00058,-0.02673,0.00331,-0.02234,-0.04565,0.00952,0.0238,0.0141,-0.00562,0.01917,-0.00381,-0.00369,0.01013,0.03687,-0.01807,-0.0094,0.01562,0.0141,-0.04321,-0.03564,0.00977,0.00394,0.00047,-0.0127,0.03418,-0.02515,-0.01648,0.01343,0.00085,0.01501,0.021,-0.01648,0.03149,-0.01221,-0.04639,-0.01733,0.02356,-0.04004,-0.01343,0.01154,-0.00842,0.02283,0.06641,-0.01562,-0.02832,0.0083,-0.03064,0.00766,-0.01453,0.01416,0.01807,-0.01129,-0.021,0.03589,-0.00983,-0.02686,0.021,-0.0062,-0.02673,-0.00903,0.00113,0.13574,-0.03882,-0.0332,0.0481,-0.03247,0.0415,-0.00873,-0.02881,0.02356,0.03003,0.0097,-0.02649,0.00616,-0.0007,-0.00476,-0.07227,0.00879,0.01941,0.02307,0.41602,-0.01263,-0.04468,0.01556,-0.03369,-0.02917,0.03809,-0.01953,0.00903,-0.03223,0.05933,-0.05054,-0.02173,0.0009,0.02063,-0.03687,-0.00144,-0.02454,-0.03516,0.00291,-0.02368,0.01105,-0.00021,0.00504,-0.00879,0.01917,0.04346,0.00861,-0.01648,0.03613,0.00613,-0.0152,-0.021,0.04004,0.0354,-0.0017,-0.03687,-0.03149,-0.0437,-0.00433,0.03857,0.00534,-0.00958,-0.0013,0.02014,-0.02429,0.01422,-0.00307,0.00123,-0.01758,-0.01093,-0.04712,-0.02722,0.02783,-0.0076,-0.03833,0.00653,0.00439,-0.02844,-0.04077,-0.02649,0.05664,-0.05396,-0.04175,-0.03882,-0.02551,-0.01117,0.01184,-0.03931,0.02307,0.02832,-0.00412,-0.00775,0.01416,0.0025,-0.02026,-0.02661,-0.00446,0.00389,0.06006,4e-05,-0.01331,-0.03857,0.0293,-0.01929,-0.02173,-0.01031,0.02075,0.00696,0.03882,-0.02161,0.02014,-0.05566,0.02612,-0.04443,0.02686,0.00952,0.03662,0.0332,0.00922,0.03516,0.01709,-0.01483,0.03735,-0.00482,-0.04346,0.00543,0.01624,0.01196,0.0094,-0.01697,0.01758,-0.03564,0.05444,-0.0354,-0.00806,0.01917,0.02173,0.00504,-0.07715,-0.01312,0.02734,0.0332,-0.04053,0.03345,0.03613,-0.00504,-0.00696,-0.01123,0.0166,-0.05249,-0.02478,-0.00182,-0.008,0.05078,-0.03516,0.04932,-0.03882,0.00546,0.00635,-0.04785,0.02332,-0.05688,-0.00616,-0.0061,-0.01495,-0.00824,-0.03662,-0.052,-0.02844,0.03931,0.01685,0.02478,-0.01819,-0.04663,0.00143,-0.04712,0.03113,0.00854,0.02258,-0.02087,0.05762,0.01575,-0.0076,0.03491,-0.05615,-0.04126,0.01459,0.05469,-0.02441,-0.02161,-0.00842,-0.03662,-0.03369,0.02173,0.00328,0.00187,-0.01373,-0.02075,-0.00626,0.00164,-0.00158,-0.02368,-0.03442,-0.00388,-0.01538,-0.02429,-0.05005,0.0376,0.00632,-0.03735,-0.02844,-0.00276,-0.02222,0.02283,-0.0166,-0.00074,0.0043,0.01343,-0.0004,-0.01587,0.05103,0.02161,-0.01263,-0.0108,0.02905,0.00848,0.01575,-0.00885,0.02917,-0.00757,0.02234,0.02087,-0.02026,-0.03857,0.03296,0.0152,0.02368,0.02014,-0.01276,0.00288,0.00867,-0.00083,0.04858,-0.00723,-0.02148,0.00197,0.02405,0.01855,0.00317,-0.02234,-0.00854,-0.00334,-0.01324,-0.02039,0.05103,-0.02063,0.00385,-0.01221,-0.01587,-0.04224,-0.0152,-0.01514,0.01416,0.00272,-0.00177,0.01343,0.00421,-0.01648,-0.01013,-0.00653,-0.02844,0.00662,-0.00366,-0.01831,-0.02759,0.0024,-0.03442,0.01721,-0.00301,0.01538,0.00064,-0.03296,0.01758,0.01129,-0.00836,-0.03589,-0.00433,0.01373,-0.02832,0.00211,-0.00055,-0.0155,-0.00702,0.0376,0.00086,0.00479,-0.04004,-0.00574,-0.02844,-0.04224,0.02539,0.0304,-0.01257,0.02356,-0.04858,-0.013,-0.03906,-0.01532,-0.0076,-0.00577,0.02502,-0.01587,-0.02722,-0.0105,-0.00964,0.00879,-0.01123,-0.03198,0.00668,-0.04346,-0.03369,-0.04297,0.00439,-0.00272,0.03809,0.02087,-0.00476,-0.02576,0.02283,0.01294,0.03149,-0.03149,0.01868,-0.00934,0.01215,0.02661,0.02087,0.01105,0.00461,-0.01367,-0.01556,-0.00519,0.03979,-0.0238,-0.02722,-0.02808,-0.01056,-0.04785,-0.02454,-5e-05,0.00546,0.01147,0.02039,-0.00037,0.00452,-0.01831,0.00372,-0.01624,0.00378,-0.00013,0.03906,0.01337,-0.01031],[-0.00702,-0.00351,-0.01978,-0.0415,-0.00403,-0.03064,-0.01019,-0.02258,0.00726,0.1084,-0.03223,-0.03418,-0.01892,-0.00562,-0.02283,0.0072,-0.04614,0.0282,-0.00166,0.00677,0.00534,-0.02234,-0.02014,-0.04907,-0.04028,0.00094,-0.02075,-0.03113,-0.00365,0.01868,0.01697,0.01733,0.00836,0.00601,0.04663,-0.03516,-0.00485,-0.00793,0.02258,-0.0061,0.01306,0.05078,-0.04346,-0.00842,-0.01892,-0.00946,0.02087,-0.00033,-0.02673,0.04102,0.01434,-0.01129,0.0047,-0.01611,0.01477,0.00873,0.00995,0.0238,0.01117,0.01154,-0.05591,0.04688,-0.00726,-0.00616,-0.00647,0.03711,0.05005,-0.01611,-0.00952,0.04565,0.01178,-0.03711,-0.00298,-0.02454,-0.00467,0.00156,-0.0564,-0.03833,-0.0437,0.0072,-0.00494,-0.0199,-0.02588,0.03467,0.01178,-0.03113,-0.00583,-0.02136,-0.00677,-0.0166,-0.03113,-0.00842,0.02539,0.0127,-0.01117,0.00018,0.02832,-0.02393,0.01483,0.00702,0.0022,-0.00011,-0.01562,0.01508,0.05225,-0.00916,-0.00812,0.00995,0.04346,0.00035,-0.01038,-0.01227,-0.04175,0.00467,0.01404,0.02454,-0.01404,-0.00032,-0.01324,0.01294,-0.01154,-0.04663,0.01373,0.01648,0.03296,0.03711,0.0032,0.01636,0.01685,-0.02808,-0.00623,-0.02698,0.02722,-0.00334,-0.01404,-0.03223,-0.00772,0.03418,-0.06836,0.01648,-0.00173,0.0152,-0.0119,-0.01166,-0.03198,-0.02673,-0.00504,-0.01129,-0.08154,-0.01349,-0.00128,-0.01636,-0.01074,0.00179,-0.00885,-0.01129,0.01233,-0.03833,-0.01007,0.00121,-0.02563,-0.05493,-0.02136,-0.04688,-0.00772,-0.01758,-0.00708,-0.00334,-0.00467,-0.03088,0.00041,-0.06445,-0.00024,-0.02795,-0.04565,0.00916,-0.02307,-0.0304,0.03027,-0.02014,0.03516,0.02832,0.00662,0.01819,0.00607,0.00167,-0.00458,0.01166,-0.0166,-0.05273,-0.01172,-0.00266,0.00371,-0.00397,0.03833,-0.01099,0.01501,-0.03027,-0.02075,-0.00325,-0.00586,-0.00601,-0.05396,0.00497,0.03442,-0.02087,-0.02856,-0.01416,-0.01276,-0.02502,-0.00317,-0.02771,-0.00331,-0.31641,0.0498,0.04883,-0.31055,-0.00397,-0.00964,-0.00346,-0.04688,-0.00159,0.03442,-0.04346,0.03076,-0.02954,0.00156,-0.00025,0.00989,-0.03198,0.02014,-0.04688,-0.01221,0.03638,-0.01337,-0.05005,0.00708,0.00589,-0.01733,0.0282,-0.01941,-0.01025,-0.00252,0.03491,-0.02869,-0.08789,-0.01733,-0.01929,-0.01733,-0.02405,-0.03125,-0.01251,-0.29883,-0.00482,-0.01251,0.05908,0.01007,-0.026,0.02979,-0.01843,-0.01685,-0.06152,-0.01178,0.02454,-0.01709,0.00531,0.02344,-0.026,-0.00217,0.01709,0.01892,0.03076,0.01038,0.01965,-0.00519,-0.00513,-0.00928,0.00571,-0.01062,-0.02588,-0.00333,-0.01483,0.01514,-0.05005,-0.0238,-0.04395,-0.00357,0.04517,-0.04883,0.05029,-0.0097,0.05444,0.01178,-0.01227,0.02222,0.00193,-0.00903,-0.01324,0.02502,-0.04785,0.00867,0.00232,0.00769,-0.05347,-0.01624,-0.0038,0.0032,0.0282,0.01611,0.06592,0.02747,0.00653,0.00046,-0.0271,-0.02014,0.02454,-0.01483,-0.04395,0.01093,0.00522,-0.04688,-0.00147,-0.00412,0.00052,-0.02039,-0.03931,-0.00818,0.0354,0.00772,0.02429,-0.02954,-0.00351,-0.02637,0.00099,-0.01093,-0.00455,-0.03369,-0.03589,-0.05298,0.03516,0.05347,0.05029,-0.00638,0.00928,-0.03223,-0.02905,0.0481,-0.02307,-0.00403,-0.03369,0.02441,0.01636,0.02014,0.0293,-0.02661,-0.01373,0.026,0.06885,-0.0166,-0.03784,0.04565,0.03516,-0.03955,0.00078,0.03564,-0.02869,-0.02002,0.00668,0.0271,-0.04077,-0.00319,-0.03174,0.01453,-0.02454,-0.021,-0.03149,-0.00066,0.00439,-0.03882,0.00491,0.03491,-0.00867,-0.0034,-0.02466,-0.03027,0.01178,0.02832,0.00377,0.00885,0.03882,-0.03027,-0.02441,0.00995,-0.00717,0.05103,-0.01294,0.0097,0.02795,0.00301,-0.03931,0.00182,-0.01672,0.00586,-0.00304,-0.00757,0.18652,-0.0625,-0.02222,0.03638,-0.03931,0.01831,0.03296,0.00325,-0.0332,0.04248,-0.01587,-0.03198,-0.00491,-0.01819,0.02307,-0.06055,-0.01459,0.03198,0.00191,0.35156,0.01575,-0.00013,0.01709,-0.03027,-0.02307,0.02136,-0.02173,0.00861,-5e-05,0.06494,-0.07031,-0.01434,0.01434,-0.01575,-0.05664,-0.00473,-9e-05,-0.05005,-0.00607,-0.01599,0.01355,0.02222,-0.00739,0.01892,-0.05176,0.04346,0.01843,-0.03149,-0.04932,0.03687,-0.03955,0.00531,-0.01276,0.05396,0.02454,-3e-05,-0.06396,-0.00772,-0.02637,0.0249,-0.00012,-0.00331,0.00412,0.02722,-0.05615,0.00522,0.01251,0.00092,-0.00327,0.01953,-0.0376,-0.04297,0.0304,0.0304,-0.00861,-0.00397,0.04712,-0.04565,-0.0481,-0.04858,0.04053,-0.05664,-0.02649,-0.06299,0.0166,0.00172,0.01318,-0.02722,-0.01941,0.03345,-0.05127,-0.00133,0.04883,-0.00702,-0.02197,0.01807,0.01196,0.02393,0.04492,0.00812,0.05298,-0.00022,-0.0072,-0.03442,-0.03174,-0.04907,-0.00922,-0.03369,0.01746,-0.00595,-0.04663,-0.04614,0.03369,-0.05029,-0.00574,0.00037,0.00934,0.02185,-0.0166,0.0293,0.02087,-0.01062,0.00766,-0.00867,-0.00909,0.0459,0.013,-0.00359,0.04028,-0.01404,-0.0354,-0.00439,0.03113,-0.0304,-0.00934,0.00082,0.0293,-0.00149,-0.04395,-0.01477,0.01575,0.05176,-0.02429,0.013,0.03882,-0.00247,-0.01221,-0.02051,-0.00674,-0.01129,0.02832,-0.00033,0.02722,0.0119,-0.005,0.02661,-0.06128,0.00635,0.02612,-0.00439,0.0415,-0.06641,-0.02771,0.04077,0.01215,-0.00681,-0.00391,-0.0271,-0.00766,0.01892,-0.02026,0.00485,0.0304,-0.004,-0.00074,-0.01807,0.01892,-0.03589,0.0166,-0.00436,0.00192,0.00946,-0.00084,0.0069,-0.01355,-0.01123,-0.02502,0.05127,-0.03882,0.00497,0.02832,-0.01782,-0.03491,0.03882,-0.0293,0.01172,-0.021,-0.052,-0.03076,0.03491,-0.00126,-0.04688,-0.04028,0.01093,-0.03271,-0.00169,-0.00327,0.02832,-0.03711,-0.03027,-0.04004,-0.05591,0.02649,-0.02283,-0.00897,0.03247,-0.00377,0.01575,-0.02026,-0.02405,0.02332,0.01636,-0.01111,-0.02283,0.0069,0.00167,0.0069,0.00237,0.04883,0.00739,0.01459,0.01929,0.00153,-0.02832,0.03784,0.03638,0.03015,-0.01459,-0.02441,0.02246,0.03735,-0.03516,0.00861,0.00094,-0.03979,0.02356,-0.00589,-0.00339,0.01807,-0.04614,-0.0199,-0.02344,-0.03662,-0.00787,0.06299,0.01245,-0.01538,-0.02356,-0.04199,-0.01758,0.00641,-0.04688,0.02795,0.01843,0.04175,0.02197,0.00574,-0.01501,-0.02344,-0.02661,-0.01245,0.00056,-0.0105,-0.02234,0.00519,0.04614,0.02148,0.00479,-0.00781,-0.0498,0.00928,0.00574,0.01831,-0.01196,-0.02588,-0.02283,0.00327,-0.01624,0.00243,0.02087,-0.00879,0.03125,-0.00739,0.00928,-0.0332,-0.01746,0.01379,-0.01007,-0.04907,-0.01544,0.00198,0.00934,-0.02795,0.01544,-0.02881,0.00479,-0.00714,-0.04272,-0.02661,0.00854,-0.01807,-0.03931,-0.04492,-0.01892,0.00025,0.01965,-0.0238,-0.00077,0.05127,-0.00201,0.0072,0.00586,-0.00952,0.00141,0.03784,0.00427,0.02539,-0.01147,0.02563,0.02136,-0.00891,-0.00598,0.00635,-0.03174,0.013,-0.03223,0.03088,0.01514,0.02747,-0.02673,-0.02002,0.01025,0.02014,0.03064,-0.02307,-0.0564,-0.01843,-0.01129,-0.00638,-0.00337,0.02124,-0.03076,0.02039,-0.01324,-0.06689,-0.01685,0.00665,-0.01855,-0.03467,-0.00705,0.03418,0.03198,-0.00394]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."],"revision":"4ab61ffd423be45cd932b21a7c696063d82bf45f"},"ibm-granite/granite-embedding-278m-multilingual":{"embeddings":[[-0.03931,0.04541,0.01062,0.01904,0.0752,-0.04492,0.07568,0.05591,0.05127,0.03247,0.00861,0.00191,0.02673,0.02856,0.01685,0.08447,0.07861,-0.06934,-0.03296,-0.0177,-0.01111,0.02661,-0.01257,0.05908,-0.0094,-0.05591,-0.01782,-0.01245,-0.03979,0.02466,0.00394,0.0304,-0.01978,0.03662,-0.00313,-0.00684,0.02551,0.03345,-0.02734,-0.00039,-0.01965,0.03394,0.01599,0.01093,-0.01379,0.0155,-0.0177,0.04004,0.00458,0.03247,-0.05396,0.0141,0.00218,-0.00365,-0.04272,0.01526,0.06445,0.06592,-0.06445,0.0354,-0.0188,-3e-05,0.01685,-0.00327,0.02942,0.00763,0.06592,-0.06079,0.01337,0.05103,-0.00173,0.00668,0.05518,0.07178,-0.03955,0.00354,-0.02856,0.02319,0.00879,-0.00638,-0.04321,0.03589,-0.01709,0.01917,-0.03394,0.0155,0.02576,0.03076,0.05396,-0.03345,0.03015,-0.01685,-0.04346,-0.00226,-0.00313,-0.00365,0.01746,0.01538,0.05762,-0.03613,0.02759,-0.00052,0.00188,0.01361,0.02246,-0.00565,0.03345,-0.04663,0.0238,-0.0116,0.03979,-0.02979,0.05396,0.0304,0.03589,0.01172,0.01685,0.00069,0.05762,0.02051,0.07764,-0.02832,0.04907,-0.00885,0.02124,0.021,-0.0238,0.04175,0.03931,-0.01068,-0.00977,-0.08154,-0.02917,0.02222,-0.06348,-0.02942,-0.00644,0.01135,0.01019,0.00842,-0.013,-0.01782,-0.02271,0.04126,-0.01013,-0.03638,-0.02393,0.04199,0.03906,0.02051,-0.02917,-0.00885,0.05396,0.03711,-0.05469,0.03809,-0.01843,-0.00668,-0.01062,0.01318,0.0376,-0.008,0.00897,0.021,0.00659,-0.02832,-0.00056,0.05786,0.0141,-0.00209,-0.01831,0.02014,0.00824,-0.04053,-0.02722,0.04224,-0.00195,0.00729,-0.00964,-0.01025,0.02026,-0.02527,-0.11084,0.0354,0.0038,-0.00337,0.0006,-0.04834,-0.00775,0.03857,0.01556,0.00319,0.00287,0.15527,0.0791,-0.03394,-0.05225,0.00235,-0.08691,0.00616,0.0069,0.05322,0.02246,-0.0332,0.00159,0.021,0.05908,0.06006,0.01538,0.04858,-0.02271,0.03247,0.03711,0.05518,0.01599,-0.03101,0.2041,-0.00069,0.00107,-0.06079,0.04541,0.04053,0.01398,0.0199,0.00068,0.02515,0.01941,0.04175,0.05322,0.02161,0.05273,0.04102,-0.02893,-0.05957,0.07617,0.00308,0.02405,0.0332,0.00735,-0.03979,-0.02502,-0.02917,0.0119,0.02734,0.01202,0.0498,0.04224,-0.0437,0.02698,-0.0238,-0.00754,0.00885,0.02527,0.06104,0.07471,0.02234,0.01611,-0.01929,0.06592,0.05664,0.07422,-0.00182,0.00946,-0.00589,0.0376,0.0282,-0.02368,-0.03931,0.04248,0.06641,0.00243,0.06543,0.00934,-0.00916,0.05396,-0.00684,-0.01953,-0.03149,0.01733,0.01282,0.02979,-0.03003,0.0083,0.0304,0.00243,0.07227,0.0188,-0.03662,0.03113,0.00375,-0.02283,0.03369,0.04419,-0.00778,-0.03003,0.02612,0.0415,0.02881,0.02917,0.00342,-0.00127,0.10449,0.04443,-0.00446,0.02539,0.02222,-0.01508,-0.02771,0.0199,-0.01471,0.03345,-0.00365,0.0188,0.05273,-0.01965,0.05078,0.0282,-0.03149,-0.01477,-0.00818,-0.03589,0.03662,0.052,0.0097,0.00285,-0.05469,-0.00256,0.00909,0.00604,0.01208,0.02051,0.01624,0.01349,-0.00211,0.02332,-0.04053,-0.01855,0.12793,-0.04053,0.0072,0.00218,0.07471,-0.00623,-0.00934,0.0108,0.02136,0.0498,-0.03662,0.03198,0.00061,-0.01489,0.00665,-0.0108,0.02832,-0.00221,-0.03198,0.04565,-0.02979,0.00497,0.03467,0.03613,0.04712,-0.00842,-0.01361,0.026,0.01501,0.01471,-0.01538,-0.0304,0.03149,-0.03247,-0.02148,-0.05518,0.02295,-0.00885,0.03101,0.05664,0.01965,0.00787,-0.03015,0.00464,0.01538,0.01556,-0.03711,0.03809,-0.01471,0.00653,-0.01428,-0.07617,0.0141,0.06445,0.0603,-0.00149,-0.00958,0.01019,0.04565,-0.02771,0.02637,0.02417,0.0004,0.04272,-0.03589,0.0022,0.03735,0.04932,-0.02051,0.0097,-0.03271,-0.02283,0.01416,0.03589,0.0354,-0.00287,0.03589,-0.00583,0.03174,-0.02295,-0.01367,0.03271,0.02661,0.03589,-0.02551,0.04663,0.02734,-0.01709,0.03809,0.06396,0.01489,0.00464,0.03955,0.02234,0.0752,0.0062,0.01385,0.00571,0.0031,0.03906,-0.021,0.03491,0.00681,-0.008,0.02576,0.04761,-0.02673,0.02393,-0.04028,0.013,-0.00497,0.01599,0.08984,0.026,-0.0603,-0.0144,-0.02979,0.05322,0.06592,0.08643,0.07422,0.00705,0.01379,0.00452,-0.02832,0.00589,0.0498,0.05542,0.03198,0.04199,-0.01007,-0.02063,0.05884,0.04639,0.0332,0.0188,0.00169,0.03149,0.00818,0.0459,-0.05664,0.0047,-0.0177,-0.01508,0.03052,0.0564,-0.0015,0.01508,0.01831,0.00019,0.02576,0.00699,0.02295,0.0282,-0.04077,-0.00443,-0.01489,-0.02502,0.02112,-0.03418,-0.13672,-0.01276,0.02051,-0.00543,-0.00522,-0.04688,-0.00909,0.00525,-0.05225,0.03198,0.02148,-0.00368,0.03296,-0.09619,0.00305,0.01196,-0.0459,0.02942,0.02612,-0.00235,-0.01398,0.01978,0.05591,0.05396,0.01025,-0.04395,0.0083,0.03247,0.052,-0.02551,0.005,0.02661,-0.01398,0.02527,-0.0376,0.01501,0.026,0.00491,0.09277,0.00455,0.02771,0.05103,-0.04053,0.00592,-0.01263,-0.00909,0.02698,1e-05,-0.01599,0.06396,0.04761,-0.02576,-0.01245,0.00595,0.11426,-0.04004,0.02161,-0.00537,-0.01709,0.05713,-0.00121,0.01135,-0.04297,-0.01044,-0.02527,0.04907,0.07324,0.01929,0.00562,0.01929,0.021,0.03589,-0.02271,-0.00516,0.02551,0.03711,-0.00494,0.04883,-0.0155,0.04517,-0.00653,-0.02893,0.0354,0.03491,0.03638,-0.00934,0.03589,0.01105,-0.02527,-0.02417,-0.00806,-0.0376,-0.03345,-0.00873,0.04541,-0.03296,-0.04492,-0.06543,0.02563,-0.04004,0.00909,-0.04199,0.01459,-0.00311,-0.0177,-0.04248,0.03015,-0.05103,0.01385,-0.04639,-0.01416,0.02478,-0.0238,-0.02954,0.03174,-0.00244,-0.00644,-0.00256,0.03857,0.02283,-0.06006,-0.00885,-0.01349,0.01398,0.013,0.02197,-0.02612,-0.01166,0.00668,0.0304,0.01416,0.01074,0.05713,-0.00699,0.01202,-0.05713,0.03052,-0.02283,-0.01733,-0.02246,0.02161,-0.02136,-0.00552,-0.00436,0.00215,0.02673,0.03955,0.01575,0.00528,0.06201,0.0199,-0.05127,0.00182,0.01733,0.01978,0.05322,-0.0481,0.03662,-0.00769,-0.00824,0.00048,0.05273,-0.01379,-0.00861,-0.00249,0.00362,0.02234,0.01819,-0.00266,0.01013,0.05762,-0.0304,-0.05347,0.01367,0.04077,0.03149,0.06299,-0.01917,0.0069,0.02185,0.04199,0.0354,0.01965,-0.02478,-0.03418,-0.04834,0.00958,-0.00885,0.02124,0.03247,-0.0188,-0.02942,0.0354,0.0708,0.01794,0.00589,-0.01794,-0.00308,0.05518,-0.03015,0.00226,0.03516,-0.03613,-0.02393,0.04663,0.04443,-0.03223,-0.02881,-0.04712,0.05713,0.05469,0.0332,0.06348,-0.02173,-0.06226,0.01556,0.06885,0.02637,0.01428,-0.00071,0.05518,0.0708,0.0058,0.0282,-0.01459,0.01379,0.03784,-0.0083,-0.08252,-0.0354,-0.02209,0.06689,0.0603,-0.00674,-0.05103,0.00873,-0.00388,-0.14941,0.01385,0.00693,-0.00595,-0.01953,-0.00156,0.02429,-0.04614,0.02063,0.0152,0.02368,0.01709,0.02209,-0.03442,0.01953,-0.04199,0.06079,0.0065,0.00337,0.05127,0.0708,0.02405,-0.06348,0.01709,0.00568,-0.05078,0.04468,-0.01636,0.01282,0.02612,-0.0072,0.0177,0.02441,0.02283],[-0.03906,0.04712,-0.02917,-0.00665,0.02295,-0.06201,0.04785,-0.01129,0.05347,0.03174,0.01471,-0.01599,0.02112,-0.02673,-0.01453,-0.00665,0.03442,0.00647,-0.01086,0.00038,-0.0188,0.02795,-0.01459,0.01929,0.0481,0.00223,0.02454,-0.00714,-0.00243,0.00333,0.03125,0.00136,0.00903,0.02014,0.05127,-0.03955,-0.03735,0.02026,-0.05591,0.02148,0.02405,-0.0097,-0.01831,0.06641,-0.08643,0.06177,-0.01892,0.00473,0.01575,0.04614,0.00276,0.03345,0.0141,-0.02979,-0.02002,0.03638,0.0332,0.02466,-0.03345,0.00226,-0.07471,-0.04053,0.02197,-0.02356,0.04199,0.03564,0.07031,0.00278,0.02844,-0.03857,0.01337,-0.00629,0.04053,0.05957,-0.01831,0.0199,-0.04175,0.04639,-0.06396,-0.02026,-0.01562,0.00446,0.03491,-0.00952,-0.01086,0.06079,-0.00415,-0.03516,0.02405,0.00705,0.01868,-0.01367,0.008,0.04102,0.03931,-0.0332,-0.04199,0.05078,0.01575,-0.00745,0.02734,0.00191,0.0199,0.03101,0.0043,-0.00958,0.05713,-0.02185,0.02197,-0.00301,0.03516,0.00537,-0.02234,-0.00577,-0.04077,0.03369,-0.02759,-0.02588,0.03711,0.03687,0.01294,0.0199,0.04761,-0.04395,0.04492,0.01331,0.05054,0.0108,0.03638,-0.0332,-0.06641,-0.03979,-0.03809,-0.00446,-0.00616,-0.0069,-0.01056,0.04614,0.00928,0.02429,0.03369,-0.02429,-0.04468,0.0127,0.02551,-0.00595,0.07373,0.02783,0.04297,0.07617,0.02795,-0.00513,-0.01672,-0.0105,-0.08057,-0.0033,-0.00033,0.0481,-0.03662,0.04932,0.03906,-0.06885,0.05029,0.0105,-0.01135,0.06299,0.05273,0.02441,-0.01392,0.026,0.04858,0.06006,0.02734,-0.09521,0.03149,0.01294,0.02942,0.07031,-0.0199,-0.02112,0.01855,0.01532,-0.03589,0.09326,0.06299,0.09131,0.02881,-0.01025,0.02161,0.0097,0.0304,0.05176,-0.01172,-0.00751,-0.01483,0.00659,-0.04248,0.02466,-0.026,-0.05054,0.01648,-0.04199,0.01453,-0.03088,0.02722,0.01544,-0.0155,0.03149,-0.00296,0.04321,-0.00169,0.01398,-0.00668,0.0354,-0.01379,0.01746,0.07275,0.01721,0.06738,-0.0752,0.01257,0.03418,0.02161,0.01746,-0.03003,-0.02551,-0.0006,-0.01685,0.0013,-0.02905,0.02844,0.05444,-0.01196,-0.04736,0.07617,-0.01013,0.02515,0.01068,-0.02271,0.00507,-0.06885,-0.02527,-0.00482,-0.01917,0.04663,0.02441,-0.03064,-0.05444,0.03613,-0.05273,0.0332,0.02759,0.08594,0.0034,0.04297,-0.09863,0.00256,0.02014,0.03223,-0.00253,0.03662,0.03516,0.03247,-0.02222,-0.02478,-0.0072,-0.01831,0.01697,-0.0238,0.08398,-0.00415,-0.02734,-0.03125,0.00836,0.06982,0.0332,0.01172,0.00525,0.00577,0.00861,0.02539,0.00473,-0.03369,0.0354,0.00626,0.02856,0.06201,0.00549,0.05029,0.00726,0.05835,0.04297,-0.02356,-0.02905,-0.0007,-0.00537,-0.01141,-0.00632,0.02759,-0.00375,0.0282,0.04224,-0.03809,-0.05273,0.04883,0.04199,-0.01843,-0.00121,0.02942,0.01483,0.00952,0.06836,0.00293,0.04224,-0.00903,0.00699,-0.04053,-0.02466,-0.05054,0.00793,-0.00342,-0.02612,0.04224,-0.01868,-0.03088,-0.01892,-0.00089,0.05103,0.04224,-0.08398,-0.0083,0.05469,-0.02417,0.03979,-0.03027,-0.04517,-0.01965,0.05127,-0.07031,-0.00146,-0.01056,-0.00035,0.00989,0.02161,-0.0199,0.0459,-0.04077,0.05444,0.03125,0.0481,0.06445,0.03149,-0.01007,0.02795,-0.00455,-0.03955,0.02576,-0.02698,-0.05054,-0.03442,-0.00946,0.04102,-0.01929,0.08105,0.04932,0.05444,0.02515,-0.04004,-0.0238,0.0144,0.03662,0.05151,-0.01624,0.01599,0.04907,-0.03088,0.01758,0.01221,0.0835,-0.0437,0.03149,0.00983,0.01917,-0.02979,-0.02722,0.01672,0.02783,-0.02051,0.0238,-0.0564,0.01306,0.06079,0.05566,0.0481,0.05273,0.05762,-0.00705,-0.00072,0.00275,-0.00308,0.02051,0.04077,0.07861,0.02698,-0.01562,-0.06055,0.04932,-0.03064,-0.0437,0.07617,0.02942,0.03882,-0.00415,-0.02856,-0.05713,0.03516,-0.02148,-0.0009,0.01428,0.0029,0.05127,0.02258,-0.01575,-0.02161,-0.00604,0.02539,0.01459,-0.00111,0.01471,0.03931,-0.02722,0.04297,0.0459,0.01361,0.02295,0.04932,-0.00632,0.0437,-0.00284,0.01361,0.06885,0.00751,0.00766,0.01172,0.05396,-0.02002,-0.03345,-0.02039,0.02271,-0.00446,-0.00647,-0.08594,0.01483,0.01367,0.06787,0.00604,0.07227,-0.01331,0.01398,-0.05713,-0.01483,0.026,-0.00246,0.03491,-0.01904,-0.002,-0.00182,-0.0166,0.06885,-0.00861,0.0152,0.021,0.02734,-0.01331,-0.05957,0.06299,0.01599,0.00699,-0.04541,-0.03174,-0.06445,0.02332,-0.01978,-0.01294,0.05444,0.01367,0.01062,0.0001,0.02197,0.01965,0.02441,-0.05054,0.04468,-0.01331,-0.02759,0.03491,-0.01636,-0.08887,-0.03442,-0.00375,0.04761,-0.00604,0.0105,-0.01697,0.02466,-0.01068,0.03223,0.03088,0.00147,-0.03101,-0.04248,-0.02637,0.05396,0.01758,0.02551,0.00074,-0.01379,-0.04199,0.01733,0.04639,0.02612,-0.00028,0.00635,-0.04321,0.00014,0.03088,0.06543,0.0437,-0.02271,-0.03662,0.04102,0.0076,0.06592,0.00172,-0.00873,0.11035,0.01038,0.02539,0.05054,0.02795,0.008,-0.02405,0.00372,0.02966,-0.04688,-0.06396,0.00488,0.06006,-0.01855,0.00313,-0.00299,0.03613,-0.03687,-0.01544,-0.00977,0.03003,0.01263,0.03271,0.03955,0.01831,0.00494,-0.02515,0.04346,0.02942,0.01807,0.05566,-0.04883,0.01007,0.03027,-0.04126,0.07275,0.01843,-0.03442,0.06885,-0.00989,-0.04175,0.04736,0.02612,-4e-05,-0.0119,0.06006,0.01184,0.04175,0.00555,0.02722,0.00155,-0.08398,0.01379,0.00046,-0.01904,0.12109,0.03247,-0.06055,0.00745,-0.01007,0.07617,-0.03223,-0.04468,-0.0332,0.03271,-0.00087,0.03882,-0.02637,0.05957,0.008,-0.00638,-0.01392,-0.05566,-0.03125,-0.01685,-0.03003,0.00357,0.0249,0.03394,0.02942,-0.00854,0.05005,0.03198,0.03442,0.0177,0.06128,-0.02173,0.09277,-0.0415,-0.0625,-0.04858,-0.01782,0.01611,0.01245,-0.03711,0.03442,-0.01794,-0.01782,0.01501,-0.00879,0.04443,-0.00013,0.0332,-0.07764,0.03149,0.02002,0.00684,0.03833,0.01233,0.05396,0.02405,0.04956,0.01965,0.00772,0.05811,-0.02466,-0.04443,0.04956,-0.02332,-0.05566,0.00012,-0.02039,-0.01501,0.04443,-0.0249,0.05566,0.01965,0.05957,-0.0282,-0.00836,0.00653,0.0304,0.05591,-0.0144,0.0001,0.03687,0.01471,0.03564,0.00552,0.02356,0.04492,0.03394,-0.00043,-0.01501,0.05396,0.0304,0.04321,0.01166,0.01483,0.05566,0.02185,0.02014,0.0199,-0.01746,0.0166,0.04419,0.00221,0.02405,-0.00696,0.04419,-0.01965,-0.02161,0.01471,-0.00238,0.00218,0.03589,0.10791,0.05469,-0.06177,0.04565,-0.02283,-0.01001,0.02026,0.06641,0.04443,-0.0304,-0.01807,0.03687,0.02588,0.02637,0.0791,0.05835,0.02234,0.04419,-0.00995,0.0007,-0.01483,-0.04053,-0.00427,0.02917,-0.04077,-0.02673,-0.02966,-0.02917,0.06592,0.04639,-0.02405,0.04468,0.00818,-0.03101,0.05811,0.02271,0.02222,0.01398,-0.01062,0.0054,-0.0127,0.03198,-0.05078,-0.01208,0.06641,0.01599,0.00714,0.00476,0.01031,0.10791,-0.02222,0.01721,0.03931,-0.00266,0.00516,-0.06201,0.0625,-0.01294,-0.01782,0.04736,-0.00127,0.06348,0.02856,0.0332,0.03882,0.02124,0.00854]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."],"revision":"a9cb5338491faf32b73dd17b714a31821c021bbf"},"intfloat/multilingual-e5-large":{"embeddings":[[0.00615,0.00167,-0.01884,-0.03221,0.02153,-0.02805,-0.03668,0.03462,0.05085,-0.03967,0.02994,0.02673,-0.01631,-0.03023,-0.00784,-0.02785,-0.01527,0.01416,-0.0175,-0.0024,0.01564,-0.01382,-0.03142,-0.05291,-0.01336,-0.00906,-0.05366,-0.03229,0.00018,-0.02371,0.02445,0.0198,-0.057,-0.02765,-0.01388,0.04315,0.01792,0.04386,-0.04145,0.02259,-0.04027,0.06549,-0.00615,-0.04685,-0.02545,0.01882,-7e-05,0.00995,-0.00658,0.0094,0.01294,0.03544,-0.01205,-0.00679,-0.05014,0.02546,-0.03369,-0.00695,-0.06792,0.02104,-0.00758,-0.01068,0.02602,-0.00884,-0.05043,0.04765,0.04374,0.01041,-0.0627,0.01121,-0.01663,0.02877,-0.00805,-0.01111,-0.0205,-0.00844,0.01198,-0.00926,0.02134,-0.04331,0.06926,-0.00373,0.02077,-0.00268,0.04092,0.05955,0.05645,0.03653,0.04056,-0.0068,-0.02024,0.04116,0.04302,-0.03086,-0.06146,-0.0102,0.02745,0.01578,-0.0087,0.02171,-0.02511,0.01118,0.04607,-0.04068,-0.01473,0.02825,0.04167,0.03878,0.01572,0.01375,0.02937,0.06091,0.01171,-0.03897,-0.02686,-0.0017,-0.01511,-0.0206,-0.01555,0.02105,0.03059,0.05445,-0.00398,-0.02821,0.03217,-0.0198,0.00498,0.01906,-0.03535,-0.00161,0.01966,0.0339,-0.04975,-0.00636,-0.03221,-0.01732,-0.03673,0.01371,-0.0158,0.00554,0.00017,0.03149,0.03001,-0.03474,-0.00475,-0.03925,-0.06055,0.02426,0.0156,-0.02372,0.01525,-0.03527,-0.01349,0.01126,0.04015,-0.02439,-0.00961,0.02856,0.01881,-0.0098,-0.03581,-0.04898,-0.02589,-0.02458,0.00259,-0.01206,0.02079,0.03067,0.00878,0.01893,-0.0057,-0.01867,-0.00036,-0.06085,0.0105,0.03999,0.03616,0.01615,0.04167,-0.02559,0.00959,0.04495,0.03505,-0.04797,-0.0236,0.04523,0.01781,-0.00457,0.04794,0.03589,0.04838,-0.03042,-0.01687,0.00771,-0.02514,0.02846,0.01811,0.02622,-0.04418,-0.04573,-0.00331,0.01715,-0.04608,0.03864,-0.00672,-0.018,-0.03178,-0.03277,0.04316,-0.021,-0.02684,0.00573,0.01766,0.01733,-0.03753,-0.04431,-0.0218,-0.04106,0.02515,-0.02909,0.02671,0.04492,0.00073,0.03602,0.02872,0.02141,0.04402,0.02366,0.07388,0.04,0.01923,0.01851,-0.02748,-0.01402,0.00598,-0.02162,-0.02918,0.00522,0.03035,0.06531,-0.03152,0.02151,0.00102,-0.0669,-0.00033,-0.01244,0.03099,-0.02126,0.06486,-0.00309,0.01367,0.00166,-0.03883,-0.0096,0.02686,-0.01297,0.03665,-0.02066,-0.00011,-0.01652,0.01446,0.01598,-0.00921,0.05124,0.03684,0.02166,-0.00765,-0.04292,-0.0186,-0.03449,-0.02408,-0.04944,-0.06127,-0.01436,-0.02312,-0.00964,-0.04073,-0.02219,-0.01258,0.01835,-0.01879,-0.02844,0.01862,-0.03649,-0.022,-0.00534,-0.01025,-0.01194,0.04166,0.0023,0.01103,-0.01967,0.09679,0.00777,0.03473,0.03102,-0.05535,-0.08699,-0.03839,0.04657,-0.03918,-0.04668,0.02044,0.02931,-0.03045,-0.05196,0.01273,0.03193,-0.06546,-0.0224,0.0303,0.04325,0.01745,-0.0319,-0.05916,-0.00678,-0.0362,-0.01359,-0.03348,0.04909,-0.04637,-0.00442,-0.03869,0.00256,-0.0091,0.0323,0.00699,-0.03583,-0.00141,-0.03772,0.0364,-0.03694,-0.00036,0.05587,-0.03022,0.01914,-0.04939,0.03388,0.02687,-0.00381,0.02428,-0.00716,-0.05017,0.01245,0.01753,-0.01649,0.01085,-0.03297,0.02048,0.01497,0.00411,-0.03754,0.04071,0.00336,-0.02853,0.06207,-0.00445,-0.01898,0.02212,-0.01418,0.00869,0.02196,0.00822,-0.02099,-0.04342,-0.00806,-0.01612,0.02104,-0.04276,-0.04005,0.00753,-0.00583,-0.01166,0.00018,-0.00525,-0.01323,-0.08784,-0.01248,0.0087,-0.06877,-0.04399,-0.03193,0.01033,-0.03153,0.0083,-0.03041,0.15939,0.00369,0.01998,-0.05045,-0.01189,0.04456,0.0318,0.02134,0.04552,0.01934,-0.03649,-0.00998,0.02344,-0.0149,0.04619,0.05614,0.00407,0.03745,0.0283,0.00314,0.02897,-0.01966,0.05961,0.0096,-0.05195,-0.03497,0.00896,0.02183,-0.04654,0.0342,-0.01183,0.0266,-0.02103,-0.00451,0.02802,0.01168,0.02302,-0.06055,0.03326,0.0066,-0.03371,-0.00198,-0.03486,0.06223,-0.00114,0.03318,0.035,-0.01025,-0.033,0.03541,0.00551,0.01503,-0.02654,-0.03598,-0.02727,-0.02694,-0.0154,-0.06813,0.0187,0.01968,0.06567,0.02885,0.01509,-0.04656,0.02185,0.01838,0.00224,-0.05228,-0.03437,0.00359,-0.03251,-0.00873,-0.03066,0.02017,0.0496,-0.00101,-0.02414,0.04115,0.03096,0.0125,0.01225,-0.01952,-0.00629,-0.0021,0.05267,-0.03803,0.00979,0.00656,-0.02996,0.01079,0.02984,0.02911,-0.0254,0.03443,-0.00182,-0.0528,-0.00454,-0.01863,-0.02927,-0.01361,-0.02177,0.02903,-0.04105,-0.01436,-0.02122,-0.01872,0.00244,0.03449,0.02727,0.00095,0.02517,5e-05,0.04947,-0.00208,0.03659,0.0357,0.01634,-0.04529,0.0405,0.01845,-0.01805,0.02499,-0.02753,0.06908,-0.01798,0.0303,-0.02546,0.00353,0.04697,0.03999,0.02565,-0.03321,-0.02968,0.03114,0.0114,-0.02264,0.05427,-0.02309,-0.01277,0.04318,-0.01662,0.0375,-0.03253,0.03411,0.01434,0.02483,-0.04909,-0.04621,-0.02879,-0.03238,0.0281,-0.04326,0.02168,0.00135,-0.01895,-0.02044,-0.00199,0.00882,0.0467,-0.038,-0.01491,0.00713,0.02949,0.08934,0.03923,0.03082,-0.0556,0.03442,0.04656,0.01178,-0.01369,-0.0359,-0.00317,0.0165,0.03202,-0.02372,-0.02483,0.04296,-0.05164,0.0388,-0.00114,0.03858,0.01901,-0.05343,-0.02259,0.00472,0.01412,-0.03766,0.00163,0.0276,-0.01545,0.06877,-0.0334,-0.00281,-0.03326,-0.00625,0.03139,0.0168,-0.01824,-0.03339,0.00804,-0.03694,-0.00283,-0.0191,-0.03522,-0.01633,-0.00378,0.03659,0.00572,0.06713,0.01236,-0.06875,0.00218,0.03009,0.031,0.01589,0.03229,0.03717,0.04021,-0.02505,0.02784,0.02861,0.02398,-0.03976,-0.04332,-0.03079,0.07925,0.01692,-0.01335,-0.02087,0.0103,-0.02611,-0.00049,-0.0713,-0.02071,-0.01673,0.04088,-0.0074,0.01223,-0.02923,0.04267,-0.00196,0.02846,-0.01718,-0.0108,0.0252,-0.03355,-0.01167,0.02645,0.01847,-0.00764,-0.02294,0.03671,0.04711,-0.04833,-0.00658,0.01129,-0.01064,-0.04095,-0.05825,0.02147,0.02263,-0.03088,0.00999,-0.02612,-0.0002,0.02627,-0.05376,-0.00558,-0.04673,0.0439,-0.027,-0.00552,-0.05778,0.03202,0.04894,-0.02891,-0.03764,-0.02056,-0.02949,-0.03207,0.01705,0.00691,0.02757,0.01405,0.03422,-0.04516,0.05298,-0.0214,-0.03076,-0.02831,0.0001,-0.03295,-0.00082,-0.02642,-0.04646,-0.02201,-0.01685,-0.01091,-0.04717,0.02315,-0.01265,-0.04558,-0.06804,-0.01355,0.03169,-0.03092,-0.00136,0.03555,0.02648,-0.00366,-0.03205,0.0166,0.02467,0.03169,0.00783,0.03284,-0.02549,-0.00702,0.04325,-0.04913,-0.00615,-0.01033,-0.01693,0.06183,-0.00409,0.02603,0.00841,0.01077,0.00269,0.01264,0.00167,-0.02049,0.00447,-0.05125,-0.05859,-0.02217,-0.0244,-0.03446,-0.04205,-0.00539,0.03457,0.02077,-0.02666,0.03425,0.02147,-0.02431,0.01696,-0.02137,-0.01403,-0.0261,0.02934,-0.01227,-0.01623,-0.00687,0.03435,-0.03354,0.00529,0.01695,-0.04586,-0.03223,-0.02635,0.01901,-0.01786,-0.00221,0.03267,-0.01975,0.02599,-0.00909,-0.00334,-0.01975,0.04265,-0.0342,0.00266,-0.02881,0.02719,-0.01439,0.01482,0.05184,0.01778,0.03,-0.07998,-0.05054,-0.04743,0.05319,-0.02988,-0.02204,0.02126,-0.00235,0.03775,-0.04423,0.04813,-0.01801,0.03014,-0.08231,-0.01351,-0.00466,-0.04255,-0.00617,-0.00554,-0.02872,0.01829,-0.00609,-0.0465,0.01261,0.03835,0.02638,-0.01093,-0.00171,0.03271,0.03421,-0.01843,0.00153,0.00913,0.02476,-0.03444,0.00221,0.01112,0.03416,0.03274,-0.00648,0.02359,-0.03598,-0.05186,-0.01576,-0.04442,0.0109,0.03642,-0.03093,0.04372,-0.00046,0.01294,-0.00861,-0.00414,-0.0453,0.0214,-0.05431,-0.00383,-0.0212,0.0344,-0.0017,-0.03341,-0.00681,-0.03029,-0.02644,-0.03305,0.02823,0.05523,0.02451,0.03289,-0.00016,-0.04674,0.0203,0.01865,0.00989,-0.03898,-0.01892,-0.03093,-0.03522,-0.01432,-0.0524,-0.03018,-0.02161,0.02863,0.03719,-0.02986,0.02648,-0.00214,0.04029,-0.01852,-0.00938,-0.00501,0.03258,-0.00685,-0.01942,-0.0135,0.01719,0.02572,-0.00758,0.0167,0.00859,-0.01627,-0.05348,0.0355,-0.03085,-0.00022,0.03865,-0.05134,-0.03863,-0.01656,-0.02716,0.00337,-0.02018,0.04672,-0.03461,-0.00563,0.01592,-0.00138,0.02164,0.01698,0.0256,0.02619,0.02859,-0.01706,-0.03702,-0.02659,-0.01661,-0.0227,-0.00344,0.02523,-0.02006,0.0081,-0.01022,0.00937,-0.02498,-0.07168,0.03724,-0.0122,0.04004,-0.00867,0.00592,-0.04052,0.00315,-0.0341,0.01146,-0.07161,0.04147,0.01907,-0.03241,-0.01396,0.02179,0.06182,-0.04631,0.04859,-0.00359,-0.04783,-0.02793,-0.0342,-0.03299,0.03834,-0.02407,-0.02525,-0.00041,-0.0372,0.03359,0.00067,-0.03844,0.0479,0.01996,-0.00293,0.01322,-0.00658,0.02476,-0.00072,-0.00677,0.01804,-0.0576,0.03539,-0.01186,-0.01977,0.01781,0.00802,-0.03949,-0.01356,0.02769,0.03701,-0.04249,0.01506,0.0521,-0.01934,0.05178,0.03863,0.02595,0.03125,0.02879,0.01489,-0.00325,0.0311,0.0201,0.05376,0.0145,0.01407,-0.03218,0.01808,-0.02966,0.03742,-0.04484,0.04111,0.06778,-0.02128,-0.02059,0.00531,-0.02297,-0.00817,0.00135,-0.00885,0.01699,-0.03618,-0.00299,-0.02229,-0.05462,0.00601,-0.02324,0.03343,0.04094,-0.00466,0.01443,0.02148,0.03247,0.02243,0.03655,-0.0471,-0.01349,0.04415,0.0376,0.03211,-0.00811,0.01018,-0.04028,0.0433,-0.05766,0.02474,-0.04351,-0.02233,0.00075,0.01573,0.00695,0.01393,-0.01471,-0.05068,0.04218,0.02377,0.02173,0.01943,-0.02215,0.00109,-0.02419,0.02293],[0.03846,-0.0082,-0.00921,-0.02049,0.03584,-0.00922,0.00239,0.05344,0.05032,-0.03811,0.0011,0.03783,-0.06407,-0.04176,0.01019,-0.00751,-0.00255,-0.00436,0.00156,-0.03799,0.02109,-0.00684,0.01454,-0.01424,-0.03091,-0.04024,0.02576,-0.01973,-0.01787,-0.03813,-0.01958,-0.00714,-0.03606,-0.04513,-0.00805,0.02994,0.03516,-0.01577,-0.03248,0.01195,-0.02364,0.02526,-0.00402,-0.01722,0.00034,0.03871,0.04856,0.00153,-0.03923,0.00818,-0.0246,-0.00379,-0.00817,-0.0081,-0.01533,-0.00799,-0.00553,-0.0174,-0.028,-0.01267,-0.03008,-0.02038,-0.00394,-0.0224,-0.05102,0.03389,0.03188,-0.0121,-0.0522,0.03167,0.0167,0.02915,-0.0119,-0.03779,0.0002,-0.02102,0.02669,-0.01683,0.04762,0.01064,0.07822,-0.00896,-0.00347,-0.02252,0.03228,0.04546,0.01981,0.00847,0.04402,0.01891,-0.00772,0.04674,0.03865,-0.00523,-0.03102,0.00949,0.02012,0.02886,0.01917,-0.00809,-0.00408,0.02538,0.01319,-0.04884,-0.03445,0.06332,0.01959,0.03309,0.01632,0.02214,0.06101,0.03052,-0.01289,-0.01785,-0.00822,-0.01567,-0.0133,-0.01967,-0.00718,0.01587,0.05309,0.0174,0.01217,-0.02263,0.03331,-0.00693,0.01562,0.03049,0.0047,0.06342,0.03621,0.02228,-0.01601,-0.05516,-0.07028,-0.05228,0.00454,0.01021,-0.01607,0.03895,0.01881,0.01285,0.02272,-0.03872,-0.0324,-0.07229,-0.03093,0.02382,-0.01966,-0.03185,0.01974,-0.02843,-0.04072,0.01126,0.02838,-0.02311,0.02693,0.0554,0.01074,0.0093,-0.04094,-0.02215,-0.00925,-0.02206,-0.01555,-0.03603,0.03611,0.03322,0.01989,-0.03424,-0.03098,-0.095,0.0094,-0.04766,0.03406,0.01623,0.04803,0.01081,0.02131,-0.01541,0.02439,0.02984,0.00327,-0.02181,-0.01283,-0.00599,0.06374,0.03159,0.02941,0.00732,-0.00017,-0.03511,-0.00779,-0.00279,-0.00334,0.01702,0.01239,-0.00758,-0.00403,-0.05993,0.00675,0.02629,-0.02605,-0.01759,0.00797,-0.01698,-0.03316,-0.06832,0.00377,-0.04204,-0.03728,0.00784,0.06657,0.01635,-0.03653,-0.03448,-0.0042,-0.00504,0.04644,0.0107,0.02565,0.05564,0.00885,0.02644,0.05212,0.00242,0.03536,0.0192,0.02383,0.01901,-0.01006,0.02705,-0.04279,-0.00606,0.01002,-0.00333,-0.05587,-0.00189,0.01561,0.0552,-0.00953,0.04897,-0.01361,-0.00384,0.04547,-0.02567,0.02247,-0.00893,0.02218,-0.01858,0.00663,0.03462,-0.03407,-0.02493,-0.01628,0.02987,0.04452,0.02077,-0.00532,-0.02901,0.00884,0.05931,-0.01644,0.02545,0.00471,0.03595,0.01567,-0.02338,-0.00839,-0.05281,-0.00585,-0.00971,-0.04897,-0.01035,-0.0088,0.03115,-0.02032,0.01065,0.00115,-0.00542,-0.01686,-0.02556,0.01987,-0.01151,-0.01397,0.00202,0.01282,-0.01807,0.00456,0.00372,0.01989,-0.06425,0.05562,-0.0227,0.03642,0.01535,-0.03691,-0.06583,-0.02022,0.02946,-0.05371,-0.02705,0.00887,0.03171,-0.06397,-0.01168,-0.02988,0.00561,-0.06922,-0.01226,0.01551,0.06984,0.05458,-0.02257,-0.04874,-0.02622,-0.01035,-0.02683,0.00419,0.0367,-0.03065,-0.0451,-0.04322,0.00292,0.00097,0.01853,0.05196,-0.0196,0.01295,-0.01488,0.01254,-0.01084,0.00515,0.00257,-0.04237,0.01804,-0.03397,0.03957,0.02916,-0.00373,0.02016,-0.03168,-0.04602,-0.03368,0.05328,-0.05221,0.02208,-0.01633,0.044,0.05692,0.02358,-0.01209,0.05275,0.0004,-0.04706,0.05451,0.03488,0.01114,0.01685,0.02183,0.0415,0.00107,0.02898,-0.04218,-0.04362,-0.01915,0.01537,-0.02492,-0.05637,-0.04821,0.02953,0.00898,-0.01376,0.00898,-0.00765,-0.06303,-0.09321,0.01205,-0.00143,-0.04368,-0.03241,-0.04265,0.00778,-0.03445,0.01385,-0.00417,0.17389,0.02144,0.01811,-0.04282,-0.02938,0.02029,0.05084,0.03387,0.02313,0.02912,-0.02514,0.00368,0.02531,-0.03619,0.03223,0.01939,0.02452,0.04334,0.04004,-0.03382,0.0725,-0.00347,0.05657,0.01201,-0.05239,-0.05421,0.06037,0.06205,-0.02382,0.01883,-0.006,0.00015,-0.0547,0.02826,0.01621,-0.00555,0.00126,-0.00604,0.04162,-0.0451,-0.0413,0.00824,-0.06537,0.01002,-0.00597,0.03765,0.02738,-0.02459,-0.01068,-0.00148,0.02199,0.07132,-0.02875,-0.01683,0.00299,-0.00196,-0.04945,-0.04756,-0.00539,0.00111,-0.00861,0.02563,-0.02761,-0.0386,0.02013,0.04481,-0.02944,-0.02613,-0.03505,-0.03158,-0.01601,-0.00712,-7e-05,0.00155,0.02369,-0.0324,-0.01394,0.03892,0.04372,-0.00193,0.04418,-0.03928,-0.04163,-0.00398,0.06629,0.0264,0.05103,0.00748,-0.0492,0.02804,0.03052,-0.00105,-0.03316,0.02228,0.00037,-0.01572,0.0066,0.01587,-0.01735,-0.00703,-0.01271,0.01407,-0.00865,-0.0341,0.01685,-0.03788,-0.00733,0.04954,0.01142,-0.01359,0.04169,-0.01214,0.0261,-0.04434,0.02192,0.03339,0.03089,-0.06181,0.02282,0.01409,-0.00954,0.04259,-0.05463,0.02098,-0.01203,0.01701,-0.00795,0.00068,0.00697,-0.00506,0.00558,-0.0235,-0.04607,0.02162,0.02121,-0.01804,0.06836,-0.02559,-0.0312,0.0399,-0.00942,0.04542,-0.03989,0.04967,0.00898,0.03946,-0.02289,-0.04089,0.0011,-0.01858,0.00709,-0.02093,0.01581,0.00608,-0.0314,0.03741,-0.01064,0.005,0.00783,-0.0476,-0.04019,-0.02989,-0.00365,0.08605,0.04807,0.03505,-0.03834,-0.01646,0.03454,0.03367,-0.02465,-0.02021,0.02289,0.03244,-0.01561,-0.00029,-0.02201,0.03637,-0.0198,0.02235,-0.02434,0.05452,0.02209,-0.04978,-0.02817,0.02683,0.00599,-0.04426,-0.04146,-6e-05,-0.04405,0.08753,-0.01529,-0.00884,-0.04071,0.00535,0.01481,0.02575,-0.0072,-0.0473,0.008,-0.02536,-0.04046,-0.00286,-0.0297,-0.02313,0.01337,0.02312,0.02692,0.00824,-0.02022,-0.04903,-0.01026,0.00556,0.03029,0.05101,0.01232,0.00249,0.03748,-0.03579,0.04941,-0.0029,0.04438,-0.01277,-0.00476,0.03104,0.01169,0.03661,-0.02662,-0.04389,-0.00172,-0.01408,0.00323,-0.02027,-0.03216,-0.02045,0.02354,-0.00234,0.02916,-0.01754,0.00328,0.00314,0.02663,0.00437,0.00045,0.00535,-0.01126,-0.012,0.00719,0.04475,-0.02448,0.01864,0.02765,0.03468,-0.01558,-0.03756,0.03619,-0.00548,-0.04546,-0.03343,-0.00926,-0.00686,-0.03879,0.05985,-0.06223,0.0305,0.02729,-0.03702,-0.01569,-0.02114,-0.00262,-0.01627,-0.03333,-0.04311,0.07162,0.04769,0.00949,-0.0426,-0.01389,-0.00998,-0.04531,0.03624,-0.00778,-0.02647,0.02361,0.02894,-0.03759,0.03943,-0.03981,-0.03441,-0.03085,0.02417,-0.01954,0.00281,-0.04189,-0.0699,0.00481,-0.04181,-0.0118,-0.01496,0.0203,-0.04305,-0.02346,-0.0034,-0.02306,0.00302,0.0012,0.00477,0.04665,0.01654,0.0164,-0.01789,0.03566,0.00419,0.01069,-0.0121,0.00968,-0.02772,-0.01241,0.0375,-0.02738,0.00448,-0.00186,-0.00761,-0.03128,0.02369,0.03182,0.05314,0.01423,-0.0056,0.03644,0.00363,0.00432,0.0198,-0.02121,-0.034,-0.01198,-0.0029,-0.01168,-0.05005,-0.01002,0.05525,0.05908,-0.04571,0.03257,-0.02228,-0.04558,0.00035,0.01208,-0.03366,-0.02763,0.03849,-0.01452,-0.01949,-0.04626,0.00885,0.00317,-0.01688,0.01821,-0.05876,-0.0209,-0.04364,0.03995,-0.0058,-0.0252,0.02031,-0.02688,0.02564,-0.0177,0.04474,-0.03797,0.00526,-0.02617,-0.02875,-0.0162,0.01981,-0.04636,-0.01443,0.00673,0.03683,-0.03811,-0.06851,-0.04211,-0.01595,0.02201,-0.01535,-0.02497,0.00274,0.00713,0.04303,-0.0389,0.03208,-0.00082,-0.00909,-0.05695,0.02611,0.02654,0.01378,-0.01442,-0.00709,-0.07004,0.02546,0.04437,-0.02081,0.02367,-0.01867,0.02155,-0.00377,-0.02557,0.04564,0.05545,-0.00715,0.01833,-0.00652,0.00443,-0.01437,-0.03383,0.01849,0.00272,0.03644,0.00863,0.01915,-0.03701,-0.07782,-0.01782,-0.04097,0.03454,0.00914,-0.05547,0.06169,-0.04774,-0.00566,-0.00554,-0.04663,-0.00346,0.03982,-0.01269,0.01321,-0.0036,0.0255,-0.00477,0.0054,0.03494,-0.04744,-0.04603,-0.03342,0.02808,0.06497,0.03598,0.04581,0.02919,-0.04386,0.02736,0.09067,0.00201,-0.05136,-0.04285,-0.02123,-0.02951,-0.03727,-0.03569,-0.00046,-0.01406,0.05342,-0.00236,-0.0192,-0.00725,0.00241,0.0394,-0.02234,-0.01723,0.01885,0.00991,-0.02232,-0.03134,-0.01029,0.04766,-0.01141,-0.02619,0.04905,-0.01742,-0.0156,-0.06988,0.02335,-0.03614,0.01286,0.0327,-0.02571,-0.06076,-0.01087,-0.02113,0.01315,-0.03591,0.02044,-0.01217,-0.03432,-0.00891,-0.00375,0.01765,0.00592,0.02714,-0.01618,-0.00579,0.00692,-0.02262,-0.04021,0.00358,0.01012,-0.03977,0.01806,0.00885,0.0413,-0.04,-0.02114,-0.00548,-0.0637,0.03599,-0.01827,0.05482,-0.01294,0.02388,-0.01964,-0.01468,-0.0318,0.0453,-0.07325,0.00374,0.04224,-0.01943,-0.01983,-0.00196,0.05563,-0.00797,0.01338,-0.00046,-0.03666,-0.00474,-0.0165,-0.00462,-0.03112,0.00709,-0.06224,-0.03842,-0.04052,0.02456,-0.034,-0.0335,0.03863,0.02719,0.00488,-0.00227,-0.01322,0.02791,-0.01461,0.00433,0.04981,-0.04388,0.03438,-0.05131,-0.03741,-0.01007,0.01641,-0.02147,-0.02365,-0.00543,0.04933,-0.01421,0.00839,0.01756,0.00882,0.03674,-0.01625,0.05141,0.03255,0.02371,0.00419,-0.04884,0.0134,0.02299,0.03823,0.02117,0.01411,-0.04217,0.04015,0.00781,0.0088,-0.03898,0.03052,0.01402,-0.02349,-0.01019,0.05048,-0.03606,-0.00628,0.02164,-0.02057,0.04572,-0.02141,0.00474,-0.02469,-0.0389,0.00954,-0.03281,0.04934,0.01655,-0.03643,0.01387,0.04636,-0.00695,0.00747,0.02263,-0.03546,0.00307,0.0339,0.02223,0.03965,-0.0283,-0.01036,-0.00708,-0.01042,-0.04451,0.0087,-0.01809,-0.04373,0.021,0.01272,0.01337,0.00774,0.00897,0.02052,0.0461,-0.00381,0.03194,-0.00241,0.03502,-0.02532,-0.04624,0.00254]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."],"revision":"3d7cfbdacd47fdda877c5cd8a79fbcc4f2a574f3"},"sentence-transformers/all-roberta-large-v1":{"embeddings":[[-0.00491,0.0044,-0.01892,-0.00247,-0.07814,-0.06722,0.00618,0.04673,-0.02171,0.02503,-0.00043,0.0242,0.01465,0.03645,-0.03645,-0.0303,0.0148,-0.02533,-0.05983,-0.05134,0.01746,0.07743,0.04035,0.06142,-0.03255,-0.02727,-0.01221,0.00277,-0.00847,-0.02716,0.01255,0.00519,0.02993,-0.01681,-0.00143,-0.03417,-0.0274,-0.00213,-0.01146,-0.00556,0.01481,0.0074,-0.03304,0.03185,-0.01974,0.00986,-0.01792,0.04463,0.05752,-0.03997,0.07135,-0.00506,-0.02994,0.00729,-0.02176,-0.05994,0.0436,-0.06935,0.05496,0.06513,0.00618,0.03998,0.00477,0.00872,-0.01157,0.02082,0.00239,0.01787,-0.02335,0.00173,0.02505,-0.0253,-0.0048,-0.05145,-0.01483,-0.02045,0.02061,0.03904,-0.0242,0.03605,0.00746,0.05673,-0.02943,-0.03818,0.0057,-0.00487,-0.01773,0.03736,0.04141,-0.01267,0.01949,-0.09329,-0.0586,-0.01511,0.01865,0.00241,0.00344,-0.02159,-0.02566,0.03557,0.004,0.04771,0.03518,0.02876,-0.01231,-0.02624,0.00157,0.02732,-0.02482,0.00629,-0.03364,-0.02953,-0.01454,0.00136,-0.02454,0.02143,-0.02548,0.00873,0.00365,0.04238,0.003,0.03508,-0.00857,-0.01997,-0.00211,-0.03049,-0.0091,0.04757,0.00166,0.0182,0.0371,0.01116,0.02716,-0.02996,0.03532,-0.00068,0.01254,0.04435,0.01503,-0.01619,-0.0509,-0.04084,-0.02074,0.01607,0.01155,0.05921,0.02744,-0.01229,-0.00266,0.00486,-0.00701,0.01625,-0.04082,-0.00256,-0.00917,0.0095,0.00071,0.0086,0.01242,0.00685,-0.00454,-0.0298,-0.00142,-0.04007,-0.04089,-0.02946,-0.03479,0.01798,0.06451,0.04752,0.01171,0.02314,0.01749,-0.00481,0.01423,0.01982,0.0163,-0.01267,0.01271,0.03126,0.03253,-0.03753,-0.00359,-0.0126,-0.02904,0.00883,0.0046,0.01535,-0.01568,-0.00207,-0.03463,-0.02127,0.03125,-0.02658,0.00492,-0.04955,-0.00759,0.0269,-0.08531,0.01391,-0.01611,0.04433,0.01175,-0.00047,0.03434,0.00643,0.01327,-0.03346,0.01086,0.00132,0.00855,0.04931,-0.03122,-0.06766,-0.00312,0.01679,0.02908,0.00318,0.00156,-0.05245,-0.01896,0.01812,0.00441,0.00719,0.0047,-0.00169,0.01055,0.03316,-0.02826,-0.06428,0.00171,0.06791,-0.01168,0.02612,-0.03324,0.03362,-0.03644,0.02252,0.01491,0.04821,-0.02506,-0.04503,-0.04295,0.0499,0.0426,-0.05623,-0.0477,0.02001,-0.03126,-0.02243,0.00375,-0.0028,-0.03933,0.03446,0.00091,0.01792,-0.0067,0.00154,0.00762,0.02367,0.05458,-0.0094,-0.00325,0.03178,0.02171,0.04649,-0.01136,-0.00953,0.03607,-0.02213,-0.00294,0.01412,-0.00882,-0.01808,0.01598,-0.02923,0.02937,-0.04259,-0.05132,0.03444,-0.02586,-0.01684,-0.01645,-0.00298,-0.05119,-0.01414,-0.01932,0.05487,0.00152,-0.01417,0.03513,-0.03364,-0.04596,-0.01979,-0.00108,0.01705,0.03152,-0.01226,0.11015,-0.02259,-0.00777,-0.01998,-0.02853,0.02265,0.00672,0.05379,-0.03633,-0.04947,-0.04784,-0.06021,0.00587,-0.04636,0.00057,0.05334,0.01505,0.04036,-0.04575,-0.01661,-0.02811,0.02913,0.05085,-0.02248,0.02317,0.02474,-0.01465,0.03192,-0.05312,-0.00058,-0.01229,0.00217,0.02721,0.03574,0.03714,-0.01278,-0.01241,0.02278,-0.04196,-0.04708,-0.04374,-0.01287,-0.00865,-0.01168,0.0415,-0.04438,0.0311,0.02402,0.00413,-0.02244,-0.01726,0.06536,0.00535,0.01371,-0.02013,0.01311,-0.02689,0.00247,-0.02445,-0.03718,0.00307,0.00774,0.00495,-0.00145,-0.02505,-0.06807,0.05282,-0.01866,0.0034,0.06462,0.02268,-0.01834,0.00437,0.03333,-0.03028,-0.00205,-0.01992,0.00291,-0.00312,0.02033,0.01145,0.04449,0.07244,-0.01434,0.01164,-0.01788,-0.00801,-0.02117,-0.03054,0.0146,0.01765,-0.04504,0.00633,0.0427,-0.01145,0.01272,-0.04937,0.03976,-0.02522,-0.0062,-0.02807,0.02329,0.04338,-0.00469,0.03403,0.00379,0.02278,-0.01415,-0.0505,0.00756,-0.05988,-0.04672,0.01345,0.03528,0.02346,-0.02717,0.02367,0.0109,0.03678,-0.00212,0.00956,-0.0298,0.00592,-0.03096,-0.02238,0.02461,0.00253,0.01747,0.02629,-0.01784,0.01282,-0.00496,0.02501,0.01547,-0.02617,0.00983,-0.00499,-0.01497,0.01642,-0.02056,0.01205,0.06724,0.01422,0.01044,0.02634,0.01067,0.01288,0.01041,-0.00132,-0.0265,-0.03179,-0.01864,0.02567,-0.02476,-0.01453,-0.02537,0.01648,-0.03386,-0.00404,-0.01197,0.02813,0.01349,-0.00103,0.0362,0.03468,-0.01717,-0.00592,0.03509,0.03049,-0.05479,0.02247,0.00641,0.01327,0.05331,0.00286,-0.04148,0.02662,-0.02991,-0.01189,0.03,-0.01124,-0.01759,0.07745,0.02257,0.00103,0.0138,-0.05063,-0.03517,-0.03791,0.00638,0.0025,0.02377,0.00432,-0.05742,0.03785,0.00496,0.0258,0.00969,0.01271,0.02467,0.00955,-0.02789,0.0023,0.00495,0.03569,-0.05401,0.06956,-0.00585,-0.01595,0.01418,0.01695,0.03314,0.00375,-0.0264,0.02842,-0.00581,0.06678,0.07884,0.02801,-0.00835,0.04728,-0.04482,0.00496,-0.03334,-0.00353,-0.01707,-0.02384,0.01986,-0.02282,-0.01359,0.03213,-0.00551,-0.0134,0.06444,0.0102,0.01717,0.036,0.0449,-0.05578,0.02189,-0.05393,0.05831,-0.00837,0.04365,-0.03047,0.04218,0.04157,0.04729,-0.05191,0.04156,0.00472,-0.00231,0.04871,0.01498,-0.01352,-0.00074,-0.05809,0.00414,-0.01312,0.02515,0.04828,-0.029,0.00185,-0.02006,0.03079,-0.02273,-0.00054,0.00124,-0.04775,-0.03523,0.06123,0.02369,0.03288,-0.00942,-0.02377,0.03404,-0.01834,0.03162,-0.04765,0.00311,-0.04369,0.01712,-0.0046,-0.02907,-0.0196,0.01807,-0.02918,0.02673,-0.01424,0.06018,0.02108,0.05122,0.05261,-0.0844,0.01294,-0.00105,0.01756,0.06813,0.01816,-0.01834,-0.01305,0.04269,0.02993,-0.05113,-0.00208,0.01542,0.05871,-0.01004,-0.00253,0.02197,-0.00239,0.04582,-0.02667,-0.02511,-0.00493,-0.03509,-0.01359,-0.00577,-0.00793,-0.08161,0.01071,-0.02469,-0.00321,-0.02586,0.01736,0.04034,0.03213,-0.0275,0.01922,0.04221,-0.01884,0.0041,0.05143,-0.04363,-0.02603,0.01359,-0.04034,-0.02981,-0.0028,-0.00479,0.02249,0.02391,-0.03841,0.05699,0.00909,-0.05007,0.01069,0.01308,0.01425,0.01592,-0.01341,-0.03907,-0.05654,-0.0339,-0.02548,0.01985,0.00424,0.00145,-0.02947,0.0172,0.02241,0.02027,-0.01031,-0.01832,-0.10862,0.01336,0.00109,-0.09422,-0.01585,0.03974,-0.00537,-0.01127,-0.05054,0.04651,-0.03548,0.01287,-0.00876,0.08663,-0.00255,-0.00923,-0.01168,-0.00829,0.02034,-0.03456,-0.00957,0.02267,0.01669,0.01538,0.00694,0.02215,-0.01303,0.01227,0.01423,-0.01576,0.02942,-0.02075,0.05806,0.00468,-0.07045,-0.03456,0.07849,-0.02087,0.03461,0.01593,0.01949,0.00697,0.02198,0.02225,0.01448,-0.03578,0.03795,-0.02227,-0.0208,-0.01368,0.08223,-0.02417,-0.02438,0.00678,0.05668,-0.04544,0.01442,-0.02251,-0.0069,0.03637,-0.03406,-0.01678,-0.00843,-0.03108,0.04477,0.01902,-0.02718,-0.01966,-0.04527,-0.03346,0.0193,-0.03969,0.02866,-0.04977,-0.0432,-0.0015,-0.06911,-0.00447,-0.02955,0.02724,-0.0512,-0.00296,-0.03071,0.00101,-0.01681,0.03946,0.03542,-0.00468,-0.02902,0.0628,0.00848,-0.00765,-0.02137,-0.06543,-0.00419,0.00129,-0.02534,0.04327,0.03223,-0.00096,0.034,0.03474,-0.04742,0.08326,-0.02815,0.02141,-0.03308,0.00944,-0.04045,-0.02303,-0.0151,-0.01923,-0.03251,-0.01503,-0.01013,0.05736,0.00123,0.04242,0.04085,-0.08235,-0.00573,-0.00472,0.01922,8e-05,-0.00753,-0.03246,0.06445,0.02563,0.03713,0.02073,0.00281,-0.01773,-0.00507,-0.0161,-0.05207,0.0207,-0.01122,0.00498,0.05607,0.02406,-0.0077,0.00524,0.00656,0.02172,-0.00582,0.002,0.0163,0.00321,0.00288,-0.04978,0.03,-0.0309,0.0502,-0.00924,0.0218,-0.02395,0.00244,0.00688,0.01555,0.02774,-0.00523,0.00513,-0.05242,0.00226,-0.0107,0.01536,-0.06243,-0.04384,-0.02613,-0.0093,0.00854,-0.05516,-0.04737,-0.01029,0.0075,0.01335,0.00357,-0.0241,0.04834,-0.03318,0.00376,0.04555,0.01395,0.02176,0.00248,-0.00834,-0.0281,-0.01321,0.01123,0.00135,-0.0306,0.00553,-0.02512,-0.07398,-0.01681,0.02158,-0.01458,0.03779,-0.02546,0.01265,0.03814,0.00501,0.01211,-0.00943,0.02332,-0.00755,-0.01952,0.06056,-0.0167,0.02276,0.00015,0.03737,0.00537,0.02031,-0.03587,-0.002,-0.00012,0.02132,0.07326,-0.05212,-0.01719,0.02474,-0.01287,-0.05148,-0.02772,0.00794,0.01174,-0.0036,0.04666,-0.07608,-0.02778,0.03752,0.03669,0.03921,-0.04757,0.03959,0.05085,0.00134,-0.00052,0.008,-0.05218,-0.05433,-0.03486,0.00802,-0.06204,-0.03135,0.02423,0.00268,0.00185,-0.04034,-0.0165,-0.03715,-0.01869,0.00485,0.04019,0.03224,0.03855,0.04126,-0.02699,-0.01076,0.03594,-0.02552,0.00764,0.01708,0.00122,0.02882,0.06046,0.02527,0.0274,0.00398,0.00739,-0.02137,-1e-05,0.00693,0.03125,-0.01413,-0.00583,0.05256,-0.02371,0.0205,0.03769,0.048,0.02885,-0.06224,0.02187,-0.02163,0.04458,0.0028,0.04348,-0.03598,-0.0406,0.00882,0.04196,-0.05882,-0.02998,-0.03239,-0.02962,0.01559,0.00703,-0.06205,-0.0125,0.02488,-0.01178,0.0424,0.02942,0.00555,-0.0195,-0.01782,0.0299,0.01069,-0.05646,0.00072,-0.01938,-0.02975,-0.00091,-0.05657,0.0558,0.00578,-0.04121,-0.00179,0.04888,0.01047,0.00592,0.00888,0.02142,0.01432,0.01216,-0.01609,0.03247,-0.0326,0.03299,-0.03135,0.04498,0.04766,0.03267,0.06434,0.01058,0.02177,-0.03881,0.01728,-0.03258,-0.03123,-0.05876,-0.03556,-0.03094,0.03852,0.02517,-0.01034,-0.02895,-0.00488,0.02061,0.00195,-0.00882,-0.02741,0.02004,0.00786,-0.0326,0.00172,0.00652,0.01467,-0.02109,-0.01082],[-0.00206,0.04185,-0.04616,-0.03325,-0.01363,0.0082,0.01742,0.05534,0.01351,-0.01256,0.0137,-0.03019,0.00824,-0.03269,-0.01796,0.03263,-0.00355,0.0102,0.01736,0.00741,0.00189,-0.06432,-0.04325,0.04347,0.05882,-0.00604,0.02799,0.05457,-0.04737,-0.03622,0.02471,0.08941,-0.01274,0.02435,0.01284,0.03211,-0.01611,-0.03103,-0.01731,0.00477,0.01518,0.00549,-0.02322,0.00771,0.03693,-0.01392,-0.06142,0.03588,0.01451,0.00468,-0.01784,-0.01676,0.01256,-0.00215,0.00603,0.04483,-0.00245,0.01215,0.06297,-0.00953,0.05738,0.01576,0.04132,0.02367,0.00396,0.01046,-0.02265,-0.00082,0.01129,0.00746,0.06808,0.02221,-0.00141,-0.00015,-0.05808,0.0123,-0.00018,-0.04353,-0.04387,0.01658,0.0369,0.01736,-0.04124,-0.00343,-0.00814,-0.00035,0.01998,-0.0409,-0.01798,-0.07698,0.00107,0.01193,0.02141,-0.00631,-0.02103,0.02256,-0.00209,-0.03015,-0.00204,-0.05704,-0.05389,0.02812,0.07857,0.00886,-0.03011,0.0054,0.02466,-0.00949,0.05027,-0.0005,-0.0035,0.0802,-0.02775,-0.06988,-0.01802,0.00885,0.03015,-0.02436,-0.01613,0.01159,-0.0055,-0.03583,0.03152,0.02712,-0.02054,-0.01463,-0.10985,-0.02312,-0.00278,0.04664,0.04068,0.02423,0.01606,-0.03462,0.03663,0.00702,0.03483,-0.01558,0.01751,-0.04317,-0.00442,-0.0077,-0.01749,-0.07771,0.0298,-0.02019,0.00635,-0.00566,0.05242,0.00437,-0.04975,-0.01857,-0.02515,-0.03745,-0.01927,-0.01457,-0.01866,-0.03556,-0.00124,-0.05705,0.03255,0.03204,-0.0137,0.0852,0.03619,-0.00916,-0.02803,0.01214,-0.01861,-0.01464,0.0049,0.01991,0.06227,-0.01888,0.01824,-0.00231,0.02944,0.04604,-0.0331,0.02263,0.0051,-0.00735,0.00551,0.0072,0.03683,0.0038,0.02445,0.00724,-0.0013,0.04939,-0.02201,-0.00442,0.02748,-0.0205,-0.02536,0.01176,-0.06234,0.02924,-0.04934,0.08147,0.00588,-0.03138,0.03934,0.02914,0.04506,-0.00323,-0.04797,-0.03986,-0.00735,0.05942,-0.02488,-0.02243,-0.07256,0.02463,0.00706,0.00536,0.00423,0.04708,0.04336,-0.00569,-0.02316,0.02415,-0.02535,0.00019,0.01234,0.01061,-0.00099,0.09205,0.00833,0.00127,0.0048,-0.04443,-0.00441,0.03757,-0.02662,0.004,0.03018,-0.00481,0.02466,0.06229,0.00082,-0.00233,-0.03372,-0.00677,-0.01455,0.05469,0.02689,-0.0046,-0.05951,-0.00254,0.00873,0.06474,-0.00062,-0.00091,0.00322,-0.00839,-0.05954,-0.04145,-0.03907,0.00237,0.04903,0.03624,0.0301,-0.00821,0.0074,0.01214,-0.00045,-0.01308,0.02098,-0.03085,0.00782,0.04569,0.01247,-0.00435,0.01807,-0.01932,-0.00761,0.01233,-0.00909,-0.01043,-0.00562,-0.05074,0.01968,0.06434,0.00438,-0.0325,-0.01599,-0.00363,0.00043,0.00717,-0.00278,-0.04382,-0.03562,-0.03171,-0.08286,0.00014,0.01623,-0.02824,-0.03717,0.00498,-0.04826,-0.00263,-0.05056,-0.04194,-0.06332,0.06747,0.00319,-0.0241,0.08014,0.04745,0.03,-0.02278,0.01141,0.00233,-0.05887,-0.0393,0.00773,-0.04164,0.05626,-0.03491,0.03835,-0.05679,-0.0455,0.0033,0.05183,-0.02316,0.00121,-0.01228,0.03183,0.00488,0.01489,0.0265,0.03702,0.03868,-0.00911,0.06631,-0.01747,-0.03507,0.00137,-0.00545,-0.04762,0.01688,-0.01308,-0.00574,-0.03176,0.0231,-0.03408,-0.02792,-0.04425,0.05368,0.02461,-0.00494,-0.03715,-0.02371,-0.00814,-0.00642,-0.03314,0.01627,-0.00674,0.0027,-0.00055,0.04965,-0.01034,-0.0039,0.0244,-0.01923,0.02178,0.03745,-0.0033,-0.00019,-0.01738,-0.01694,-0.01221,-0.00066,-0.03128,-0.0155,0.03564,0.02341,-0.02074,-0.00488,0.017,0.01026,-0.08508,-0.01792,-0.03493,-0.01157,0.03419,-0.00872,-0.01336,-0.0178,0.0033,0.02473,-0.01584,-0.00609,0.07753,-0.05776,-0.09227,0.00851,0.00028,0.02119,0.09262,0.02519,0.01116,0.01204,0.00134,0.00788,0.00674,-0.00923,-0.02457,-0.04842,-0.01902,0.01435,0.00651,0.04692,0.02628,-0.02251,-0.04578,0.03277,0.03743,-0.0183,-0.0186,-0.03579,-0.04275,-0.00343,0.03249,0.01309,0.00876,-0.04225,-0.01847,-0.01134,0.00493,-0.02431,0.00356,0.00173,-0.0063,-0.00407,-0.05433,-0.0562,0.04584,0.02586,-0.04907,0.01275,-0.01099,-0.01046,0.02472,0.00286,-0.0039,-0.03374,0.00101,0.01051,0.01628,-0.00488,-0.07494,-0.02079,0.0633,-0.01488,-0.00093,-0.02143,-0.02613,0.00891,-0.0145,0.0108,0.01447,0.01659,-0.01807,-0.02359,-0.02848,-0.02412,0.05532,0.07516,-0.02723,0.02305,-0.04249,-0.03747,-0.03154,-0.0061,0.03429,-0.00176,-0.02662,0.02845,-0.01058,0.07094,-0.0081,-0.02039,-0.04784,0.04751,-0.01331,-0.02918,-0.02033,0.07103,0.0086,0.01157,0.0155,-0.02949,0.03427,0.00323,0.03064,0.03099,-0.02994,-0.02622,0.05816,0.07138,-0.01573,-0.03923,0.02165,-0.03942,-0.04769,0.02503,-0.06287,0.00134,-0.035,-0.01707,0.00082,0.00742,-0.00899,0.02446,-0.03779,-0.04808,0.01033,0.01803,-0.02838,-0.00682,-0.01022,-0.03513,-0.00151,0.03024,-0.01573,0.01315,-0.00141,0.01937,0.01113,0.03308,0.00821,0.03881,0.05997,0.02937,0.00052,0.03524,0.01438,-0.00951,0.0479,0.01645,0.02437,0.04108,-0.04179,0.01595,0.00915,0.01474,-0.01479,-0.01138,-0.00546,-0.00542,-0.02009,0.01217,-0.00839,-0.00528,0.03692,-0.02462,0.04018,0.01027,-0.02305,0.01413,-0.03221,-0.01619,-0.00366,0.00257,-0.01686,-0.01958,0.04608,-0.04776,-0.0189,-0.01264,-0.01905,0.00327,-0.00213,0.01189,-0.01534,0.04735,-0.02226,0.02636,0.00891,-0.02736,-0.0681,0.01168,-0.03668,0.04577,0.0606,-0.00887,0.02025,0.00978,-0.01444,0.01215,-0.03604,0.02607,-0.01037,0.00013,0.01262,0.00477,-0.01335,0.00707,-0.00663,-0.03453,-0.01628,0.01052,0.00832,-0.06043,0.03126,0.03244,0.00244,0.0503,-0.00291,0.0403,0.0198,-0.02373,0.03449,0.04352,0.02152,-0.04059,0.01888,-0.01627,-0.03998,0.02578,-0.0159,-0.00028,-0.0416,0.00671,0.00205,0.02315,0.00656,-0.00318,-0.01772,-0.02864,0.02008,-0.0263,0.01811,-0.0233,0.01575,-0.04911,-0.01261,0.02936,0.00213,0.02135,0.03473,-0.04622,0.04333,-0.00737,-0.03358,0.0227,0.01581,-0.00803,0.04156,0.01076,-0.01992,0.04343,-0.01923,-0.04269,-0.01079,0.01713,-0.03031,0.00322,0.02112,0.00213,-0.02852,0.04839,0.04973,0.05696,0.05122,-0.02071,0.00917,0.0374,0.00752,0.05741,0.01297,-0.0085,0.05919,-0.09915,-0.00748,-0.02673,-0.05797,-0.04767,0.05608,0.00457,0.03995,0.00935,-0.0533,0.02858,-0.00054,0.06385,0.00376,0.01214,0.05129,-0.01525,0.02155,0.01229,0.01694,-0.02441,-0.05561,0.02383,0.05626,-0.01144,-0.00459,0.00665,-0.04026,-0.02969,-0.00016,-0.03639,-0.0002,0.01423,-0.01365,0.0454,0.02831,0.04642,0.00878,0.10063,-0.04216,0.04849,-0.01032,-0.04384,-0.01958,0.03792,-0.0294,0.04493,-0.01516,-0.02431,0.01484,-0.05908,-0.0253,-0.00734,-0.0294,-0.0166,-0.01069,-0.01722,-0.01005,0.03595,0.00291,-0.05385,-0.02243,0.04624,-0.02963,0.0159,0.03085,-0.03689,-0.01771,0.01029,-0.01832,-0.02373,-0.04424,0.03274,-0.00124,-0.029,0.00258,0.00913,-0.00039,0.01209,0.03608,0.00296,0.02905,-0.01307,0.0118,-0.01356,0.00902,-0.04151,0.01593,-0.06927,0.02683,-0.02332,0.00528,0.07809,0.04598,0.00329,-0.01174,-0.02161,0.0144,0.03986,-0.00577,-0.00054,0.01055,0.10328,-0.00854,0.04123,-0.0281,-0.01627,-0.00809,0.00506,-0.01148,-0.00705,-0.02379,-0.00815,0.01779,-0.02028,0.01839,0.00314,0.00357,-0.00945,0.0606,-0.02172,-0.00588,-0.00081,-0.00645,-0.0295,0.0403,0.02714,0.03324,-0.02197,0.02591,0.07449,-0.02155,-0.03062,-0.01114,-0.02688,-0.00346,-0.03228,-0.03849,0.00818,-0.02564,0.01415,-0.01518,0.00811,-0.0192,-0.03064,-0.0161,-0.01336,-0.01244,-0.03178,0.03565,0.01717,0.02071,-0.03996,-0.04877,-0.04514,-0.00076,0.03818,-0.0116,-0.0291,-0.02425,-0.02174,0.02149,0.00177,0.02188,-0.01879,0.04391,-0.00093,0.01344,0.05639,-0.03074,-0.01048,0.02899,-0.01762,0.01265,-0.04332,-0.00886,0.04306,-0.02101,-0.04151,-0.02396,0.03238,0.00911,0.02717,0.01738,-0.01189,-0.03267,0.00315,-0.03517,-0.05413,0.0594,0.00687,0.02194,0.05496,-0.05165,-0.00918,0.05983,-0.01215,-0.01962,-0.03941,-0.00014,-0.01077,0.00628,0.02216,-0.01699,-0.04789,0.01132,-0.03225,0.00092,-0.01141,-0.01038,-0.05065,0.03999,-0.00573,0.06075,0.03056,0.0105,0.01771,-0.03294,-0.00462,0.02377,0.0391,-0.04897,0.0438,0.02617,8e-05,0.00045,-0.03178,-0.03441,-0.05191,-0.02425,-0.01991,-0.02728,0.03142,0.01688,0.00766,0.0004,-0.01117,-0.03277,-0.01741,-0.00079,-0.01226,0.06248,0.03794,-0.03267,0.01789,0.00371,0.01055,-0.04564,-0.03566,-0.01644,0.01897,-0.00794,-0.03457,-0.00889,-0.03042,-0.01716,-0.0378,-0.05054,-0.02853,0.02834,-0.00845,0.03893,0.02269,0.05895,0.04078,0.01726,-0.01072,-0.00909,0.05516,0.00596,0.02647,-0.00682,0.01746,0.03387,0.00924,0.06054,0.01094,-0.0156,-0.05887,-0.01205,0.04112,0.00109,0.05043,-0.03037,-0.01829,-0.00996,-0.00122,0.00906,-0.01136,-0.00464,0.01854,-0.06368,-0.01129,-0.0293,0.002,0.00344,0.00424,-0.00762,-0.02214,-0.01772,0.04501,-0.04189,-0.00136,-0.02585,0.00197,0.00762,-0.0319,0.01055,0.0245,-0.00664,-0.03415,0.02254,0.03979,-0.03625,-0.02966,0.04111,0.01457,0.02157,0.02104,-0.00403,0.00085,-0.00641,0.04651,-0.0203,0.01873,0.02191,0.02907,-0.03252,-0.03089,0.09486,0.02133,0.04798,-0.01348,-0.00205,-0.0566,-0.02041,0.03671,-0.01997,-0.01283,-0.00655,0.0332,0.04795,-0.01107,-0.01457,0.01461,0.01305,0.05409,-0.01398,0.01082]],"prompts":["Hello world.","The quick brown fox jumps over the lazy dog."],"revision":"cf74d8acd4f198de950bf004b262e6accfed5d2c"}} \ No newline at end of file diff --git a/tests/data/generate_decoder_output_refs.py b/tests/data/generate_decoder_output_refs.py deleted file mode 100644 index ce70d7704..000000000 --- a/tests/data/generate_decoder_output_refs.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2026 The Spyre-Inference Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Write CPU HF greedy references for tests/e2e/test_model_quality.py. - -The models are too large to run through transformers in CI, so the references are -generated here and checked in. Each run merges into the existing file. - - python tests/data/generate_decoder_output_refs.py - python tests/data/generate_decoder_output_refs.py --models ibm-granite/granite-4.1-8b -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from typing import Any - -import torch -from transformers import AutoModelForCausalLM, AutoTokenizer - -# Model ids must match tests/e2e/test_model_quality.py. -DECODER_MODELS = [ - "ibm-granite/granite-3.3-8b-instruct", - "ibm-granite/granite-4.1-8b", - "google/gemma-4-31B", - "google/gemma-4-26B-A4B", - "meta-llama/Llama-3.1-8B-Instruct", -] - -MODEL_REVISIONS = { - "ibm-granite/granite-3.3-8b-instruct": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b", - "ibm-granite/granite-4.1-8b": "1504002f650e656a0a3789d99574df12e3e94ed0", - "google/gemma-4-31B": "5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89", - "google/gemma-4-26B-A4B": "24548b62aa021d562695c04aaf7758a1ea47990b", - "meta-llama/Llama-3.1-8B-Instruct": "0e9e39f249a16976918f6564b8830bc894c89659", -} - -# Must stay within MAX_NUM_BATCHED_TOKENS (test_model_quality.py) so each prefill lands -# in a single compiled bucket; that test asserts it before building its engine. -_TEMPLATE = ( - "Below is an instruction that describes a task. Write a response that " - "appropriately completes the request.\n\n### Instruction:\n{}\n\n### Response:" -) -PROMPTS = [ - _TEMPLATE.format("Provide a list of instructions for preparing chicken soup."), - _TEMPLATE.format("What are the main businesses of IBM?"), - _TEMPLATE.format("Convert char to string in Java."), -] - -# gemma-4 diverges from HF on the prompts above because torch-spyre runs RMSNorm in fp16; -# short prompts match token for token. Drop these entries once it normalises in fp32. -_GEMMA4_PROMPTS = [ - "What are IBMs main businesses?", - "The capital of France is", - "Q: What is the largest planet in our solar system?\nA:", -] -MODEL_PROMPTS = { - "google/gemma-4-31B": _GEMMA4_PROMPTS, - "google/gemma-4-26B-A4B": _GEMMA4_PROMPTS, -} - -MAX_TOKENS = 16 -_ROUND = 6 - -OUT_PATH = Path(__file__).parent / "decoder_output_refs.json" - - -def generate_reference(model_id: str, revision: str, dtype: torch.dtype) -> dict[str, Any]: - tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision) - model = AutoModelForCausalLM.from_pretrained(model_id, revision=revision, dtype=dtype) - model.eval() - # The test runs with ignore_eos=True, so the reference needs all MAX_TOKENS steps. - model.generation_config.eos_token_id = None - - results = [] - for prompt in MODEL_PROMPTS.get(model_id, PROMPTS): - input_ids = tokenizer(prompt, return_tensors="pt").input_ids - with torch.inference_mode(): - output = model.generate( - input_ids, - do_sample=False, - max_new_tokens=MAX_TOKENS, - return_dict_in_generate=True, - output_scores=True, - ) - # normalize_logits gives logprobs over the vocabulary, matching what vLLM reports. - logprobs = model.compute_transition_scores( - output.sequences, output.scores, normalize_logits=True - )[0] - new_token_ids = output.sequences[0, input_ids.shape[1] :] - - results.append( - { - "prompt": prompt, - "text": tokenizer.decode(new_token_ids), - "token_ids": [int(t) for t in new_token_ids], - "tokens": [tokenizer.decode(t) for t in new_token_ids], - "logprobs": [round(float(lp), _ROUND) for lp in logprobs], - } - ) - print(f" {prompt!r}\n -> {results[-1]['text']!r}", flush=True) - - return { - "revision": revision, - "max_tokens": MAX_TOKENS, - "dtype": str(dtype).removeprefix("torch."), - "results": results, - } - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--models", nargs="+", default=DECODER_MODELS) - parser.add_argument("--dtype", default="float32", choices=["float32", "bfloat16", "float16"]) - parser.add_argument("--out", type=Path, default=OUT_PATH) - args = parser.parse_args() - - data = json.loads(args.out.read_text()) if args.out.exists() else {} - for model_id in args.models: - print(f"Generating {model_id} ...", flush=True) - data[model_id] = generate_reference( - model_id, MODEL_REVISIONS[model_id], getattr(torch, args.dtype) - ) - # Written per model so an interrupted run keeps what it already generated. - args.out.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") - print(f"Wrote {args.out}", flush=True) - - -if __name__ == "__main__": - main() diff --git a/tests/data/generate_encoder_embed_refs.py b/tests/data/generate_encoder_embed_refs.py deleted file mode 100644 index da90e65d3..000000000 --- a/tests/data/generate_encoder_embed_refs.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2026 The Spyre-Inference Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Write CPU HF embeddings to encoder_embed_refs.json for cosine checks. - -python tests/data/generate_encoder_embed_refs.py -""" - -from __future__ import annotations - -import json -from pathlib import Path - -from sentence_transformers import SentenceTransformer - -# Model ids must match tests/e2e/test_encoder_models.py. -EMBEDDING_MODELS = [ - "ibm-granite/granite-embedding-125m-english", - "ibm-granite/granite-embedding-278m-multilingual", - "intfloat/multilingual-e5-large", - "sentence-transformers/all-roberta-large-v1", -] - -# Written into the JSON per model and read back by the test, so the gate loads the same -# weights measured here and an upstream re-upload cannot redefine what it compares against. -MODEL_REVISIONS = { - "ibm-granite/granite-embedding-125m-english": "4ab61ffd423be45cd932b21a7c696063d82bf45f", - "ibm-granite/granite-embedding-278m-multilingual": "a9cb5338491faf32b73dd17b714a31821c021bbf", - "intfloat/multilingual-e5-large": "3d7cfbdacd47fdda877c5cd8a79fbcc4f2a574f3", - "sentence-transformers/all-roberta-large-v1": "cf74d8acd4f198de950bf004b262e6accfed5d2c", -} - -EMBEDDING_PROMPTS = [ - "Hello world.", - "The quick brown fox jumps over the lazy dog.", -] - -_ROUND = 5 - - -def main() -> None: - prompts = [p.strip() for p in EMBEDDING_PROMPTS] - data: dict[str, dict] = {} - - for model in EMBEDDING_MODELS: - revision = MODEL_REVISIONS[model] - print(f"Encoding {model} @ {revision} ...") - st = SentenceTransformer(model, revision=revision, device="cpu") - embeddings = st.encode(prompts, normalize_embeddings=True) - data[model] = { - "revision": revision, - "prompts": prompts, - "embeddings": [[round(float(x), _ROUND) for x in row] for row in embeddings], - } - - out_path = Path(__file__).parent / "encoder_embed_refs.json" - out_path.write_text(json.dumps(data, separators=(",", ":"), sort_keys=True)) - size_kb = out_path.stat().st_size / 1024 - print(f"Wrote {out_path} ({size_kb:.1f} KB)") - - -if __name__ == "__main__": - main() diff --git a/tests/data/generate_rerank_score_refs.py b/tests/data/generate_rerank_score_refs.py deleted file mode 100644 index a3c233d23..000000000 --- a/tests/data/generate_rerank_score_refs.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2026 The Spyre-Inference Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Write CPU HF cross-encoder scores to rerank_score_refs.json for the reranker gates. - -Each run merges into the existing file. - - python tests/data/generate_rerank_score_refs.py - python tests/data/generate_rerank_score_refs.py --models BAAI/bge-reranker-large -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from typing import Any - -import torch -from transformers import AutoModelForSequenceClassification, AutoTokenizer - -# Model ids must match tests/e2e/test_encoder_models.py. -RERANKER_MODELS = [ - "BAAI/bge-reranker-v2-m3", - "BAAI/bge-reranker-large", -] - -MODEL_REVISIONS = { - "BAAI/bge-reranker-v2-m3": "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e", - "BAAI/bge-reranker-large": "55611d7bca2a7133960a6d3b71e083071bbfc312", -} - -# One query against documents a reranker should separate widely, most relevant first: the -# ranking gate needs neighbouring scores farther apart than the tolerance each can drift. -QUERY = "What is the capital of France?" -DOCUMENTS = [ - "The capital of France is Paris.", - "Paris is the largest city in France by population.", - "The Eiffel Tower stands on the Champ de Mars in Paris.", - "France is a country in Western Europe with about 68 million inhabitants.", - "Berlin is the capital of Germany.", - "The IBM Spyre accelerator runs AI inference workloads.", -] - -# bge-reranker-large scores every Paris-adjacent document above 0.9994, so on the shared -# list its top five land within fp16 noise of each other and their order is arbitrary. -MODEL_DOCUMENTS = { - "BAAI/bge-reranker-large": [ - "The capital of France is Paris.", - "France is a country in Western Europe with about 68 million inhabitants.", - "France moved its seat of government several times in its history.", - "Berlin is the capital of Germany.", - "The IBM Spyre accelerator runs AI inference workloads.", - ], -} - -# Scores span ~1e-5 to ~1 and the test bounds small ones *relatively*, so a fixed number -# of decimals has to keep several significant digits at the bottom of that range. -_ROUND = 8 - -OUT_PATH = Path(__file__).parent / "rerank_score_refs.json" - - -def generate_reference(model_id: str, revision: str) -> dict[str, Any]: - tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision) - model = AutoModelForSequenceClassification.from_pretrained( - model_id, revision=revision, dtype=torch.float32 - ) - model.eval() - - documents = MODEL_DOCUMENTS.get(model_id, DOCUMENTS) - scores = [] - for document in documents: - # The same tokenizer call vLLM's cross-encoder io_processor makes, one pair at a - # time so no padding reaches the reference. - inputs = tokenizer(text=QUERY, text_pair=document, return_tensors="pt") - with torch.inference_mode(): - logit = model(**inputs).logits.reshape(-1) - assert logit.numel() == 1, f"{model_id}: expected num_labels=1, got {logit.numel()}" - # vLLM's PoolerClassify sigmoids a single-label head, so the reference is a - # probability on [0, 1]. - scores.append(round(float(torch.sigmoid(logit)[0]), _ROUND)) - print(f" {document!r}\n -> {scores[-1]:.6f}", flush=True) - - return { - "revision": revision, - "query": QUERY, - "documents": documents, - "scores": scores, - } - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--models", nargs="+", default=RERANKER_MODELS) - parser.add_argument("--out", type=Path, default=OUT_PATH) - args = parser.parse_args() - - data = json.loads(args.out.read_text()) if args.out.exists() else {} - for model_id in args.models: - print(f"Scoring {model_id} ...", flush=True) - data[model_id] = generate_reference(model_id, MODEL_REVISIONS[model_id]) - args.out.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") - print(f"Wrote {args.out}", flush=True) - - -if __name__ == "__main__": - main() diff --git a/tests/data/rerank_score_refs.json b/tests/data/rerank_score_refs.json deleted file mode 100644 index cacf8cebe..000000000 --- a/tests/data/rerank_score_refs.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "BAAI/bge-reranker-large": { - "documents": [ - "The capital of France is Paris.", - "France is a country in Western Europe with about 68 million inhabitants.", - "France moved its seat of government several times in its history.", - "Berlin is the capital of Germany.", - "The IBM Spyre accelerator runs AI inference workloads." - ], - "query": "What is the capital of France?", - "revision": "55611d7bca2a7133960a6d3b71e083071bbfc312", - "scores": [ - 0.99946493, - 0.0866667, - 0.00186743, - 0.0005165, - 7.623e-05 - ] - }, - "BAAI/bge-reranker-v2-m3": { - "documents": [ - "The capital of France is Paris.", - "Paris is the largest city in France by population.", - "The Eiffel Tower stands on the Champ de Mars in Paris.", - "France is a country in Western Europe with about 68 million inhabitants.", - "Berlin is the capital of Germany.", - "The IBM Spyre accelerator runs AI inference workloads." - ], - "query": "What is the capital of France?", - "revision": "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e", - "scores": [ - 0.99996591, - 0.9614659, - 0.59719944, - 0.04726163, - 0.00421512, - 1.604e-05 - ] - } -} diff --git a/tests/e2e/test_compile.py b/tests/e2e/test_compile.py index 56a334889..5dacdc7c0 100644 --- a/tests/e2e/test_compile.py +++ b/tests/e2e/test_compile.py @@ -16,13 +16,7 @@ from __future__ import annotations -import json -import math -from pathlib import Path - import pytest -import torch -import torch.nn.functional as F # enforce_eager=False builds a subprocess EngineCore, so uses_subprocess runs these # before any in-process test initializes the Spyre device (a subprocess cannot open @@ -30,8 +24,8 @@ pytestmark = pytest.mark.uses_subprocess _POOLING_MODEL = "ibm-granite/granite-embedding-125m-english" -_POOLING_REFS = Path(__file__).parent.parent / "data" / "encoder_embed_refs.json" -_COSINE_MIN = 0.99 +_POOLING_REVISION = "4ab61ffd423be45cd932b21a7c696063d82bf45f" +_POOLING_PROMPTS = ["Hello world.", "The quick brown fox jumps over the lazy dog."] @pytest.mark.parametrize( @@ -75,8 +69,10 @@ def test_whole_model_granularity(monkeypatch: pytest.MonkeyPatch) -> None: ) -def test_compiled_pooling_encoder_buckets(monkeypatch: pytest.MonkeyPatch) -> None: - """Compiled pooling pads to ``(B, L)`` and matches cached HF refs. +def test_compiled_pooling_encoder_buckets( + hf_embeddings, assert_embeddings_close, monkeypatch: pytest.MonkeyPatch +) -> None: + """Compiled pooling pads to ``(B, L)`` and matches live HF. Two prompts at ``max_num_seqs=2`` / ``max_model_len=64`` warmup body ``T`` and attention ``(1, 64)`` / ``(2, 64)``. Runtime 1D-pads the body; SDPA @@ -84,9 +80,8 @@ def test_compiled_pooling_encoder_buckets(monkeypatch: pytest.MonkeyPatch) -> No """ from vllm import LLM - refs = json.loads(_POOLING_REFS.read_text())[_POOLING_MODEL] - prompts = refs["prompts"] monkeypatch.setenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "36000") + hf_embs = hf_embeddings(_POOLING_MODEL, _POOLING_REVISION, _POOLING_PROMPTS) engine = LLM( model=_POOLING_MODEL, @@ -95,18 +90,9 @@ def test_compiled_pooling_encoder_buckets(monkeypatch: pytest.MonkeyPatch) -> No max_model_len=64, max_num_seqs=2, ) - outputs = engine.embed(prompts) - assert len(outputs) == len(prompts) - for out, ref_emb in zip(outputs, refs["embeddings"]): - emb = out.outputs.embedding - assert len(emb) == len(ref_emb) - assert all(math.isfinite(x) for x in emb) - sim = F.cosine_similarity( - torch.tensor(emb, dtype=torch.float32), - torch.tensor(ref_emb, dtype=torch.float32), - dim=0, - ).item() - assert sim >= _COSINE_MIN, f"cosine {sim:.4f} < {_COSINE_MIN}" + outputs = engine.embed(_POOLING_PROMPTS) + assert len(outputs) == len(_POOLING_PROMPTS) + assert_embeddings_close(_POOLING_MODEL, [out.outputs.embedding for out in outputs], hf_embs) def _assert_compiled_output(model: str, ref_output: str, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/e2e/test_encoder_models.py b/tests/e2e/test_encoder_models.py index 176576465..5787419c7 100644 --- a/tests/e2e/test_encoder_models.py +++ b/tests/e2e/test_encoder_models.py @@ -12,18 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Spyre product encoder tests vs cached HF refs: embeddings, reranker scores, labels. +"""Spyre product encoder tests vs live CPU HF: embeddings, reranker scores, labels. -Regenerate refs: ``python tests/data/generate_encoder_embed_refs.py`` and -``python tests/data/generate_rerank_score_refs.py`` +References are computed in-run through upstream's ``HfRunner`` and compared with its +``check_embeddings_close``, the way vLLM's ``tests/models/language/pooling/`` tests do. """ from __future__ import annotations -import json import math import os -from pathlib import Path import pytest import torch @@ -69,29 +67,35 @@ "George Washington went to Washington", ] -# Match upstream check_embeddings_close(tol=1e-2). -COSINE_MIN = 0.99 +# Upstream check_embeddings_close's tolerance: it asserts cosine >= 1 - tol. +EMBEDDING_TOL = 1e-2 -# Reranker references are sigmoid probabilities and most sit just above zero, where an -# absolute bound permits an arbitrary relative error, so the stricter of the two applies. +# Local bounds because upstream has no reranker helper, and its cross-encoder test's inlined +# ones only hold with both sides at the same precision. Here fp16 on the card runs against an +# fp32 CPU reference, where a mid-range sigmoid score drifts by ~1e-2. Most scores sit just +# above zero, where an absolute bound admits any relative error, so the stricter one applies. SCORE_ABS_TOL = float(os.environ.get("SPYRE_TEST_SCORE_ABS_TOL", "0.03")) SCORE_REL_TOL = float(os.environ.get("SPYRE_TEST_SCORE_REL_TOL", "0.5")) -_REF_PATH = Path(__file__).parent.parent / "data" / "encoder_embed_refs.json" -_REFERENCES: dict = json.loads(_REF_PATH.read_text()) if _REF_PATH.exists() else {} - -_RERANK_REF_PATH = Path(__file__).parent.parent / "data" / "rerank_score_refs.json" -_RERANK_REFERENCES: dict = ( - json.loads(_RERANK_REF_PATH.read_text()) if _RERANK_REF_PATH.exists() else {} -) - - -def _cosine(a: list[float], b: list[float]) -> float: - return F.cosine_similarity( - torch.tensor(a, dtype=torch.float32), - torch.tensor(b, dtype=torch.float32), - dim=0, - ).item() +# Every gated model is pinned, matching .github/cache_config/hf_models_and_datasets.yaml. +MODEL_REVISIONS = { + "ibm-granite/granite-embedding-125m-english": "4ab61ffd423be45cd932b21a7c696063d82bf45f", + "ibm-granite/granite-embedding-278m-multilingual": "a9cb5338491faf32b73dd17b714a31821c021bbf", + "intfloat/multilingual-e5-large": "3d7cfbdacd47fdda877c5cd8a79fbcc4f2a574f3", + "sentence-transformers/all-roberta-large-v1": "cf74d8acd4f198de950bf004b262e6accfed5d2c", + "BAAI/bge-reranker-v2-m3": "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e", + "BAAI/bge-reranker-large": "55611d7bca2a7133960a6d3b71e083071bbfc312", +} + +RERANK_QUERY = "What is the capital of France?" +# A relevant document, near-misses, and two irrelevant ones, so the scores have to spread. +RERANK_DOCUMENTS = [ + "The capital of France is Paris.", + "Paris is the largest city in France by population.", + "France is a country in Western Europe with about 68 million inhabitants.", + "Berlin is the capital of Germany.", + "The IBM Spyre accelerator runs AI inference workloads.", +] def _hf_last_token_embeddings(model: str, revision: str, prompts: list[str]) -> list[list[float]]: @@ -118,104 +122,86 @@ def _hf_last_token_embeddings(model: str, revision: str, prompts: list[str]) -> @pytest.mark.uses_subprocess @pytest.mark.parametrize("model", EMBEDDING_MODELS) -def test_encoder_embed_models(model: str) -> None: - """Spyre embeddings match cached HF references within cosine tolerance.""" - _assert_embeddings_match_refs(model, enforce_eager=True) +def test_encoder_embed_models( + hf_embeddings, assert_embeddings_close, example_prompts, model: str +) -> None: + """Spyre embeddings match live HF within cosine tolerance.""" + _assert_embeddings_match_hf( + hf_embeddings, assert_embeddings_close, example_prompts, model, enforce_eager=True + ) @pytest.mark.model_quality @pytest.mark.uses_subprocess @pytest.mark.parametrize("model", EMBEDDING_MODELS) -def test_encoder_embed_models_compiled(model: str) -> None: - """Same models and references, compiled rather than eager.""" - _assert_embeddings_match_refs(model, enforce_eager=False) +def test_encoder_embed_models_compiled( + hf_embeddings, assert_embeddings_close, example_prompts, model: str +) -> None: + """Same models and comparison, compiled rather than eager.""" + _assert_embeddings_match_hf( + hf_embeddings, assert_embeddings_close, example_prompts, model, enforce_eager=False + ) -def _assert_embeddings_match_refs(model: str, enforce_eager: bool) -> None: - ref = _REFERENCES.get(model) - if ref is None: - pytest.skip(f"No HF ref for {model}; run tests/data/generate_encoder_embed_refs.py") +def _assert_embeddings_match_hf( + hf_embeddings, + assert_embeddings_close, + prompts: list[str], + model: str, + enforce_eager: bool, + max_num_seqs: int = 1, +) -> None: + revision = MODEL_REVISIONS[model] + # sentence-transformers strips its inputs, so the vLLM side must send the same text. + prompts = [prompt.strip() for prompt in prompts] + hf_embs = hf_embeddings(model, revision, prompts) - prompts = ref["prompts"] llm = LLM( model=model, - revision=ref["revision"], - tokenizer_revision=ref["revision"], + revision=revision, + tokenizer_revision=revision, runner="pooling", max_model_len=64, - max_num_seqs=1, + max_num_seqs=max_num_seqs, enforce_eager=enforce_eager, ) outputs = llm.embed(prompts) assert len(outputs) == len(prompts) - for prompt, out, ref_emb in zip(prompts, outputs, ref["embeddings"]): - emb = out.outputs.embedding - assert len(emb) == len(ref_emb), ( - f"{model}: dim mismatch {len(emb)} vs cached {len(ref_emb)}" - ) - assert all(math.isfinite(x) for x in emb) - sim = _cosine(emb, ref_emb) - assert sim >= COSINE_MIN, ( - f"{model}: cosine {sim:.4f} < {COSINE_MIN} vs cached HF reference for prompt {prompt!r}" - ) + assert_embeddings_close( + model, [out.outputs.embedding for out in outputs], hf_embs, EMBEDDING_TOL + ) @pytest.mark.uses_subprocess @pytest.mark.parametrize("model", MEAN_POOLING_MODELS) -def test_encoder_embed_mean_multi_seq(model: str) -> None: +def test_encoder_embed_mean_multi_seq( + hf_embeddings, assert_embeddings_close, example_prompts, model: str +) -> None: """MEAN with ``max_num_seqs=2`` so two requests share one packed ``[T, H]``. The default embed e2e is ``max_num_seqs=1`` and never hits two sequences in one packed ``[T, H]`` copy. """ - ref = _REFERENCES.get(model) - if ref is None: - pytest.skip(f"No HF ref for {model}; run tests/data/generate_encoder_embed_refs.py") - - prompts = ref["prompts"] - llm = LLM( - model=model, - revision=ref["revision"], - tokenizer_revision=ref["revision"], - runner="pooling", - max_model_len=64, - max_num_seqs=2, + _assert_embeddings_match_hf( + hf_embeddings, + assert_embeddings_close, + example_prompts, + model, enforce_eager=True, + max_num_seqs=2, ) - outputs = llm.embed(prompts) - assert len(outputs) == len(prompts) - - for prompt, out, ref_emb in zip(prompts, outputs, ref["embeddings"]): - emb = out.outputs.embedding - assert len(emb) == len(ref_emb), ( - f"{model}: dim mismatch {len(emb)} vs cached {len(ref_emb)}" - ) - assert all(math.isfinite(x) for x in emb) - sim = _cosine(emb, ref_emb) - assert sim >= COSINE_MIN, ( - f"{model} batched MEAN: cosine {sim:.4f} < {COSINE_MIN} " - f"vs cached HF reference for prompt {prompt!r}" - ) @pytest.mark.uses_subprocess -def test_encoder_embed_last_pooling() -> None: +def test_encoder_embed_last_pooling(assert_embeddings_close) -> None: """SpyreLastPool path: force LAST on granite-125m and match HF last-token. Product encoder models in ``EMBEDDING_MODELS`` are CLS or MEAN only; this override exercises the LAST gather + normalize path that ``configure_pooling_for_spyre`` patches to ``SpyreLastPool``. """ - # Both sides are computed in this run, so the pin buys reproducibility rather than a - # valid comparison; read from the embed refs so it cannot drift off the gated weights. - ref = _REFERENCES.get(LAST_POOLING_MODEL) - if ref is None: - pytest.skip( - f"No HF ref for {LAST_POOLING_MODEL}; run tests/data/generate_encoder_embed_refs.py" - ) - - revision = ref["revision"] + revision = MODEL_REVISIONS[LAST_POOLING_MODEL] prompts = LAST_POOLING_PROMPTS ref_embs = _hf_last_token_embeddings(LAST_POOLING_MODEL, revision, prompts) @@ -232,55 +218,51 @@ def test_encoder_embed_last_pooling() -> None: outputs = llm.embed(prompts) assert len(outputs) == len(prompts) - for prompt, out, ref_emb in zip(prompts, outputs, ref_embs): - emb = out.outputs.embedding - assert len(emb) == len(ref_emb), ( - f"LAST {LAST_POOLING_MODEL}: dim mismatch {len(emb)} vs HF {len(ref_emb)}" - ) - assert all(math.isfinite(x) for x in emb) - sim = _cosine(emb, ref_emb) - assert sim >= COSINE_MIN, ( - f"LAST {LAST_POOLING_MODEL}: cosine {sim:.4f} < {COSINE_MIN} " - f"vs HF last-token for prompt {prompt!r}" - ) + assert_embeddings_close( + f"LAST {LAST_POOLING_MODEL}", + [out.outputs.embedding for out in outputs], + ref_embs, + EMBEDDING_TOL, + ) @pytest.mark.uses_subprocess @pytest.mark.parametrize("model", RERANKER_MODELS) -def test_encoder_rerank_models(model: str) -> None: - """Spyre reranker scores match the cached HF references within tolerance.""" - _assert_rerank_scores_match_refs(model, enforce_eager=True) +def test_encoder_rerank_models(hf_runner, model: str) -> None: + """Spyre reranker scores match live HF within tolerance.""" + _assert_rerank_scores_match_hf(hf_runner, model, enforce_eager=True) @pytest.mark.model_quality @pytest.mark.uses_subprocess @pytest.mark.parametrize("model", RERANKER_MODELS) -def test_encoder_rerank_models_compiled(model: str) -> None: - """Same models and references, compiled rather than eager.""" - _assert_rerank_scores_match_refs(model, enforce_eager=False) +def test_encoder_rerank_models_compiled(hf_runner, model: str) -> None: + """Same models and comparison, compiled rather than eager.""" + _assert_rerank_scores_match_hf(hf_runner, model, enforce_eager=False) + +def _assert_rerank_scores_match_hf(hf_runner, model: str, enforce_eager: bool) -> None: + """Spyre runs the encoder body, not the score: the classifier head stays float32 and + torch-spyre has no FP32 batchmatmul (torch-spyre#1794), so the pooling tail runs on CPU + even in the compiled case.""" + revision = MODEL_REVISIONS[model] -def _assert_rerank_scores_match_refs(model: str, enforce_eager: bool) -> None: - """What runs on Spyre is the encoder body, not the score: the classifier head stays - float32 and torch-spyre has no FP32 batchmatmul (torch-spyre#1794), so the pooling tail - runs on CPU even in the compiled case.""" - ref = _RERANK_REFERENCES.get(model) - if ref is None: - pytest.skip(f"No HF ref for {model}; run tests/data/generate_rerank_score_refs.py") + # fp32 reference: the point of comparison is the fp16 device path against ground truth. + pairs = [[RERANK_QUERY, document] for document in RERANK_DOCUMENTS] + with hf_runner(model, revision=revision, dtype="float32", is_cross_encoder=True) as hf_model: + hf_scores = hf_model.predict(pairs).tolist() - documents = ref["documents"] - ref_scores = ref["scores"] llm = LLM( model=model, - revision=ref["revision"], - tokenizer_revision=ref["revision"], + revision=revision, + tokenizer_revision=revision, runner="pooling", max_model_len=64, max_num_seqs=1, enforce_eager=enforce_eager, ) - outputs = llm.score(ref["query"], documents) - assert len(outputs) == len(documents) + outputs = llm.score(RERANK_QUERY, RERANK_DOCUMENTS) + assert len(outputs) == len(RERANK_DOCUMENTS) scores = [out.outputs.score for out in outputs] assert all(math.isfinite(s) for s in scores), f"{model}: non-finite score in {scores}" @@ -288,17 +270,15 @@ def _assert_rerank_scores_match_refs(model: str, enforce_eager: bool) -> None: # Ranking is checked apart from the per-score bound: all scores can drift the same # direction without reordering, and a pair can swap while both stay inside tolerance. order = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True) - ref_order = sorted(range(len(ref_scores)), key=lambda i: ref_scores[i], reverse=True) - assert order == ref_order, ( - f"{model}: ranked documents {order} vs cached HF {ref_order}; " - f"scores {scores} vs {ref_scores}" + hf_order = sorted(range(len(hf_scores)), key=lambda i: hf_scores[i], reverse=True) + assert order == hf_order, ( + f"{model}: ranked documents {order} vs HF {hf_order}; scores {scores} vs {hf_scores}" ) - for document, score, ref_score in zip(documents, scores, ref_scores, strict=True): - tol = min(SCORE_ABS_TOL, SCORE_REL_TOL * ref_score) - assert abs(score - ref_score) <= tol, ( - f"{model}: score {score:.6f} vs cached HF {ref_score:.6f} (tol {tol:.6f}) " - f"for {document!r}" + for document, score, hf_score in zip(RERANK_DOCUMENTS, scores, hf_scores, strict=True): + tol = min(SCORE_ABS_TOL, SCORE_REL_TOL * hf_score) + assert abs(score - hf_score) <= tol, ( + f"{model}: score {score:.6f} vs HF {hf_score:.6f} (tol {tol:.6f}) for {document!r}" ) diff --git a/tests/e2e/test_model_quality.py b/tests/e2e/test_model_quality.py index 64f8d6a1e..4cb4f7592 100644 --- a/tests/e2e/test_model_quality.py +++ b/tests/e2e/test_model_quality.py @@ -12,22 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Output-quality gate for the product decoder models: compiled Spyre output against a -cached CPU HF reference, comparing token ids and per-token probabilities. +"""Output-quality gate for the product decoder models: compiled Spyre output vs. live HF. -Prompts and references: ``python tests/data/generate_decoder_output_refs.py`` +Modelled on ``tests/models/language/generation/test_hybrid.py::test_models`` from upstream +vLLM, down to using its own ``check_logprobs_close`` as the comparison. """ from __future__ import annotations -import json -import math -import os -from pathlib import Path -from typing import Any +import functools import pytest -from vllm import LLM, RequestOutput, SamplingParams +from vllm import SamplingParams pytestmark = [pytest.mark.model_quality, pytest.mark.uses_subprocess] @@ -39,45 +35,27 @@ "meta-llama/Llama-3.1-8B-Instruct", ] -# Prompts come from the unquantized sibling; the smoke case only needs ones that fit a bucket. -FP8_DECODER_MODELS = { - "ibm-granite/granite-3.3-8b-instruct-FP8": "ibm-granite/granite-3.3-8b-instruct", - "ibm-granite/granite-4.1-8b-fp8": "ibm-granite/granite-4.1-8b", +MODEL_REVISIONS = { + "ibm-granite/granite-3.3-8b-instruct": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b", + "ibm-granite/granite-4.1-8b": "1504002f650e656a0a3789d99574df12e3e94ed0", + "google/gemma-4-31B": "5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89", + "google/gemma-4-26B-A4B": "24548b62aa021d562695c04aaf7758a1ea47990b", + "meta-llama/Llama-3.1-8B-Instruct": "0e9e39f249a16976918f6564b8830bc894c89659", } + +FP8_DECODER_MODELS = [ + "ibm-granite/granite-3.3-8b-instruct-FP8", + "ibm-granite/granite-4.1-8b-fp8", +] FP8_REVISIONS = { "ibm-granite/granite-3.3-8b-instruct-FP8": "4b5990b8d402a75febe0086abbf1e490af494e3d", "ibm-granite/granite-4.1-8b-fp8": "070021b3608433b6107a00733d561c9779b9937e", } -# Nothing is compared, so the run only has to prove decode advances. FP8_MAX_TOKENS = 8 -# fp16 on device reorders accumulation against the fp32 reference, so probabilities are -# compared with a tolerance. Two bounds, because the failure modes are opposites. -# -# The mean is the sensitive bound: drift spread over a prompt shows up here while no single -# step looks unusual. Worst measured over the five gated decoders is 0.006 (granite-3.3). -MEAN_ABS_TOL = float(os.environ.get("SPYRE_TEST_MEAN_ABS_TOL", "0.03")) -# The per-step cap only has to catch gross breakage, so it is deliberately loose: a reference -# near p=0.5 is maximally ill-conditioned (dp/dlogit peaks at p(1-p)), and one graph measured -# 0.115 apart there across two CI pods with every token still exact (PR #723). Worst measured -# step is 0.027 -- 3x under the 0.08 this replaces, so that bound was thin for every model. -ABS_TOL = float(os.environ.get("SPYRE_TEST_ABS_TOL", "0.20")) -# Low-probability steps keep a relative bound; a flat one would permit an arbitrary ratio. -REL_TOL = float(os.environ.get("SPYRE_TEST_REL_TOL", "0.5")) -# A token disagreement is a stronger signal than drift, so judging one as a near-tie keeps the -# original tight bound rather than inheriting ABS_TOL. -TIE_ABS_TOL = float(os.environ.get("SPYRE_TEST_TIE_ABS_TOL", "0.08")) - -# HF's greedy token must be in Spyre's distribution even when Spyre picks another; 20 is -# vLLM's `max_logprobs`. -NUM_LOGPROBS = 20 - -# A near-tie split ends the comparison, so without a floor a case that mispredicts at step 0 -# on every prompt would pass having compared nothing. Counted over the case, not per prompt: -# where the reference itself is a coin flip (granite-4.1's second prompt opens on p=0.4961) -# fp16 drift alone decides the argmax, and one prompt truncating to zero says nothing about -# output quality -- but every prompt truncating still fails. -MIN_MATCHED_FRACTION = 0.5 +MAX_TOKENS = 16 +NUM_LOGPROBS = 5 +HF_DTYPE = "float32" MAX_MODEL_LEN = 256 MAX_NUM_SEQS = 3 @@ -85,129 +63,137 @@ # max_num_batched_tokens down to the largest one, so this is the top of COMPILE_SIZES. MAX_NUM_BATCHED_TOKENS = 64 COMPILE_SIZES = [MAX_NUM_SEQS, MAX_NUM_BATCHED_TOKENS] +# Stated rather than inherited: VllmRunner always passes a block size, so platform.py never +# applies the Spyre default, and VllmRunner's 16 would align up to 64 and resize the KV cache. +BLOCK_SIZE = 128 # Slack for the prompt-fit guard below: it counts with a bare `tokenizer(prompt)` while the # engine tokenizes through vLLM, which can differ by a special token or two. PROMPT_TOKEN_MARGIN = 8 -_REF_PATH = Path(__file__).parent.parent / "data" / "decoder_output_refs.json" -_REFERENCES: dict = json.loads(_REF_PATH.read_text()) if _REF_PATH.exists() else {} + +@functools.cache +def _check_logprobs_close(): + """Upstream's comparison, resolved on first use rather than at import: the tree is a git + clone, and every CI job imports this module during collection even when ``-m`` deselects it. + """ + from spyre_testing_plugin.upstream import ensure_upstream_tests_importable + + ensure_upstream_tests_importable() + from tests.models.utils import check_logprobs_close + + return check_logprobs_close @pytest.mark.parametrize("model", DECODER_MODELS) -def test_decoder_model_output(model: str, monkeypatch: pytest.MonkeyPatch) -> None: - """Compiled Spyre output matches the cached HF reference for `model`.""" - ref = _REFERENCES.get(model) - assert ref is not None, ( - f"No HF reference for {model} in {_REF_PATH.name}; regenerate with " - f"`python tests/data/generate_decoder_output_refs.py --models {model}`" - ) +def test_decoder_model_output( + hf_runner, + vllm_runner, + example_prompts, + monkeypatch: pytest.MonkeyPatch, + model: str, +) -> None: + """Compiled Spyre output matches live HF for ``model``.""" + revision = MODEL_REVISIONS[model] + prompts = example_prompts monkeypatch.setenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "36000") + tokenizer = _pinned_tokenizer(model, revision) + _assert_prompts_fit_prefill_bucket(tokenizer, model, prompts) - prompts = [result["prompt"] for result in ref["results"]] - max_tokens = ref["max_tokens"] - revision = ref["revision"] - - _assert_prompts_fit_prefill_bucket(model, revision, prompts) + # fp32 explicitly: HfRunner's "auto" resolves to CpuPlatform's first supported dtype, + # bfloat16, whose mantissa is shorter than the fp16 device path this is adjudicating. + with hf_runner(model, dtype=HF_DTYPE, revision=revision, processor=tokenizer) as hf_model: + hf_outputs = hf_model.generate_greedy_logprobs_limit(prompts, MAX_TOKENS, NUM_LOGPROBS) - engine = LLM( - model=model, + with vllm_runner( + model, revision=revision, tokenizer_revision=revision, enforce_eager=False, + trust_remote_code=False, + enable_chunked_prefill=None, max_model_len=MAX_MODEL_LEN, max_num_seqs=MAX_NUM_SEQS, max_num_batched_tokens=MAX_NUM_BATCHED_TOKENS, + block_size=BLOCK_SIZE, compilation_config={"compile_sizes": COMPILE_SIZES}, - ) - - outputs = engine.generate( - prompts, - SamplingParams( - temperature=0.0, - max_tokens=max_tokens, - logprobs=NUM_LOGPROBS, - ignore_eos=True, # the reference is a fixed-length run with EOS disabled - ), - use_tqdm=False, - ) - - assert [output.prompt for output in outputs] == prompts, "Model output contained wrong prompt!" - matched = [ - _compare_against_hf(model, hf_result, output) - for hf_result, output in zip(ref["results"], outputs) - ] - per_prompt = ", ".join(f"{n}/{max_tokens}" for n in matched) - print( - f"\n{model}: matched {sum(matched)}/{len(prompts) * max_tokens} reference steps " - f"({per_prompt} per prompt). Prompts that stop after a step or two diverged on a " - f"near-tie and gate little -- see MODEL_PROMPTS in " - f"tests/data/generate_decoder_output_refs.py." - ) - total_steps = len(prompts) * max_tokens - min_matched = math.ceil(MIN_MATCHED_FRACTION * total_steps) - assert sum(matched) >= min_matched, ( - f"{model}: matched {sum(matched)}/{total_steps} reference steps ({per_prompt} per " - f"prompt), under the {min_matched}/{total_steps} floor -- the near-tie splits came too " - f"early to gate anything. Every prompt matched all {max_tokens} steps when the " - f"reference was taken, so treat this as a regression, not as a floor to lower." + ) as spyre_model: + spyre_outputs = spyre_model.generate_greedy_logprobs(prompts, MAX_TOKENS, NUM_LOGPROBS) + + check_logprobs_close = _check_logprobs_close() + check_logprobs_close( + outputs_0_lst=hf_outputs, + outputs_1_lst=spyre_outputs, + name_0="hf", + name_1="spyre", ) @pytest.mark.parametrize("model", FP8_DECODER_MODELS) -def test_fp8_decoder_model_smoke(model: str, monkeypatch: pytest.MonkeyPatch) -> None: - """A compiled FP8 checkpoint loads and decodes; no reference, the generator does not - dequantize compressed-tensors on CPU.""" - base = FP8_DECODER_MODELS[model] - base_ref = _REFERENCES.get(base) - assert base_ref is not None, ( - f"No HF reference for {base} in {_REF_PATH.name}, and {model} borrows its prompts; " - f"regenerate with `python tests/data/generate_decoder_output_refs.py --models {base}`" - ) - - monkeypatch.setenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "36000") - - prompts = [result["prompt"] for result in base_ref["results"]] +def test_fp8_decoder_model_smoke( + vllm_runner, + example_prompts, + monkeypatch: pytest.MonkeyPatch, + model: str, +) -> None: + """A compiled FP8 checkpoint loads and decodes. + + No HF comparison: transformers does not dequantize compressed-tensors on CPU, so there is + nothing to compare against. The unquantized siblings gate the numerics. + """ revision = FP8_REVISIONS[model] + prompts = example_prompts - _assert_prompts_fit_prefill_bucket(model, revision, prompts) + monkeypatch.setenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "36000") + _assert_prompts_fit_prefill_bucket(_pinned_tokenizer(model, revision), model, prompts) - engine = LLM( - model=model, + with vllm_runner( + model, revision=revision, tokenizer_revision=revision, enforce_eager=False, + trust_remote_code=False, + enable_chunked_prefill=None, max_model_len=MAX_MODEL_LEN, max_num_seqs=MAX_NUM_SEQS, max_num_batched_tokens=MAX_NUM_BATCHED_TOKENS, + block_size=BLOCK_SIZE, compilation_config={"compile_sizes": COMPILE_SIZES}, - ) - - outputs = engine.generate( - prompts, - SamplingParams(temperature=0.0, max_tokens=FP8_MAX_TOKENS, ignore_eos=True), - use_tqdm=False, - ) + ) as spyre_model: + # generate_w_logprobs rather than generate_greedy, which prepends the prompt ids + # to every completion and would defeat the token count below. + outputs = spyre_model.generate_w_logprobs( + prompts, + SamplingParams(temperature=0.0, max_tokens=FP8_MAX_TOKENS, ignore_eos=True), + ) - assert [output.prompt for output in outputs] == prompts, "Model output contained wrong prompt!" - for output in outputs: - completion = output.outputs[0] - print(f"\n{model} prompt: {output.prompt!r}\n Spyre: {completion.text!r}") + for prompt, (token_ids, text, _) in zip(prompts, outputs, strict=True): + print(f"\n{model} prompt: {prompt!r}\n Spyre: {text!r}") # Token count only: this case must not assume the tokens decode to non-empty text. - assert len(completion.token_ids) == FP8_MAX_TOKENS, ( - f"{model}: generated {len(completion.token_ids)} of {FP8_MAX_TOKENS} tokens" + assert len(token_ids) == FP8_MAX_TOKENS, ( + f"{model}: generated {len(token_ids)} of {FP8_MAX_TOKENS} tokens" ) -def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[str]) -> None: - """Fail loudly if a prompt outgrew the largest compiled prefill bucket. +def _pinned_tokenizer(model: str, revision: str): + """The model's tokenizer at `revision`, also passed to `HfRunner` as its `processor`. - Past the largest bucket `SpyreShapeBucketer.find_bucket` returns None and the shape runs - unpadded, so an over-long prompt is a silent Dynamo recompile inside generate(). + `HfRunner` forwards `revision` only to the weights, so its own tokenizer would come from + main; passing this one also skips its unconditional `AutoProcessor` load, which for the + multimodal checkpoints pulls an image processor needing torchvision -- absent from the + Spyre torch build -- for a text-only comparison. """ from transformers import AutoTokenizer - tokenizer = AutoTokenizer.from_pretrained(model, revision=revision) + return AutoTokenizer.from_pretrained(model, revision=revision) + + +def _assert_prompts_fit_prefill_bucket(tokenizer, model: str, prompts: list[str]) -> None: + """Fail loudly if a prompt outgrew the largest compiled prefill bucket. + + Past the largest bucket ``SpyreShapeBucketer.find_bucket`` returns None and the shape runs + unpadded, so an over-long prompt is a silent Dynamo recompile inside generate(). + """ limit = MAX_NUM_BATCHED_TOKENS - PROMPT_TOKEN_MARGIN for prompt in prompts: num_tokens = len(tokenizer(prompt).input_ids) @@ -215,92 +201,5 @@ def _assert_prompts_fit_prefill_bucket(model: str, revision: str, prompts: list[ f"{model}: prompt is {num_tokens} tokens, over the {limit}-token bound this " f"guard holds ({PROMPT_TOKEN_MARGIN} below the largest compiled bucket, " f"{MAX_NUM_BATCHED_TOKENS}) -- past the bucket it would recompile at generate() " - f"time. Shorten it, or raise MAX_NUM_BATCHED_TOKENS here and in the generator: " - f"{prompt!r}" - ) - - -def _prob_tol(reference_prob: float) -> float: - """Per-step cap when Spyre and HF picked the same token.""" - return min(ABS_TOL, REL_TOL * reference_prob) - - -def _tie_tol(reference_prob: float) -> float: - """Bound for accepting a token disagreement as a near-tie rather than a regression.""" - return min(TIE_ABS_TOL, REL_TOL * reference_prob) - - -def _assert_mean_prob_error(model: str, prompt: str, diffs: list[float]) -> None: - """The bound that holds quality: drift spread over a prompt fails here well before any - single step reaches ``ABS_TOL``.""" - if not diffs: - return - mean = sum(diffs) / len(diffs) - print(f" prob error over {len(diffs)} compared steps: mean={mean:.4f} max={max(diffs):.4f}") - assert mean <= MEAN_ABS_TOL, ( - f"{model}: mean probability error {mean:.4f} over {len(diffs)} steps exceeds " - f"{MEAN_ABS_TOL:.4f} for prompt {prompt!r} -- the distribution drifted as a whole, " - f"which no single-step bound catches. A regression, not a tolerance to raise." - ) - - -def _compare_against_hf(model: str, hf_result: dict[str, Any], output: RequestOutput) -> int: - completion = output.outputs[0] - token_ids = list(completion.token_ids) - logprobs = [completion.logprobs[i][t].logprob for i, t in enumerate(token_ids)] - - print(f"\n{model} prompt: {hf_result['prompt']!r}") - print(f" HF: {hf_result['text']!r}") - print(f" Spyre: {completion.text!r}") - - assert len(token_ids) == len(hf_result["token_ids"]), ( - f"{model}: generated {len(token_ids)} tokens, reference has {len(hf_result['token_ids'])}" - ) - - diffs: list[float] = [] - for step, (hf_id, hf_logprob, token_id, logprob) in enumerate( - zip(hf_result["token_ids"], hf_result["logprobs"], token_ids, logprobs, strict=True) - ): - hf_prob, prob = math.exp(hf_logprob), math.exp(logprob) - tol = _prob_tol(hf_prob) - detail = ( - f"step {step}: token {token_id} ({completion.logprobs[step][token_id].decoded_token!r}," - f" p={prob:.4f}) vs HF {hf_id} ({hf_result['tokens'][step]!r}, p={hf_prob:.4f})" + f"time. Shorten it, or raise MAX_NUM_BATCHED_TOKENS here: {prompt!r}" ) - - if hf_id != token_id: - # Two equally confident models agree on p(sampled) however far apart they - # picked, so judge the tie on HF's token. - spyre_hf = completion.logprobs[step].get(hf_id) - assert spyre_hf is not None, ( - f"{model}: wrong token and HF's token is outside Spyre's top " - f"{NUM_LOGPROBS}, so the distributions disagree outright, {detail}" - ) - spyre_hf_prob = math.exp(spyre_hf.logprob) - ref_tol = _tie_tol(hf_prob) - assert abs(spyre_hf_prob - hf_prob) <= ref_tol, ( - f"{model}: wrong token and p(HF token) differs by more than {ref_tol:.4f} " - f"(Spyre {spyre_hf_prob:.4f} vs HF {hf_prob:.4f}), {detail}" - ) - # A tie also means Spyre ranks the two level, so a flat HF distribution cannot - # excuse Spyre being confident elsewhere. Doubled: both may drift by `ref_tol`. - tie_tol = 2 * ref_tol - assert abs(prob - spyre_hf_prob) <= tie_tol, ( - f"{model}: wrong token, and Spyre puts it {prob - spyre_hf_prob:.4f} > " - f"{tie_tol:.4f} above HF's token (p={spyre_hf_prob:.4f}), so this is not " - f"a near-tie, {detail}" - ) - print( - f" diverged on a near-tie at {detail}; p(HF token) on Spyre " - f"{spyre_hf_prob:.4f}; not comparing further" - ) - _assert_mean_prob_error(model, hf_result["prompt"], diffs) - return step - - assert abs(hf_prob - prob) <= tol, ( - f"{model}: probability differs by more than {tol:.4f}, {detail}" - ) - diffs.append(abs(hf_prob - prob)) - - _assert_mean_prob_error(model, hf_result["prompt"], diffs) - return len(token_ids) diff --git a/tests/plugin/spyre_testing_plugin/pytest_plugin.py b/tests/plugin/spyre_testing_plugin/pytest_plugin.py index c487434a2..031994db5 100644 --- a/tests/plugin/spyre_testing_plugin/pytest_plugin.py +++ b/tests/plugin/spyre_testing_plugin/pytest_plugin.py @@ -60,13 +60,10 @@ import fnmatch import os -import re import socket import subprocess import sys import tempfile -import time -import tomllib from pathlib import Path import pytest @@ -89,6 +86,10 @@ UpstreamTestConfig, ) from spyre_testing_plugin.tags import result_tags +from spyre_testing_plugin.upstream import ( + apply_temp_upstream_code_edits, + prepare_upstream_tests_dir, +) from spyre_testing_plugin.vfio_reaper import ( reap_vfio_holders, spyre_hardware_present, @@ -241,206 +242,6 @@ def _get_paths_from_yaml() -> str: return ",".join(paths) -# --------------------------------------------------------------------------- -# vLLM Repository Cloning -# --------------------------------------------------------------------------- - - -def _cache_root() -> Path: - """ - Cache directory for cloned tests (persists across runs) - """ - # Respect XDG if present, fallback to ~/.cache - xdg = os.environ.get("XDG_CACHE_HOME") - base = Path(xdg) if xdg else Path.home() / ".cache" - return base / "vllm-upstream-tests" - - -def _extract_vllm_commit_from_pyproject(repo_root_dir: Path) -> str: - """ - Extract the vLLM git reference from pyproject.toml [tool.uv.sources] section. - Raises FileNotFoundError if pyproject.toml is missing, or KeyError - if the expected source entry is not found. - """ - pyproject_path = repo_root_dir / "pyproject.toml" - if not pyproject_path.exists(): - raise FileNotFoundError(f"pyproject.toml not found in {repo_root_dir}") - - with open(pyproject_path, "rb") as f: - data = tomllib.load(f) - - try: - vllm_source = data["tool"]["uv"]["sources"]["vllm"] - except KeyError as e: - raise KeyError( - "Ensure vllm is specified with 'rev' in pyproject.toml" - f" [tool.uv.sources]: missing key {e}" - ) from e - - # Handle both a single source dict and a list of sources (e.g. index + git fallback) - if isinstance(vllm_source, list): - for source in vllm_source: - if isinstance(source, dict) and "git" in source and "rev" in source: - return source["rev"] - elif isinstance(vllm_source, dict) and "git" in vllm_source and "rev" in vllm_source: - return vllm_source["rev"] - - raise KeyError("Ensure vllm is specified with 'rev' in pyproject.toml [tool.uv.sources]") - - -def _resolve_vllm_commit(repo_root_dir: Path) -> str: - """ - Resolve the vLLM git reference to use for cloning upstream tests. - Priority: VLLM_COMMIT env var > pyproject.toml > error - """ - # Allow env var override for testing/CI - env_commit = os.environ.get("VLLM_COMMIT", "").strip() - if env_commit: - if not re.match(r"^(?:[0-9a-f]{7,40}|v\d+\.\d+\.\d+(?:-[a-zA-Z0-9.]+)?)$", env_commit): - raise ValueError(f"Invalid VLLM_COMMIT format: {env_commit}") - return env_commit - - # Extract from pyproject.toml - return _extract_vllm_commit_from_pyproject(repo_root_dir) - - -def _run(cmd: list[str], cwd: Path | None = None, max_retries: int = 3) -> None: - """Run command with optional retries for network operations.""" - for attempt in range(max_retries): - try: - subprocess.run(cmd, cwd=str(cwd) if cwd else None, check=True) - return - except subprocess.CalledProcessError: - if attempt < max_retries - 1: - time.sleep(2**attempt) # Exponential backoff: 1s, 2s, 4s - else: - raise - - -def _ensure_repo_at_commit(repo_dir: Path, url: str, commit: str, sparse_paths: list[str]) -> Path: - """ - Ensure repo cloned at 'repo_dir/commit' with sparse checkout of 'sparse_paths'. - Returns the path to the working tree at that commit. - """ - # We create a separate worktree per commit to allow co-existence of different commits - base_dir = repo_dir - base_dir.mkdir(parents=True, exist_ok=True) - git_dir = base_dir / "repo.git" - - if not git_dir.exists(): - _run(["git", "init", "--bare", str(git_dir)]) - - # Prepare a worktree dir per commit - wt_dir = base_dir / f"worktree-{commit[:12]}" - if wt_dir.exists(): - _log(f"[vllm-upstream] Using cached worktree at {wt_dir}") - return wt_dir - - # Create temp dir to set up the sparse worktree then move into place atomically - with tempfile.TemporaryDirectory(dir=str(base_dir)) as td: - td_path = Path(td) - - # Ensure origin remote exists and points to the correct URL - result = subprocess.run( - ["git", "--git-dir", str(git_dir), "remote", "get-url", "origin"], - capture_output=True, - text=True, - ) - if result.returncode != 0: - # Origin doesn't exist - add it - _run(["git", "--git-dir", str(git_dir), "remote", "add", "origin", url]) - elif result.stdout.strip() != url: - # Origin exists but points to different URL - update it - _log(f"[vllm-upstream] Updating origin URL: {result.stdout.strip()} -> {url}") - _run(["git", "--git-dir", str(git_dir), "remote", "set-url", "origin", url]) - - # Determine if commit is a tag (starts with 'v' and matches semver pattern) or a SHA - is_tag = re.match(r"^v\d+\.\d+\.\d+(?:-[a-zA-Z0-9.]+)?$", commit) - - if is_tag: - _log(f"[vllm-upstream] Fetching tag {commit} from {url}") - # For tags, fetch the tag reference - _run( - [ - "git", - "--git-dir", - str(git_dir), - "fetch", - "--depth=1", - "origin", - f"refs/tags/{commit}:refs/tags/{commit}", - ] - ) - else: - _log(f"[vllm-upstream] Fetching commit {commit[:12]} from {url}") - # For commit SHAs, fetch the commit directly - _run(["git", "--git-dir", str(git_dir), "fetch", "--depth=1", "origin", commit]) - - # Create a new worktree at temp - # For tags, use the full tag reference; for commits, use the commit SHA directly - worktree_ref = f"refs/tags/{commit}" if is_tag else commit - _run( - [ - "git", - "--git-dir", - str(git_dir), - "worktree", - "add", - "--detach", - str(td_path), - worktree_ref, - ] - ) - - # Enable sparse checkout at the worktree - _run(["git", "sparse-checkout", "init", "--cone"], cwd=td_path) - _run(["git", "sparse-checkout", "set", *sparse_paths], cwd=td_path) - - # Ensure we're exactly at the commit (detached HEAD) - _run(["git", "checkout", "--detach", commit], cwd=td_path) - - # Atomically move into place - td_path.rename(wt_dir) - - return wt_dir - - -def _prepare_upstream_tests_dir(repo_root_dir: Path) -> Path: - """Clone vLLM to cache and return path to tests directory.""" - commit = _resolve_vllm_commit(repo_root_dir) - cache_root = _cache_root() - wt_dir = _ensure_repo_at_commit( - repo_dir=cache_root, - url=os.environ.get("VLLM_REPO_URL", "https://github.com/vllm-project/vllm"), - commit=commit, - sparse_paths=["tests"], - ) - tests_dir = wt_dir / "tests" - if not tests_dir.is_dir(): - raise RuntimeError(f"Upstream tests directory not found at {tests_dir}") - return tests_dir - - -def _temp_upstream_code_edits(upstream_tests_dir: Path): - """Apply small code edits to the upstream tests directory before importing. - - These should be _temporary_ edits to source code for vllm tests while we work to make them more - portable. This should only be used where mocking is not possible or too cumbersome. - """ - - # Mocking out torch.device seems impossible to do (at least multiple rounds of Bob and Claude - # were unsuccessful). So we patch the source code to change the hardcoded - # `torch.device("cuda:0")` to `torch.device("cpu")`. - hardcoded_cuda_test_path = ( - upstream_tests_dir / "v1" / "attention" / "test_attention_backends.py" - ) - with open(hardcoded_cuda_test_path) as f: - content = f.read() - content = content.replace('torch.device("cuda:0")', 'torch.device("cpu")') - with open(hardcoded_cuda_test_path, "w") as f: - f.write(content) - - # --------------------------------------------------------------------------- # Upstream Opt-In # --------------------------------------------------------------------------- @@ -557,8 +358,8 @@ def pytest_configure(config): else: try: # Clone vLLM to cache - upstream_tests_base = _prepare_upstream_tests_dir(repo_root) - _temp_upstream_code_edits(upstream_tests_base) + upstream_tests_base = prepare_upstream_tests_dir(repo_root, log=_log) + apply_temp_upstream_code_edits(upstream_tests_base) config._upstream_tests_base = upstream_tests_base # Determine which test paths to inject diff --git a/tests/plugin/spyre_testing_plugin/upstream.py b/tests/plugin/spyre_testing_plugin/upstream.py new file mode 100644 index 000000000..29b0bfbe8 --- /dev/null +++ b/tests/plugin/spyre_testing_plugin/upstream.py @@ -0,0 +1,269 @@ +# Copyright 2026 The Spyre-Inference Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Materialize the upstream vLLM ``tests/`` tree at the pinned commit. + +Two consumers: + +* ``pytest_plugin``, which collects upstream tests out of the tree (opt-in, see the + marker gate there). +* Local tests, which import upstream test helpers -- ``check_logprobs_close``, + ``check_embeddings_close``, ``HfRunner``, ``VllmRunner`` -- rather than reimplementing + them. Those live in vLLM's ``tests/`` tree, not the installed wheel, so reaching them + needs the clone even when no upstream test is collected. + +Deliberately free of pytest and vllm imports so a plain script can call it. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +import tempfile +import time +import tomllib +from collections.abc import Callable +from pathlib import Path + +Log = Callable[[str], None] + + +def _stderr_log(msg: str) -> None: + print(msg, file=sys.stderr) + + +def cache_root() -> Path: + """ + Cache directory for cloned tests (persists across runs) + """ + # Respect XDG if present, fallback to ~/.cache + xdg = os.environ.get("XDG_CACHE_HOME") + base = Path(xdg) if xdg else Path.home() / ".cache" + return base / "vllm-upstream-tests" + + +def repo_root() -> Path: + """This repo's root, derived from the installed-editable plugin at + ``/tests/plugin/spyre_testing_plugin/``.""" + return Path(__file__).resolve().parents[3] + + +def _extract_vllm_commit_from_pyproject(repo_root_dir: Path) -> str: + """ + Extract the vLLM git reference from pyproject.toml [tool.uv.sources] section. + Raises FileNotFoundError if pyproject.toml is missing, or KeyError + if the expected source entry is not found. + """ + pyproject_path = repo_root_dir / "pyproject.toml" + if not pyproject_path.exists(): + raise FileNotFoundError(f"pyproject.toml not found in {repo_root_dir}") + + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + + try: + vllm_source = data["tool"]["uv"]["sources"]["vllm"] + except KeyError as e: + raise KeyError( + "Ensure vllm is specified with 'rev' in pyproject.toml" + f" [tool.uv.sources]: missing key {e}" + ) from e + + # Handle both a single source dict and a list of sources (e.g. index + git fallback) + if isinstance(vllm_source, list): + for source in vllm_source: + if isinstance(source, dict) and "git" in source and "rev" in source: + return source["rev"] + elif isinstance(vllm_source, dict) and "git" in vllm_source and "rev" in vllm_source: + return vllm_source["rev"] + + raise KeyError("Ensure vllm is specified with 'rev' in pyproject.toml [tool.uv.sources]") + + +def resolve_vllm_commit(repo_root_dir: Path) -> str: + """ + Resolve the vLLM git reference to use for cloning upstream tests. + Priority: VLLM_COMMIT env var > pyproject.toml > error + """ + # Allow env var override for testing/CI + env_commit = os.environ.get("VLLM_COMMIT", "").strip() + if env_commit: + if not re.match(r"^(?:[0-9a-f]{7,40}|v\d+\.\d+\.\d+(?:-[a-zA-Z0-9.]+)?)$", env_commit): + raise ValueError(f"Invalid VLLM_COMMIT format: {env_commit}") + return env_commit + + # Extract from pyproject.toml + return _extract_vllm_commit_from_pyproject(repo_root_dir) + + +def _run(cmd: list[str], cwd: Path | None = None, max_retries: int = 3) -> None: + """Run command with optional retries for network operations.""" + for attempt in range(max_retries): + try: + subprocess.run(cmd, cwd=str(cwd) if cwd else None, check=True) + return + except subprocess.CalledProcessError: + if attempt < max_retries - 1: + time.sleep(2**attempt) # Exponential backoff: 1s, 2s, 4s + else: + raise + + +def ensure_repo_at_commit( + repo_dir: Path, + url: str, + commit: str, + sparse_paths: list[str], + log: Log = _stderr_log, +) -> Path: + """ + Ensure repo cloned at 'repo_dir/commit' with sparse checkout of 'sparse_paths'. + Returns the path to the working tree at that commit. + """ + # We create a separate worktree per commit to allow co-existence of different commits + base_dir = repo_dir + base_dir.mkdir(parents=True, exist_ok=True) + git_dir = base_dir / "repo.git" + + if not git_dir.exists(): + _run(["git", "init", "--bare", str(git_dir)]) + + # Prepare a worktree dir per commit + wt_dir = base_dir / f"worktree-{commit[:12]}" + if wt_dir.exists(): + log(f"[vllm-upstream] Using cached worktree at {wt_dir}") + return wt_dir + + # Create temp dir to set up the sparse worktree then move into place atomically + with tempfile.TemporaryDirectory(dir=str(base_dir)) as td: + td_path = Path(td) + + # Ensure origin remote exists and points to the correct URL + result = subprocess.run( + ["git", "--git-dir", str(git_dir), "remote", "get-url", "origin"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + # Origin doesn't exist - add it + _run(["git", "--git-dir", str(git_dir), "remote", "add", "origin", url]) + elif result.stdout.strip() != url: + # Origin exists but points to different URL - update it + log(f"[vllm-upstream] Updating origin URL: {result.stdout.strip()} -> {url}") + _run(["git", "--git-dir", str(git_dir), "remote", "set-url", "origin", url]) + + # Determine if commit is a tag (starts with 'v' and matches semver pattern) or a SHA + is_tag = re.match(r"^v\d+\.\d+\.\d+(?:-[a-zA-Z0-9.]+)?$", commit) + + if is_tag: + log(f"[vllm-upstream] Fetching tag {commit} from {url}") + # For tags, fetch the tag reference + _run( + [ + "git", + "--git-dir", + str(git_dir), + "fetch", + "--depth=1", + "origin", + f"refs/tags/{commit}:refs/tags/{commit}", + ] + ) + else: + log(f"[vllm-upstream] Fetching commit {commit[:12]} from {url}") + # For commit SHAs, fetch the commit directly + _run(["git", "--git-dir", str(git_dir), "fetch", "--depth=1", "origin", commit]) + + # Create a new worktree at temp + # For tags, use the full tag reference; for commits, use the commit SHA directly + worktree_ref = f"refs/tags/{commit}" if is_tag else commit + _run( + [ + "git", + "--git-dir", + str(git_dir), + "worktree", + "add", + "--detach", + str(td_path), + worktree_ref, + ] + ) + + # Enable sparse checkout at the worktree + _run(["git", "sparse-checkout", "init", "--cone"], cwd=td_path) + _run(["git", "sparse-checkout", "set", *sparse_paths], cwd=td_path) + + # Ensure we're exactly at the commit (detached HEAD) + _run(["git", "checkout", "--detach", commit], cwd=td_path) + + # Atomically move into place + td_path.rename(wt_dir) + + return wt_dir + + +def prepare_upstream_tests_dir(repo_root_dir: Path, log: Log = _stderr_log) -> Path: + """Clone vLLM to cache and return path to tests directory.""" + commit = resolve_vllm_commit(repo_root_dir) + wt_dir = ensure_repo_at_commit( + repo_dir=cache_root(), + url=os.environ.get("VLLM_REPO_URL", "https://github.com/vllm-project/vllm"), + commit=commit, + sparse_paths=["tests"], + log=log, + ) + tests_dir = wt_dir / "tests" + if not tests_dir.is_dir(): + raise RuntimeError(f"Upstream tests directory not found at {tests_dir}") + return tests_dir + + +def apply_temp_upstream_code_edits(upstream_tests_dir: Path) -> None: + """Apply small code edits to the upstream tests directory before importing. + + These should be _temporary_ edits to source code for vllm tests while we work to make them more + portable. This should only be used where mocking is not possible or too cumbersome. + """ + + # Mocking out torch.device seems impossible to do (at least multiple rounds of Bob and Claude + # were unsuccessful). So we patch the source code to change the hardcoded + # `torch.device("cuda:0")` to `torch.device("cpu")`. + hardcoded_cuda_test_path = ( + upstream_tests_dir / "v1" / "attention" / "test_attention_backends.py" + ) + with open(hardcoded_cuda_test_path) as f: + content = f.read() + content = content.replace('torch.device("cuda:0")', 'torch.device("cpu")') + with open(hardcoded_cuda_test_path, "w") as f: + f.write(content) + + +def ensure_upstream_tests_importable(log: Log = _stderr_log) -> Path: + """Make the pinned vLLM ``tests`` tree importable, returning its path. + + Upstream's test modules import each other absolutely (``from tests.models.utils + import ...``), so the tree has to own the top-level name ``tests``. It does: upstream's + ``tests/`` is a regular package, and a regular package wins over this repo's + ``__init__.py``-less namespace directory of the same name whatever the sys.path order -- + so nothing local may import ``tests.*`` (sibling test modules import each other as + top-level modules instead, since none of those directories is a package either). + """ + tests_dir = prepare_upstream_tests_dir(repo_root(), log=log) + upstream_root = str(tests_dir.parent) + if upstream_root not in sys.path: + sys.path.append(upstream_root) + return tests_dir