diff --git a/docs/notebooks/TruthTorchLM_demo.ipynb b/docs/notebooks/TruthTorchLM_demo.ipynb new file mode 100644 index 00000000..2da55287 --- /dev/null +++ b/docs/notebooks/TruthTorchLM_demo.ipynb @@ -0,0 +1,393 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "# TruthTorchLM — Truthfulness Checks and Long‑form Generation Demo\n", + "\n", + "This notebook demonstrates two things with the `TruthTorchLM` package:\n", + "\n", + "1. Multiple‑LLM truthfulness checking for factual claims.\n", + "2. Long‑form generation with a truth value/score.\n", + "\n", + "Notes:\n", + "- You said the package is already installed. If not, uncomment the install cell below.\n", + "- You will likely need API keys for whichever LLM backends the library uses (e.g., OpenAI, Anthropic, etc.). Set them in environment variables before running.\n", + "- Function names in the package may differ slightly. The notebook includes light introspection to locate the correct functions if they exist with a close name.\n" + ], + "id": "2ba1f40179361b70" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# If the package isn't installed in your environment, uncomment and run:\n", + "#!pip install -U TruthTorchLM\n" + ], + "id": "4f2eefa20af5c2c1", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Imports and package discovery\n", + "id": "799d20520d95d57d" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "import importlib, importlib.util, inspect, os, re, types, json, textwrap\n", + "\n", + "# Keep discovery simple and targeted to the intended package names\n", + "pkg_candidates = [\n", + " 'TruthTorchLM',\n", + " 'truthtorchlm',\n", + "]\n", + "\n", + "ttlm = None\n", + "PKG_NAME = None\n", + "for name in pkg_candidates:\n", + " try:\n", + " if importlib.util.find_spec(name) is not None:\n", + " ttlm = importlib.import_module(name)\n", + " PKG_NAME = name\n", + " break\n", + " except Exception:\n", + " pass\n", + "\n", + "if ttlm is None:\n", + " raise ImportError(\n", + " \"TruthTorchLM package not found in this kernel.\\n\"\n", + " \"Try installing it in this exact kernel/env:\\n\"\n", + " \" pip install -U TruthTorchLM\\n\"\n", + " \"Or directly from GitHub if needed:\\n\"\n", + " \" pip install -U git+https://github.com/Ybakman/TruthTorchLM\\n\"\n", + " \"Also ensure your Jupyter kernel uses the same virtualenv where you installed it.\"\n", + " )\n", + "\n", + "print(f'Using package: {PKG_NAME}')\n", + "attrs = [a for a in dir(ttlm) if not a.startswith('_')]\n", + "print('Available attributes (filtered):')\n", + "print([a for a in attrs if re.search(r'(truth|long|gen|check|verify)', a, re.I)])\n", + "\n", + "# Try to get a version if exposed\n", + "version = getattr(ttlm, '__version__', None) or getattr(getattr(ttlm, 'version', None) or types.SimpleNamespace(), '__version__', None)\n", + "print('TruthTorchLM version:', version)\n" + ], + "id": "9b83b23715b4f0c1" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Configure LLM providers (environment variables)\n", + "\n", + "Set API keys as environment variables before running, for example:\n", + "\n", + "- `OPENAI_API_KEY`\n", + "- `ANTHROPIC_API_KEY`\n", + "- `GOOGLE_API_KEY` (Gemini)\n", + "\n", + "Only the providers supported by `TruthTorchLM` are needed. If the library exposes its own client configuration, feel free to adapt this cell accordingly.\n" + ], + "id": "769347ee58028580" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "providers_present = {\n", + " 'OPENAI_API_KEY': bool(os.getenv('OPENAI_API_KEY')),\n", + " 'ANTHROPIC_API_KEY': bool(os.getenv('ANTHROPIC_API_KEY')),\n", + " 'GOOGLE_API_KEY': bool(os.getenv('GOOGLE_API_KEY')),\n", + "}\n", + "print('Provider keys present:', providers_present)\n", + "CAN_CALL = any(providers_present.values())\n", + "\n", + "if not CAN_CALL:\n", + " print(textwrap.dedent('''\n", + " No provider API keys detected. The demo will skip network calls.\n", + " To enable full execution, set one or more of the following environment variables and re-run this notebook:\n", + " - OPENAI_API_KEY\n", + " - ANTHROPIC_API_KEY\n", + " - GOOGLE_API_KEY\n", + " '''))\n" + ], + "id": "250f9758df58ea4c" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Helper: locate TruthTorchLM functions by name\n", + "\n", + "The notebook tries to find closely named functions if the exact names differ.\n" + ], + "id": "83f73cf983b9e16a" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-29T05:46:13.341191Z", + "start_time": "2025-10-29T05:46:13.337833Z" + } + }, + "cell_type": "code", + "source": [ + "from typing import Callable, Optional, Tuple, Iterable\n", + "\n", + "def find_callable(module, candidates: Iterable[str]) -> Tuple[Optional[str], Optional[Callable]]:\n", + " # Exact match first\n", + " for name in candidates:\n", + " fn = getattr(module, name, None)\n", + " if callable(fn):\n", + " return name, fn\n", + " # Fuzzy search fallbacks\n", + " lowered = [c.lower() for c in candidates]\n", + " for a in dir(module):\n", + " al = a.lower()\n", + " if any(c in al for c in lowered):\n", + " obj = getattr(module, a)\n", + " if callable(obj):\n", + " return a, obj\n", + " return None, None\n", + "\n", + "def describe_signature(fn):\n", + " try:\n", + " return str(inspect.signature(fn))\n", + " except Exception:\n", + " return '(unknown signature)'\n" + ], + "id": "a0ea179f462330e3", + "outputs": [], + "execution_count": 6 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## 1) Multiple‑LLM truthfulness check\n", + "id": "c224e957383e16f4" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-29T05:47:17.402385Z", + "start_time": "2025-10-29T05:47:17.394053Z" + } + }, + "cell_type": "code", + "source": [ + "# Candidate function names for multi‑LLM truthfulness checking\n", + "multi_truth_candidates = [\n", + " 'multiple_llm_truthfulness_check',\n", + " 'multi_llm_truthfulness_check',\n", + " 'multiple_llm_based_truthfulness_check',\n", + " 'truthfulness_check_multiple_llm',\n", + " 'truthfulness_across_llms',\n", + " 'check_truthfulness_multi_llm',\n", + " 'check_claim_truth_multi_llm',\n", + " 'check_truthfulness_across_models',\n", + "]\n", + "\n", + "multi_truth_name, multi_truth_fn = find_callable(ttlm, multi_truth_candidates)\n", + "if not multi_truth_fn:\n", + " print('Could not find a multi‑LLM truthfulness function by the expected names.')\n", + " print('Available attributes that look relevant:')\n", + " possible = [a for a in dir(ttlm) if re.search(r'truth|check|verify', a, re.I)]\n", + " print(possible)\n", + "else:\n", + " print(f'Using function: {multi_truth_name}{describe_signature(multi_truth_fn)}')\n", + "\n", + "# Example claims to evaluate\n", + "claims = [\n", + " 'The Eiffel Tower is in Berlin.', # false\n", + " 'Water boils at 100 degrees Celsius at standard atmospheric pressure.', # true\n", + "]\n", + "\n", + "# Example list of model names to consult (adjust to what's supported in your environment)\n", + "llm_models = [\n", + " 'gpt-4o-mini',\n", + " 'gpt-4o',\n", + "]\n", + "\n", + "results_multi = []\n", + "if multi_truth_fn and CAN_CALL:\n", + " for claim in claims:\n", + " # Try to adaptively call the function based on its signature\n", + " sig = None\n", + " try:\n", + " sig = inspect.signature(multi_truth_fn)\n", + " except Exception:\n", + " pass\n", + " kwargs = {}\n", + " if sig:\n", + " for pname in sig.parameters:\n", + " p = pname.lower()\n", + " if 'claim' in p or 'statement' in p or 'text' in p or 'prompt' in p or 'question' in p:\n", + " kwargs[pname] = claim\n", + " elif 'llm' in p or 'model' in p:\n", + " kwargs[pname] = llm_models\n", + " elif 'provider' in p:\n", + " # provide empty or inferred providers list if needed\n", + " kwargs[pname] = None\n", + " try:\n", + " res = multi_truth_fn(**kwargs) if kwargs else multi_truth_fn(claim)\n", + " results_multi.append({'claim': claim, 'result': res})\n", + " except TypeError as te:\n", + " print('Signature mismatch while calling', multi_truth_name, 'with kwargs', kwargs)\n", + " print(te)\n", + " except Exception as e:\n", + " print('Error calling', multi_truth_name, '->', e)\n", + "else:\n", + " if not CAN_CALL:\n", + " print('Skipping multi‑LLM truthfulness checks due to missing provider API keys.')\n" + ], + "id": "5fdd093a4471f8b3", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Could not find a multi‑LLM truthfulness function by the expected names.\n", + "Available attributes that look relevant:\n", + "['GOOGLE_CHECK_QUERY_SYSTEM_PROMPT', 'GOOGLE_CHECK_QUERY_USER_PROMPT', 'GOOGLE_CHECK_VERIFICATION_SYSTEM_PROMPT', 'GOOGLE_CHECK_VERIFICATION_USER_PROMPT', 'TruthMethod', 'calibrate_truth_method', 'evaluate_truth_method', 'generate_with_truth_value', 'truth_methods']\n", + "Skipping multi‑LLM truthfulness checks due to missing provider API keys.\n" + ] + } + ], + "execution_count": 7 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## 2) Long‑form generation with truth value/score\n", + "\n", + "Some variants name this function `long_form_generation_with_truth_value`. The notebook will look for that or similarly named functions.\n" + ], + "id": "25d221230be9e555" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "long_form_candidates = [\n", + " 'long_form_generation_with_truth_value',\n", + " 'long_form_generation_with_truthfulness',\n", + " 'generate_long_form_with_truth',\n", + " 'long_form_generate_with_truth_value',\n", + " 'longform_generation_with_truth_value',\n", + "]\n", + "\n", + "long_form_name, long_form_fn = find_callable(ttlm, long_form_candidates)\n", + "if not long_form_fn:\n", + " print('Could not find the long‑form generation with truth value function by the expected names.')\n", + " print('Available attributes that look relevant:')\n", + " possible = [a for a in dir(ttlm) if re.search(r'long|truth|gen|compose|article', a, re.I)]\n", + " print(possible)\n", + "else:\n", + " print(f'Using function: {long_form_name}{describe_signature(long_form_fn)}')\n", + "\n", + "prompt = (\n", + " 'Explain the history of the Eiffel Tower in about 150-250 words, and include factual references. '\n", + " 'Return both the generated text and a truthfulness score if supported.'\n", + ")\n", + "\n", + "long_form_output = None\n", + "if long_form_fn and CAN_CALL:\n", + " sig = None\n", + " try:\n", + " sig = inspect.signature(long_form_fn)\n", + " except Exception:\n", + " pass\n", + " kwargs = {}\n", + " if sig:\n", + " for pname in sig.parameters:\n", + " p = pname.lower()\n", + " if 'prompt' in p or 'question' in p or 'topic' in p or 'query' in p or 'instruction' in p or 'text' in p:\n", + " kwargs[pname] = prompt\n", + " elif 'llm' in p or 'model' in p:\n", + " kwargs[pname] = 'gpt-4o-mini'\n", + " elif 'max_tokens' in p or 'length' in p or 'words' in p:\n", + " kwargs[pname] = 300\n", + " elif 'return' in p and 'score' in p:\n", + " kwargs[pname] = True\n", + " try:\n", + " long_form_output = long_form_fn(**kwargs) if kwargs else long_form_fn(prompt)\n", + " except TypeError as te:\n", + " print('Signature mismatch while calling', long_form_name, 'with kwargs', kwargs)\n", + " print(te)\n", + " except Exception as e:\n", + " print('Error calling', long_form_name, '->', e)\n", + "else:\n", + " if not CAN_CALL:\n", + " print('Skipping long‑form generation due to missing provider API keys.')\n" + ], + "id": "7c7a436a12a76820", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Pretty‑print results (if structures are returned)\n", + "id": "8f61c2c27b7ebe0a" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "\n", + "def to_jsonable(obj):\n", + " try:\n", + " json.dumps(obj)\n", + " return obj\n", + " except Exception:\n", + " if isinstance(obj, dict):\n", + " return {k: to_jsonable(v) for k, v in obj.items()}\n", + " elif isinstance(obj, (list, tuple, set)):\n", + " return [to_jsonable(x) for x in obj]\n", + " else:\n", + " return str(obj)\n", + "\n", + "if 'results_multi' in globals() and results_multi:\n", + " print('Multi‑LLM truthfulness results:')\n", + " print(json.dumps(to_jsonable(results_multi), indent=2))\n", + "\n", + "if 'long_form_output' in globals() and long_form_output is not None:\n", + " print('\\nLong‑form generation output:')\n", + " print(json.dumps(to_jsonable(long_form_output), indent=2))\n" + ], + "id": "e551b5a75e0d622f", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Tips\n", + "- If your package exposes classes instead of top‑level functions (e.g., a `TruthTorch` or `Verifier` class), instantiate it and re‑run the introspection for methods on that instance.\n", + "- If a function requires specific provider objects rather than model names, consult the library README and pass those objects instead of strings.\n", + "- For reproducibility, consider setting a seed if the library offers one.\n" + ], + "id": "1204cc2c1024c02b" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/notebooks/TruthTorchLM_quickstart.ipynb b/docs/notebooks/TruthTorchLM_quickstart.ipynb new file mode 100644 index 00000000..0f3bb1f6 --- /dev/null +++ b/docs/notebooks/TruthTorchLM_quickstart.ipynb @@ -0,0 +1,353 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "# TruthTorchLM — Quickstart: simple method tests and examples\n", + "\n", + "This minimal notebook shows how to:\n", + "\n", + "- Run a multiple‑LLM truthfulness check on 1–2 claims\n", + "- Generate a short long‑form answer with an optional truth/score\n", + "\n", + "Prereqs:\n", + "- Install the package in this kernel/env (uncomment if needed):\n", + " - `%pip install -U TruthTorchLM`\n", + " - or `%pip install -U git+https://github.com/Ybakman/TruthTorchLM`\n", + "- Set at least one provider API key supported by your setup:\n", + " - `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or `GOOGLE_API_KEY`\n" + ], + "id": "9090fca4ccc0fae9" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "# Quick environment diagnostics (helps ensure the correct venv/kernel)\n", + "import sys, os, importlib.util\n", + "print('Python:', sys.version)\n", + "print('Kernel executable:', sys.executable)\n", + "print('Has TruthTorchLM?', bool(importlib.util.find_spec('TruthTorchLM')))\n", + "print('Has truthtorchlm?', bool(importlib.util.find_spec('truthtorchlm')))\n", + "print('OPENAI_API_KEY set?', bool(os.getenv('OPENAI_API_KEY')))\n", + "print('ANTHROPIC_API_KEY set?', bool(os.getenv('ANTHROPIC_API_KEY')))\n", + "print('GOOGLE_API_KEY set?', bool(os.getenv('GOOGLE_API_KEY')))\n" + ], + "id": "88f3a3ea532456eb" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Import package (simple)\n", + "id": "50687417dbbe730d" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "import importlib, importlib.util, types\n", + "\n", + "def import_tt():\n", + " for name in ('TruthTorchLM', 'truthtorchlm'):\n", + " if importlib.util.find_spec(name) is not None:\n", + " mod = importlib.import_module(name)\n", + " return name, mod\n", + " raise ImportError(\n", + " 'TruthTorchLM not found in this kernel.\\n'\n", + " 'Install into THIS kernel or switch kernel, e.g.:\\n'\n", + " ' %pip install -U TruthTorchLM\\n'\n", + " 'Or from GitHub:\\n'\n", + " ' %pip install -U git+https://github.com/Ybakman/TruthTorchLM\\n'\n", + " 'Then restart the kernel.'\n", + " )\n", + "\n", + "PKG_NAME, ttlm = import_tt()\n", + "version = getattr(ttlm, '__version__', None) or getattr(getattr(ttlm, 'version', None) or types.SimpleNamespace(), '__version__', None)\n", + "print(f'Using package: {PKG_NAME}, version: {version}')\n" + ], + "id": "c3839aa15cd4069e" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Provider keys check (network calls will be skipped if none present)\n", + "id": "19af9b2f4527ca9b" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "import textwrap, json\n", + "CAN_CALL = any([\n", + " bool(os.getenv('OPENAI_API_KEY')),\n", + " bool(os.getenv('ANTHROPIC_API_KEY')),\n", + " bool(os.getenv('GOOGLE_API_KEY')),\n", + "])\n", + "if not CAN_CALL:\n", + " print(textwrap.dedent('''\n", + " No provider API keys detected — examples will be skipped.\n", + " Set one of: OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY and re‑run.\n", + " '''))\n" + ], + "id": "cd28d7de65f26369" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Helpers: find functions and extract scores\n", + "id": "6e5e1588bbc1d390" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "import inspect, re\n", + "from typing import Callable, Iterable, Optional, Tuple, Any\n", + "\n", + "def find_callable(module, candidates: Iterable[str]) -> Tuple[Optional[str], Optional[Callable]]:\n", + " # Exact match only (keep it simple)\n", + " for name in candidates:\n", + " fn = getattr(module, name, None)\n", + " if callable(fn):\n", + " return name, fn\n", + " # If none exact, show a short hint and return None\n", + " looks = [a for a in dir(module) if re.search(r'(truth|long|gen|check|verify)', a, re.I)]\n", + " print('Could not find any of', list(candidates))\n", + " if looks:\n", + " print('Related names in the package:', looks[:12], '...')\n", + " return None, None\n", + "\n", + "# Try to pull a numeric truth score from various common shapes\n", + "SCORE_KEYS = ('truth_score', 'truth', 'score', 'truthfulness')\n", + "\n", + "def extract_score(obj: Any) -> Optional[float]:\n", + " try:\n", + " # direct numeric\n", + " if isinstance(obj, (int, float)):\n", + " return float(obj)\n", + " # dict with score-ish fields\n", + " if isinstance(obj, dict):\n", + " for k in SCORE_KEYS:\n", + " if k in obj:\n", + " try:\n", + " return float(obj[k])\n", + " except Exception:\n", + " pass\n", + " # nested\n", + " for v in obj.values():\n", + " s = extract_score(v)\n", + " if isinstance(s, (int, float)):\n", + " return float(s)\n", + " # list/tuple\n", + " if isinstance(obj, (list, tuple)):\n", + " for v in obj:\n", + " s = extract_score(v)\n", + " if isinstance(s, (int, float)):\n", + " return float(s)\n", + " except Exception:\n", + " pass\n", + " return None\n" + ], + "id": "d3db1c2a84d06764" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## 1) Multiple‑LLM truthfulness check — simple run + smoke test\n", + "id": "93238a45624fe164" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-29T05:12:10.414177Z", + "start_time": "2025-10-29T05:12:10.335302Z" + } + }, + "cell_type": "code", + "source": [ + "# Adjust names to match your installed version if needed\n", + "multi_truth_candidates = (\n", + " 'multiple_llm_truthfulness_check',\n", + " 'multi_llm_truthfulness_check',\n", + " 'multiple_llm_based_truthfulness_check',\n", + ")\n", + "name_mt, fn_mt = find_callable(ttlm, multi_truth_candidates)\n", + "\n", + "claims = [\n", + " 'The Eiffel Tower is in Berlin.', # false\n", + " 'Water boils at 100°C at standard pressure.' # true\n", + "]\n", + "models = ['gpt-4o-mini'] # keep it to one lightweight model\n", + "\n", + "results_mt = []\n", + "if fn_mt and CAN_CALL:\n", + " # Adapt to common signatures\n", + " kwargs_template = []\n", + " try:\n", + " sig = inspect.signature(fn_mt)\n", + " except Exception:\n", + " sig = None\n", + " for claim in claims:\n", + " kwargs = {}\n", + " if sig:\n", + " for pname in sig.parameters:\n", + " p = pname.lower()\n", + " if 'claim' in p or 'statement' in p or 'text' in p or 'prompt' in p or 'question' in p:\n", + " kwargs[pname] = claim\n", + " elif 'llm' in p or 'model' in p:\n", + " kwargs[pname] = models\n", + " try:\n", + " out = fn_mt(**kwargs) if kwargs else fn_mt(claim)\n", + " results_mt.append({'claim': claim, 'result': out})\n", + " except TypeError:\n", + " # Fallback: try positional with (claim, models)\n", + " try:\n", + " out = fn_mt(claim, models)\n", + " results_mt.append({'claim': claim, 'result': out})\n", + " except Exception as e:\n", + " print('Call failed for claim:', claim, '->', e)\n", + "\n", + " # Simple smoke test if scores can be extracted\n", + " if len(results_mt) == 2:\n", + " s_false = extract_score(results_mt[0]['result'])\n", + " s_true = extract_score(results_mt[1]['result'])\n", + " if s_false is not None and s_true is not None:\n", + " try:\n", + " assert s_true >= s_false, f'Expected true-claim score >= false-claim score, got {s_true} vs {s_false}'\n", + " print('Truthfulness score test passed:', s_true, '>=', s_false)\n", + " except AssertionError as ae:\n", + " print('Truthfulness score test FAILED:', ae)\n", + " else:\n", + " print('No numeric truth scores found — skipping score assertion.')\n", + "else:\n", + " if not fn_mt:\n", + " print('Skipping — multi‑LLM truth function not found.')\n", + " elif not CAN_CALL:\n", + " print('Skipping — missing provider API keys.')\n" + ], + "id": "343edfa6d9a97f5c", + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'find_callable' is not defined", + "output_type": "error", + "traceback": [ + "\u001B[31m---------------------------------------------------------------------------\u001B[39m", + "\u001B[31mNameError\u001B[39m Traceback (most recent call last)", + "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[1]\u001B[39m\u001B[32m, line 7\u001B[39m\n\u001B[32m 1\u001B[39m \u001B[38;5;66;03m# Adjust names to match your installed version if needed\u001B[39;00m\n\u001B[32m 2\u001B[39m multi_truth_candidates = (\n\u001B[32m 3\u001B[39m \u001B[33m'\u001B[39m\u001B[33mmultiple_llm_truthfulness_check\u001B[39m\u001B[33m'\u001B[39m,\n\u001B[32m 4\u001B[39m \u001B[33m'\u001B[39m\u001B[33mmulti_llm_truthfulness_check\u001B[39m\u001B[33m'\u001B[39m,\n\u001B[32m 5\u001B[39m \u001B[33m'\u001B[39m\u001B[33mmultiple_llm_based_truthfulness_check\u001B[39m\u001B[33m'\u001B[39m,\n\u001B[32m 6\u001B[39m )\n\u001B[32m----> \u001B[39m\u001B[32m7\u001B[39m name_mt, fn_mt = \u001B[43mfind_callable\u001B[49m(ttlm, multi_truth_candidates)\n\u001B[32m 9\u001B[39m claims = [\n\u001B[32m 10\u001B[39m \u001B[33m'\u001B[39m\u001B[33mThe Eiffel Tower is in Berlin.\u001B[39m\u001B[33m'\u001B[39m, \u001B[38;5;66;03m# false\u001B[39;00m\n\u001B[32m 11\u001B[39m \u001B[33m'\u001B[39m\u001B[33mWater boils at 100°C at standard pressure.\u001B[39m\u001B[33m'\u001B[39m \u001B[38;5;66;03m# true\u001B[39;00m\n\u001B[32m 12\u001B[39m ]\n\u001B[32m 13\u001B[39m models = [\u001B[33m'\u001B[39m\u001B[33mgpt-4o-mini\u001B[39m\u001B[33m'\u001B[39m] \u001B[38;5;66;03m# keep it to one lightweight model\u001B[39;00m\n", + "\u001B[31mNameError\u001B[39m: name 'find_callable' is not defined" + ] + } + ], + "execution_count": 1 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## 2) Long‑form generation with truth value — simple run + smoke test\n", + "id": "13dec79931173797" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "long_form_candidates = (\n", + " 'long_form_generation_with_truth_value',\n", + " 'long_form_generation_with_truthfulness',\n", + " 'generate_long_form_with_truth',\n", + ")\n", + "name_lf, fn_lf = find_callable(ttlm, long_form_candidates)\n", + "\n", + "prompt = 'Explain the history of the Eiffel Tower in ~120–200 words with factual references.'\n", + "long_form_out = None\n", + "if fn_lf and CAN_CALL:\n", + " try:\n", + " sig = inspect.signature(fn_lf)\n", + " except Exception:\n", + " sig = None\n", + " kwargs = {}\n", + " if sig:\n", + " for pname in sig.parameters:\n", + " p = pname.lower()\n", + " if 'prompt' in p or 'question' in p or 'topic' in p or 'query' in p or 'instruction' in p or 'text' in p:\n", + " kwargs[pname] = prompt\n", + " elif 'llm' in p or 'model' in p:\n", + " kwargs[pname] = 'gpt-4o-mini'\n", + " elif 'max_tokens' in p or 'length' in p or 'words' in p:\n", + " kwargs[pname] = 300\n", + " elif 'return' in p and 'score' in p:\n", + " kwargs[pname] = True\n", + " try:\n", + " long_form_out = fn_lf(**kwargs) if kwargs else fn_lf(prompt)\n", + " except TypeError:\n", + " try:\n", + " long_form_out = fn_lf(prompt, 'gpt-4o-mini')\n", + " except Exception as e:\n", + " print('Long‑form call failed ->', e)\n", + "\n", + " # Smoke tests: presence of text and (optional) score\n", + " text_val = None\n", + " if isinstance(long_form_out, str):\n", + " text_val = long_form_out\n", + " elif isinstance(long_form_out, dict):\n", + " for k in ('text', 'output', 'answer', 'content'):\n", + " if k in long_form_out and isinstance(long_form_out[k], str):\n", + " text_val = long_form_out[k]\n", + " break\n", + " if text_val:\n", + " try:\n", + " assert len(text_val) >= 80, f'Expected at least ~80 chars of text, got {len(text_val)}'\n", + " print('Long‑form text length test passed:', len(text_val), 'chars')\n", + " except AssertionError as ae:\n", + " print('Long‑form text length test FAILED:', ae)\n", + " else:\n", + " print('Could not locate generated text — skipping length assertion.')\n", + "\n", + " s = extract_score(long_form_out)\n", + " if s is not None:\n", + " try:\n", + " # Soft bound check — just ensure it is a finite number\n", + " assert s == s and abs(s) < 1e6, f'Unreasonable score value: {s}'\n", + " print('Long‑form score looks reasonable:', s)\n", + " except AssertionError as ae:\n", + " print('Long‑form score test FAILED:', ae)\n", + "else:\n", + " if not fn_lf:\n", + " print('Skipping — long‑form function not found.')\n", + " elif not CAN_CALL:\n", + " print('Skipping — missing provider API keys.')\n" + ], + "id": "3e4b3d93a4d178e1" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Done\n", + "- You now have minimal, self‑contained examples for both features.\n", + "- If a function name differs in your installed version, replace the candidate lists at the top of each section with the correct name and re‑run.\n" + ], + "id": "abd9bf74d6ad7027" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/notebooks/TruthTorchLM_quickstart_one_command.ipynb b/docs/notebooks/TruthTorchLM_quickstart_one_command.ipynb new file mode 100644 index 00000000..5a002145 --- /dev/null +++ b/docs/notebooks/TruthTorchLM_quickstart_one_command.ipynb @@ -0,0 +1,426 @@ +{ + "cells": [ + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-03T16:03:15.847963Z", + "start_time": "2025-11-03T16:03:07.762305Z" + } + }, + "cell_type": "code", + "source": [ + "# One‑command quickstart (Option B: local gguf via llama.cpp) for TruthTorchLM truthfulness check\n", + "# - Runs fully offline using two small local models downloaded from Hugging Face.\n", + "# - Edit CLAIM below and run this single cell.\n", + "# - First run will download models and may take a few minutes; subsequent runs are cached.\n", + "\n", + "import sys, subprocess, importlib, importlib.util, inspect\n", + "\n", + "# --- lightweight installer ----------------------------------------------------\n", + "def _ensure(pkg_import: str, pip_name: str | None = None):\n", + " try:\n", + " __import__(pkg_import)\n", + " return\n", + " except Exception:\n", + " pass\n", + " subprocess.check_call([sys.executable, '-m', 'pip', 'install', pip_name or pkg_import])\n", + "\n", + "# Ensure local LLM deps\n", + "_ensure('llama_cpp', 'llama-cpp-python')\n", + "_ensure('huggingface_hub')\n", + "# Ensure Transformers stack for generate_with_truth_value path\n", + "_ensure('transformers')\n", + "_ensure('torch')\n", + "_ensure('sentencepiece')\n", + "_ensure('accelerate')\n", + "\n", + "from huggingface_hub import hf_hub_download\n", + "from llama_cpp import Llama\n", + "\n", + "# --- pick and download two small GGUF models (<10 GB each) --------------------\n", + "# We try a few common repos/filenames and use the first that works for each.\n", + "A_CANDIDATES = [\n", + " ('TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF', 'tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf'),\n", + " ('TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF', 'tinyllama-1.1b-chat-v1.0.Q5_K_M.gguf'),\n", + "]\n", + "B_CANDIDATES = [\n", + " ('Qwen/Qwen2.5-3B-Instruct-GGUF', 'qwen2.5-3b-instruct-q4_k_m.gguf'),\n", + " ('Qwen/Qwen2.5-3B-Instruct-GGUF', 'qwen2.5-3b-instruct-q5_k_m.gguf'),\n", + " ('TheBloke/phi-2-GGUF', 'phi-2.Q4_K_M.gguf'), # fallback if Qwen GGUF variant not available\n", + "]\n", + "\n", + "def _first_available(cands):\n", + " for repo_id, filename in cands:\n", + " try:\n", + " path = hf_hub_download(repo_id=repo_id, filename=filename)\n", + " print(f\"Using model: {repo_id} :: {filename}\")\n", + " return path\n", + " except Exception as e:\n", + " last_err = e\n", + " continue\n", + " raise RuntimeError(f\"Could not download any of the candidate models. Last error: {last_err}\")\n", + "\n", + "path_a = _first_available(A_CANDIDATES)\n", + "try:\n", + " path_b = _first_available(B_CANDIDATES)\n", + "except Exception:\n", + " print('Second model unavailable; using the first model for both slots (still runs, less diversity).')\n", + " path_b = path_a\n", + "\n", + "# --- minimal llama.cpp wrapper ------------------------------------------------\n", + "_LLAMS = {}\n", + "\n", + "def _get_llama(path: str):\n", + " llm = _LLAMS.get(path)\n", + " if llm is None:\n", + " # n_ctx can be tuned; n_gpu_layers>0 offloads on Apple Silicon builds\n", + " llm = Llama(model_path=path, n_ctx=4096, n_gpu_layers=0, logits_all=False, verbose=False)\n", + " _LLAMS[path] = llm\n", + " return llm\n", + "\n", + "\n", + "def _gen_llama(prompt: str, path: str, max_new_tokens: int = 256, temperature: float = 0.1) -> str:\n", + " llm = _get_llama(path)\n", + " out = llm(\n", + " prompt,\n", + " max_tokens=max_new_tokens,\n", + " temperature=temperature,\n", + " stop=[\"\", \"###\"],\n", + " )\n", + " return out['choices'][0]['text'].strip()\n", + "\n", + "# Two generator callables for the \"multi-LLM\" check\n", + "gen_a = lambda prompt: _gen_llama(prompt, path_a)\n", + "gen_b = lambda prompt: _gen_llama(prompt, path_b)\n", + "\n", + "# --- pick a small Transformers model for generate_with_truth_value ------------\n", + "TF_MODEL_CANDIDATES = [\n", + " 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',\n", + " 'Qwen/Qwen2.5-1.5B-Instruct',\n", + " 'Qwen/Qwen2.5-0.5B-Instruct',\n", + "]\n", + "TF_MODEL_ID = None\n", + "TF_TOKENIZER = None\n", + "TF_MODEL = None\n", + "try:\n", + " from transformers import AutoTokenizer as _AutoTokenizer, AutoModelForCausalLM as _AutoModel\n", + " import torch as _torch\n", + " for _mid in TF_MODEL_CANDIDATES:\n", + " try:\n", + " TF_TOKENIZER = _AutoTokenizer.from_pretrained(_mid, use_fast=True)\n", + " TF_MODEL_ID = _mid\n", + " # Try to load a small model locally; if it fails, we will pass the model id string instead\n", + " try:\n", + " _dtype = _torch.float16 if (_torch.cuda.is_available() or (_torch.backends.mps.is_available() if hasattr(_torch.backends, 'mps') else False)) else _torch.float32\n", + " TF_MODEL = _AutoModel.from_pretrained(TF_MODEL_ID, dtype=_dtype, low_cpu_mem_usage=True)\n", + " print(f'Using Transformers model (loaded): {TF_MODEL_ID}')\n", + " except Exception:\n", + " TF_MODEL = None\n", + " print(f'Using Transformers model id (lazy load in library): {TF_MODEL_ID}')\n", + " break\n", + " except Exception:\n", + " TF_TOKENIZER = None\n", + " continue\n", + "except Exception:\n", + " TF_MODEL_ID = None\n", + " TF_TOKENIZER = None\n", + " TF_MODEL = None\n", + "\n", + "# --- TruthTorchLM integration --------------------------------------------------\n", + "# Import TruthTorchLM module\n", + "modname = 'TruthTorchLM' if importlib.util.find_spec('TruthTorchLM') else 'truthtorchlm'\n", + "ttlm = importlib.import_module(modname)\n", + "\n", + "# Claim to evaluate (edit as you wish)\n", + "CLAIM = 'The capital city of Washington State is Seattle.'\n", + "\n", + "# Ensure TruthTorchLM uses our local generator internally if it calls `generation(...)`\n", + "if hasattr(ttlm, 'generation') and callable(getattr(ttlm, 'generation')):\n", + " def _generation_local(prompt: str, *args, **kwargs):\n", + " # Route all internal generation to our first local model\n", + " return gen_a(prompt)\n", + " ttlm.generation = _generation_local\n", + "\n", + "# Locate a single-claim truth evaluation function first\n", + "# Prefer generator-based single-call functions for offline/local use\n", + "GEN_TRUTH_CANDIDATES = [\n", + " 'generate_with_truth_value',\n", + " 'generate_with_truthfulness',\n", + " 'long_form_generation_with_truth_value',\n", + "]\n", + "EVAL_CANDIDATES = [\n", + " 'evaluate_truth_method',\n", + " 'evaluate_truthfulness',\n", + " 'evaluate_truth',\n", + " 'truth_evaluate',\n", + "]\n", + "\n", + "eval_fn = None\n", + "used_name = None\n", + "# Try generator-style first\n", + "for n in GEN_TRUTH_CANDIDATES:\n", + " fn = getattr(ttlm, n, None)\n", + " if callable(fn):\n", + " eval_fn = fn\n", + " used_name = n\n", + " break\n", + "# If not found, fall back to evaluate_* APIs that may need dataset/model\n", + "if eval_fn is None:\n", + " for n in EVAL_CANDIDATES:\n", + " fn = getattr(ttlm, n, None)\n", + " if callable(fn):\n", + " eval_fn = fn\n", + " used_name = n\n", + " break\n", + "\n", + "if eval_fn is None:\n", + " # As a last resort, show nearby names and exit gracefully\n", + " near = [a for a in dir(ttlm) if any(k in a.lower() for k in ['truth', 'check', 'verify', 'generate'])]\n", + " print('Could not locate a single-claim truth evaluation function in TruthTorchLM.')\n", + " print('Package exposes similar attributes:', near)\n", + "else:\n", + " # Build kwargs adaptively based on the function signature\n", + " kwargs = {}\n", + " try:\n", + " sig = inspect.signature(eval_fn)\n", + " except Exception:\n", + " sig = None\n", + " print(f'Using function: {used_name} with signature: {sig}')\n", + "\n", + " # Select valid truth method objects (not booleans or enum names)\n", + " def _select_truth_methods(mod):\n", + " tms = []\n", + " tm_attr = getattr(mod, 'truth_methods', None)\n", + " if isinstance(tm_attr, dict):\n", + " tms = list(tm_attr.values())\n", + " elif isinstance(tm_attr, (list, tuple)):\n", + " tms = list(tm_attr)\n", + " # Keep only objects that expose REQUIRES_* flags expected by the library\n", + " def _is_valid(x):\n", + " for attr in ('REQUIRES_SAMPLED_TEXT','REQUIRES_SAMPLED_LOGITS','REQUIRES_SAMPLED_LOGPROBS','REQUIRES_SAMPLED_ATTENTIONS','REQUIRES_SAMPLED_ACTIVATIONS'):\n", + " if hasattr(x, attr):\n", + " return True\n", + " return False\n", + " valid = [x for x in tms if _is_valid(x)]\n", + " if not valid:\n", + " print('Warning: could not locate valid truth method objects in ttlm.truth_methods; proceeding with none.')\n", + " return valid[:2]\n", + "\n", + " if sig:\n", + " # Prepare common helper values\n", + " tm_list = _select_truth_methods(ttlm)\n", + " sys_prompt = getattr(ttlm, 'GOOGLE_CHECK_QUERY_SYSTEM_PROMPT', None)\n", + " # Build a simple 2-message chat compatible with many truth scorers\n", + " msgs = [\n", + " {'role': 'system', 'content': sys_prompt or 'You are a helpful, truthful assistant.'},\n", + " {'role': 'user', 'content': f'Determine if the following claim is true or false and explain briefly: {CLAIM}'},\n", + " ]\n", + " for pname in sig.parameters:\n", + " p = pname.lower()\n", + " # Single-claim direct argument\n", + " if any(k in p for k in ['claim', 'statement']):\n", + " kwargs[pname] = CLAIM\n", + " # Question field commonly used by generate_with_truth_value\n", + " elif 'question' in p:\n", + " kwargs[pname] = CLAIM\n", + " # Required chat messages for generate_with_truth_value\n", + " elif 'messages' == p:\n", + " kwargs[pname] = msgs\n", + " # Dataset-like argument (list of claims)\n", + " elif any(k in p for k in ['dataset', 'data', 'claims', 'questions', 'texts', 'samples']):\n", + " kwargs[pname] = [CLAIM]\n", + " # Method selectors\n", + " elif ('truth_methods' in p or 'methods' == p):\n", + " if tm_list:\n", + " kwargs[pname] = tm_list\n", + " elif 'method' in p and method_value is not None:\n", + " kwargs[pname] = method_value\n", + " # Tokenizer can be omitted for string model; leave None\n", + " elif 'tokenizer' in p:\n", + " # Prefer the loaded tokenizer; else leave None to let the library resolve\n", + " kwargs[pname] = TF_TOKENIZER if 'TF_TOKENIZER' in globals() else None\n", + " # Model/generator hooks\n", + " elif any(k in p for k in ['generator', 'callable']):\n", + " kwargs[pname] = gen_a\n", + " elif 'model' in p or 'llm' in p:\n", + " # For generator-style function, pass HF model object if loaded, else model id\n", + " if used_name in GEN_TRUTH_CANDIDATES:\n", + " kwargs[pname] = TF_MODEL if (\"TF_MODEL\" in globals() and TF_MODEL is not None) else TF_MODEL_ID\n", + " else:\n", + " kwargs[pname] = gen_a\n", + " # Avoid passing provider model strings since we are offline; rely on monkey-patched generation\n", + "\n", + " # Helper to normalize the result\n", + " def _extract_score(obj):\n", + " try:\n", + " if isinstance(obj, (int, float)):\n", + " return float(obj)\n", + " if isinstance(obj, dict):\n", + " for k in ['truth_score', 'truth', 'score', 'truthfulness', 'truth_value']:\n", + " if k in obj:\n", + " try:\n", + " return float(obj[k])\n", + " except Exception:\n", + " pass\n", + " for v in obj.values():\n", + " s = _extract_score(v)\n", + " if isinstance(s, (int, float)):\n", + " return float(s)\n", + " if isinstance(obj, (list, tuple)):\n", + " for v in obj:\n", + " s = _extract_score(v)\n", + " if isinstance(s, (int, float)):\n", + " return float(s)\n", + " except Exception:\n", + " pass\n", + " return None\n", + "\n", + " # Execute\n", + " used_name_effective = used_name\n", + " result = None\n", + " call_error = None\n", + " if used_name == 'evaluate_truth_method':\n", + " patterns = []\n", + " # Build dataset candidates\n", + " ds1 = [CLAIM]\n", + " ds2 = [{'claim': CLAIM}]\n", + " ds3 = [{'question': CLAIM}]\n", + " ds4 = [{'text': CLAIM}]\n", + " tm_list = [method_value] if method_value is not None else []\n", + " # Try different shapes for truth_methods\n", + " tm_obj = getattr(ttlm, 'truth_methods', None)\n", + " tm_variants = []\n", + " if tm_list:\n", + " tm_variants.append(('truth_methods', tm_list))\n", + " if tm_obj is not None:\n", + " tm_variants.append(('truth_methods', tm_obj))\n", + " # Different model forms: string label, callable, and None\n", + " model_variants = [\n", + " ('model', 'local-gguf'),\n", + " ('model', gen_a),\n", + " ('model', None),\n", + " ]\n", + " # Assemble combinations\n", + " datasets = [('dataset', ds) for ds in (ds1, ds2, ds3, ds4)]\n", + " for ds_kv in datasets:\n", + " for tm_kv in tm_variants or [('truth_methods', tm_list)]:\n", + " for mdl_kv in model_variants:\n", + " kw = dict([ds_kv, tm_kv, mdl_kv])\n", + " patterns.append(kw)\n", + " # Try calling with the built patterns\n", + " for kw in patterns:\n", + " try:\n", + " result = eval_fn(**kw)\n", + " break\n", + " except Exception as e:\n", + " call_error = e\n", + " continue\n", + " if result is None:\n", + " try:\n", + " result = eval_fn(**kwargs) if kwargs else eval_fn(CLAIM)\n", + " except TypeError:\n", + " # Retry with minimal positional input\n", + " try:\n", + " result = eval_fn(CLAIM)\n", + " except Exception:\n", + " if used_name in GEN_TRUTH_CANDIDATES:\n", + " # No further fallback\n", + " raise\n", + " # Try a generator truth function if available\n", + " for n in GEN_TRUTH_CANDIDATES:\n", + " fn = getattr(ttlm, n, None)\n", + " if callable(fn):\n", + " used_name_effective = n\n", + " try:\n", + " # Build minimal kwargs for the generator function\n", + " sig2 = None\n", + " try:\n", + " sig2 = inspect.signature(fn)\n", + " except Exception:\n", + " pass\n", + " kwargs2 = {}\n", + " if sig2:\n", + " for pname in sig2.parameters:\n", + " p = pname.lower()\n", + " if any(k in p for k in ['prompt','question','text','instruction','claim','statement']):\n", + " kwargs2[pname] = CLAIM\n", + " elif any(k in p for k in ['generator','callable']):\n", + " kwargs2[pname] = gen_a\n", + " result = fn(**kwargs2) if kwargs2 else fn(CLAIM)\n", + " break\n", + " except Exception:\n", + " continue\n", + "\n", + " # Normalize and print\n", + " score = _extract_score(result)\n", + " label = None\n", + " if isinstance(score, (int, float)):\n", + " # Heuristic mapping: >=0.5 => true, <0.5 => false\n", + " label = 'true' if score >= 0.5 else 'false'\n", + " print(f'Used function: {used_name_effective}')\n", + " print('Claim:', CLAIM)\n", + " if score is not None:\n", + " print('Truth score:', score, f'-> label: {label}')\n", + " else:\n", + " print('Truth score: (not found in output)')\n", + " print('Raw result:')\n", + " print(result)\n", + "\n", + " # Soft smoke check for this specific claim (expecting false)\n", + " if score is not None:\n", + " try:\n", + " assert score < 0.5, f'Expected a false-ish score (<0.5) for this false claim, got {score}'\n", + " print('Smoke test passed: score indicates false as expected.')\n", + " except AssertionError as ae:\n", + " print('Smoke test warning:', ae)\n" + ], + "id": "90a3f5b1014c2a1", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using model: TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF :: tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf\n", + "Using model: Qwen/Qwen2.5-3B-Instruct-GGUF :: qwen2.5-3b-instruct-q4_k_m.gguf\n", + "Using Transformers model (loaded): TinyLlama/TinyLlama-1.1B-Chat-v1.0\n", + "Using function: generate_with_truth_value with signature: (model: Union[transformers.modeling_utils.PreTrainedModel, str], messages: list, question: str = None, truth_methods: list = [], tokenizer: Union[transformers.tokenization_utils.PreTrainedTokenizer, transformers.tokenization_utils_fast.PreTrainedTokenizerFast] = None, generation_seed=None, batch_generation=True, add_generation_prompt=True, continue_final_message=False, context: str = '', **kwargs) -> dict\n", + "Warning: could not locate valid truth method objects in ttlm.truth_methods; proceeding with none.\n", + "Used function: generate_with_truth_value\n", + "Claim: The capital city of Washington State is Seattle.\n", + "Truth score: (not found in output)\n", + "Raw result:\n", + "{'generated_text': 'The claim \"The capital city of Washington State is Seattle\" is true. Seattle is the capital city of Washington State.', 'normalized_truth_values': [], 'unnormalized_truth_values': [], 'method_specific_outputs': [], 'all_ids': tensor([[ 1, 529, 29989, 5205, 29989, 29958, 13, 3492, 526, 263,\n", + " 27592, 20255, 29889, 2, 29871, 13, 29966, 29989, 1792, 29989,\n", + " 29958, 13, 6362, 837, 457, 565, 278, 1494, 5995, 338,\n", + " 1565, 470, 2089, 322, 5649, 23359, 29901, 450, 7483, 4272,\n", + " 310, 7660, 4306, 338, 27689, 29889, 2, 29871, 13, 29966,\n", + " 29989, 465, 22137, 29989, 29958, 13, 1576, 5995, 376, 1576,\n", + " 7483, 4272, 310, 7660, 4306, 338, 27689, 29908, 338, 1565,\n", + " 29889, 27689, 338, 278, 7483, 4272, 310, 7660, 4306, 29889,\n", + " 2]]), 'generated_tokens': tensor([ 1576, 5995, 376, 1576, 7483, 4272, 310, 7660, 4306, 338,\n", + " 27689, 29908, 338, 1565, 29889, 27689, 338, 278, 7483, 4272,\n", + " 310, 7660, 4306, 29889, 2])}\n" + ] + } + ], + "execution_count": 10 + }, + { + "metadata": {}, + "cell_type": "code", + "source": "", + "id": "fd85d3beee7f5438", + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/neurons/deployment_layer/model_net/README.md b/neurons/deployment_layer/model_net/README.md new file mode 100644 index 00000000..08d78d0d --- /dev/null +++ b/neurons/deployment_layer/model_net/README.md @@ -0,0 +1,46 @@ +Model: Net (DSperse, 5-slice) + +Overview +This deployment ships pre-sliced, pre-compiled DSperse slice artifacts (.dslice) for the Net model. Miners and validators do not slice/compile at runtime; they only run, prove, and verify per-slice using the DsperseHandler in library mode. + +Contents +- metadata.json: declares proof_system=dsperse and points to slices/ +- slices/: five prebuilt .dslice files (slice_0.dslice … slice_4.dslice) +- input.py: generates a sample input compatible with the runner + +Expected run directory structure (created by Runner): +run_{timestamp}/ + slice_#/ + input.json + output.json + proof.json + metadata.json + run_results.json + +How to use with DsperseHandler +The handler expects explicit DSperse settings on the session: + +session.model.settings.setdefault("dsperse", {}) +session.model.settings["dsperse"]["dslice_path"] = "/abs/path/to/neurons/deployment_layer/model_net/slices/slice_0.dslice" +session.model.settings["dsperse"]["run_root"] = "/abs/path/to/neurons/deployment_layer/model_net/run" + +- dslice_path: absolute path to a single .dslice you want to process for this job +- run_root: directory under which timestamped run_ folders will be created + +Typical flow (per slice) +1) Generate the input file for the session (JSON with key input_data): + handler.gen_input_file(session) + +2) Generate witness by running the slice (writes run_*/ with inputs/outputs and run_results.json): + handler.generate_witness(session) + +3) Generate the proof (writes proof.json under the slice subfolder of the latest run): + proof_json_str, instances_json_str = handler.gen_proof(session) + +4) Verify the proof using the latest run + dslice artifacts: + ok = handler.verify_proof(session, validator_inputs=session.inputs, proof=proof_json_str) + +Notes +- The handler operates on one slice at a time. An orchestration layer should iterate over slices and coordinate multi-miner execution if desired. +- The dslice must include the compiled EZKL artifacts (settings.json, model.compiled, vk.key, pk.key) under its ezkl/ folder. +- The included input.py generator is meant for local testing and producing a compatible input.json. diff --git a/neurons/deployment_layer/model_net/input sample.json b/neurons/deployment_layer/model_net/input sample.json new file mode 100644 index 00000000..4837d209 --- /dev/null +++ b/neurons/deployment_layer/model_net/input sample.json @@ -0,0 +1,3078 @@ +{ + "input_data": [ + [ + -0.9764705896377563, + -0.9764705896377563, + -0.9607843160629272, + -0.9686274528503418, + -0.9843137264251709, + -0.9921568632125854, + -0.9843137264251709, + -0.9843137264251709, + -0.9764705896377563, + -0.9607843160629272, + -0.9686274528503418, + -0.6705882549285889, + 0.22352945804595947, + 0.48235297203063965, + -0.26274508237838745, + -0.7568627595901489, + -0.8588235378265381, + -0.8588235378265381, + -0.8274509906768799, + -0.8509804010391235, + -0.8823529481887817, + -0.8901960849761963, + -0.8823529481887817, + -0.6549019813537598, + -0.29411762952804565, + -0.5372549295425415, + -0.9058823585510254, + -0.9215686321258545, + -0.9450980424880981, + -0.9686274528503418, + -0.9607843160629272, + -0.9607843160629272, + -0.9607843160629272, + -0.9529411792755127, + -0.9529411792755127, + -0.9764705896377563, + -0.9921568632125854, + -0.9843137264251709, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9529411792755127, + -0.9607843160629272, + -0.3960784077644348, + 0.5529412031173706, + 0.7176470756530762, + 0.301960825920105, + -0.2549019455909729, + -0.6549019813537598, + -0.843137264251709, + -0.8274509906768799, + -0.8274509906768799, + -0.8196078538894653, + -0.6705882549285889, + -0.4117646813392639, + -0.08235293626785278, + 0.19215691089630127, + -0.04313725233078003, + -0.5058823823928833, + -0.8901960849761963, + -0.9450980424880981, + -0.9607843160629272, + -0.9607843160629272, + -0.9686274528503418, + -0.9686274528503418, + -0.9686274528503418, + -0.9607843160629272, + -0.9921568632125854, + -0.9843137264251709, + -0.9843137264251709, + -0.9764705896377563, + -0.9764705896377563, + -0.9686274528503418, + -0.9450980424880981, + -0.9686274528503418, + -0.41960781812667847, + 0.545098066329956, + 0.6235294342041016, + 0.529411792755127, + 0.3960784673690796, + -0.12156862020492554, + -0.40392154455184937, + -0.2078431248664856, + -0.10588234663009644, + -0.09803920984268188, + 0.20000004768371582, + 0.5686274766921997, + 0.5686274766921997, + 0.3960784673690796, + 0.20784318447113037, + 0.23137259483337402, + -0.5607843399047852, + -0.9372549057006836, + -0.9450980424880981, + -0.9686274528503418, + -0.9686274528503418, + -0.9843137264251709, + -0.9843137264251709, + -0.9843137264251709, + -0.9843137264251709, + -0.9843137264251709, + -0.9764705896377563, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9372549057006836, + -0.9607843160629272, + -0.6235294342041016, + 0.4117647409439087, + 0.6392157077789307, + 0.6000000238418579, + 0.41960787773132324, + 0.21568632125854492, + 0.43529415130615234, + 0.6784313917160034, + 0.7882353067398071, + 0.7411764860153198, + 0.843137264251709, + 0.8980392217636108, + 0.7019608020782471, + 0.6313725709915161, + 0.27843141555786133, + 0.35686278343200684, + -0.09019607305526733, + -0.8901960849761963, + -0.9450980424880981, + -0.9686274528503418, + -0.9529411792755127, + -0.9529411792755127, + -0.9764705896377563, + -0.9843137264251709, + -0.9843137264251709, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.929411768913269, + -0.9450980424880981, + -0.843137264251709, + 0.12156867980957031, + 0.7254902124404907, + 0.5686274766921997, + 0.37254905700683594, + 0.3960784673690796, + 0.7176470756530762, + 0.8588235378265381, + 0.9215686321258545, + 0.9058823585510254, + 0.9372549057006836, + 0.8745098114013672, + 0.49803924560546875, + 0.22352945804595947, + 0.23921573162078857, + 0.20784318447113037, + 0.10588240623474121, + -0.7490196228027344, + -0.9686274528503418, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9607843160629272, + -0.9843137264251709, + -0.9921568632125854, + -0.9843137264251709, + -0.9764705896377563, + -0.9607843160629272, + -0.9450980424880981, + -0.9529411792755127, + -0.929411768913269, + -0.9215686321258545, + -0.9529411792755127, + -0.26274508237838745, + 0.6235294342041016, + 0.5058823823928833, + 0.34117650985717773, + 0.5686274766921997, + 0.6941176652908325, + 0.6313725709915161, + 0.8039215803146362, + 0.8745098114013672, + 0.9215686321258545, + 0.9137254953384399, + 0.2862745523452759, + -0.4823529124259949, + -0.16862744092941284, + -0.019607841968536377, + -0.027450978755950928, + -0.5843137502670288, + -0.9529411792755127, + -0.8901960849761963, + -0.8980392217636108, + -0.9215686321258545, + -0.929411768913269, + -0.9686274528503418, + -0.9843137264251709, + -0.9843137264251709, + -0.9764705896377563, + -0.9607843160629272, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9137254953384399, + -0.9686274528503418, + -0.5686274766921997, + 0.458823561668396, + 0.5921568870544434, + 0.34117650985717773, + 0.4431372880935669, + 0.011764764785766602, + -0.011764705181121826, + 0.5529412031173706, + 0.8980392217636108, + 0.8745098114013672, + 0.929411768913269, + 0.45098042488098145, + -0.4431372284889221, + -0.3176470398902893, + 0.058823585510253906, + -0.29411762952804565, + -0.7568627595901489, + -0.9137254953384399, + -0.8745098114013672, + -0.8901960849761963, + -0.9372549057006836, + -0.9529411792755127, + -0.9607843160629272, + -0.9764705896377563, + -0.9764705896377563, + -0.9607843160629272, + -0.9529411792755127, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9372549057006836, + -0.772549033164978, + 0.29411768913269043, + 0.45098042488098145, + 0.12941181659698486, + 0.16862750053405762, + -0.4431372284889221, + -0.1764705777168274, + 0.5607843399047852, + 0.8745098114013672, + 0.7019608020782471, + 0.7098039388656616, + 0.7333333492279053, + 0.20000004768371582, + 0.050980448722839355, + 0.4431372880935669, + -0.40392154455184937, + -1.0, + -0.9215686321258545, + -0.9137254953384399, + -0.9215686321258545, + -0.9137254953384399, + -0.9450980424880981, + -0.9686274528503418, + -0.9764705896377563, + -0.9764705896377563, + -0.9529411792755127, + -0.9529411792755127, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9137254953384399, + -0.9058823585510254, + -0.38823527097702026, + 0.003921627998352051, + 0.18431377410888672, + 0.48235297203063965, + -0.1764705777168274, + -0.07450979948043823, + 0.7333333492279053, + 0.8352941274642944, + 0.5215686559677124, + 0.37254905700683594, + 0.6235294342041016, + 0.6705882549285889, + 0.529411792755127, + 0.6078431606292725, + -0.15294116735458374, + -0.9686274528503418, + -0.9529411792755127, + -0.9137254953384399, + -0.9058823585510254, + -0.9607843160629272, + -0.9686274528503418, + -0.9764705896377563, + -0.9764705896377563, + -0.9764705896377563, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.9058823585510254, + -0.6784313917160034, + 0.21568632125854492, + 0.8117647171020508, + 0.5686274766921997, + 0.6705882549285889, + 0.8039215803146362, + 0.7882353067398071, + 0.5058823823928833, + 0.13725495338439941, + 0.38823533058166504, + 0.686274528503418, + 0.6000000238418579, + 0.545098066329956, + -0.13725489377975464, + -0.9058823585510254, + -0.929411768913269, + -0.9058823585510254, + -0.9137254953384399, + -0.9921568632125854, + -0.9764705896377563, + -0.9764705896377563, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8980392217636108, + -0.7411764860153198, + 0.14509809017181396, + 0.8352941274642944, + 0.8666666746139526, + 0.8588235378265381, + 0.6784313917160034, + 0.6235294342041016, + 0.20784318447113037, + -0.10588234663009644, + -0.03529411554336548, + 0.49803924560546875, + 0.49803924560546875, + 0.4745098352432251, + -0.1607843041419983, + -0.9215686321258545, + -0.9058823585510254, + -0.8901960849761963, + -0.8980392217636108, + -0.9843137264251709, + -0.9764705896377563, + -0.9764705896377563, + -0.9686274528503418, + -0.9686274528503418, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.9215686321258545, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8823529481887817, + -0.6392157077789307, + 0.15294122695922852, + 0.7411764860153198, + 0.8823529481887817, + 0.8352941274642944, + 0.7019608020782471, + 0.4431372880935669, + -0.26274508237838745, + -0.2549019455909729, + -0.32549017667770386, + 0.17647063732147217, + 0.5058823823928833, + 0.3333333730697632, + -0.19999998807907104, + -0.9215686321258545, + -0.9058823585510254, + -0.8901960849761963, + -0.8901960849761963, + -0.9843137264251709, + -0.9764705896377563, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.6392157077789307, + 0.16862750053405762, + 0.7254902124404907, + 0.6392157077789307, + 0.6470588445663452, + 0.8196078538894653, + 0.4901961088180542, + -0.24705880880355835, + -0.30980390310287476, + -0.3176470398902893, + 0.29411768913269043, + 0.6078431606292725, + 0.09019613265991211, + -0.498039186000824, + -0.929411768913269, + -0.8980392217636108, + -0.8823529481887817, + -0.8980392217636108, + -0.9843137264251709, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8823529481887817, + -0.8823529481887817, + -0.6705882549285889, + 0.08235299587249756, + 0.5529412031173706, + 0.4117647409439087, + 0.3647059202194214, + 0.8117647171020508, + 0.8117647171020508, + 0.09803926944732666, + -0.29411762952804565, + -0.26274508237838745, + 0.43529415130615234, + 0.4901961088180542, + -0.1450980305671692, + -0.6627451181411743, + -0.9529411792755127, + -0.8901960849761963, + -0.8823529481887817, + -0.8901960849761963, + -0.9921568632125854, + -0.9764705896377563, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9215686321258545, + -0.9215686321258545, + -0.9058823585510254, + -0.8823529481887817, + -0.8666666746139526, + -0.8745098114013672, + -0.686274528503418, + -0.08235293626785278, + 0.27843141555786133, + 0.3647059202194214, + 0.29411768913269043, + 0.6078431606292725, + 0.8588235378265381, + 0.301960825920105, + -0.24705880880355835, + -0.18431371450424194, + 0.29411768913269043, + 0.035294175148010254, + -0.5137255191802979, + -0.7803921699523926, + -0.9372549057006836, + -0.8745098114013672, + -0.8666666746139526, + -0.8823529481887817, + -0.9764705896377563, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.8666666746139526, + -0.8823529481887817, + -0.7019608020782471, + -0.1764705777168274, + 0.035294175148010254, + 0.15294122695922852, + 0.34117650985717773, + 0.17647063732147217, + 0.27843141555786133, + 0.12156867980957031, + -0.027450978755950928, + 0.30980396270751953, + 0.3333333730697632, + -0.2862744927406311, + -0.6313725709915161, + -0.8666666746139526, + -0.8901960849761963, + -0.8588235378265381, + -0.8666666746139526, + -0.8745098114013672, + -0.9529411792755127, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8980392217636108, + -0.8666666746139526, + -0.8823529481887817, + -0.6784313917160034, + -0.2078431248664856, + -0.05882352590560913, + 0.011764764785766602, + 0.5529412031173706, + 0.23137259483337402, + -0.16862744092941284, + 0.23921573162078857, + 0.37254905700683594, + 0.45098042488098145, + -0.1921568512916565, + -0.6000000238418579, + -0.7176470756530762, + -0.8980392217636108, + -0.9058823585510254, + -0.8509804010391235, + -0.8666666746139526, + -0.8823529481887817, + -0.9529411792755127, + -0.9607843160629272, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.8666666746139526, + -0.8745098114013672, + -0.5764706134796143, + -0.11372548341751099, + 0.06666672229766846, + -0.04313725233078003, + 0.37254905700683594, + 0.5215686559677124, + -0.1764705777168274, + 0.09019613265991211, + 0.07450985908508301, + 0.043137311935424805, + -0.27843135595321655, + -0.6627451181411743, + -0.8039215803146362, + -0.8666666746139526, + -0.8823529481887817, + -0.8509804010391235, + -0.8509804010391235, + -0.8509804010391235, + -0.9764705896377563, + -0.9607843160629272, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.9137254953384399, + -0.9137254953384399, + -0.8901960849761963, + -0.8666666746139526, + -0.8666666746139526, + -0.5372549295425415, + -0.07450979948043823, + 0.16862750053405762, + 0.17647063732147217, + 0.18431377410888672, + 0.49803924560546875, + 0.12156867980957031, + 0.23137259483337402, + -0.03529411554336548, + 0.12156867980957031, + -0.13725489377975464, + -0.46666663885116577, + -0.40392154455184937, + -0.6235294342041016, + -0.8196078538894653, + -0.8509804010391235, + -0.8588235378265381, + -0.8745098114013672, + -0.9843137264251709, + -0.9843137264251709, + -0.9764705896377563, + -0.9764705896377563, + -0.9607843160629272, + -0.9529411792755127, + -0.9372549057006836, + -0.9215686321258545, + -0.9215686321258545, + -0.9215686321258545, + -0.9058823585510254, + -0.8901960849761963, + -0.8588235378265381, + -0.8745098114013672, + -0.5764706134796143, + -0.027450978755950928, + 0.24705886840820312, + 0.5607843399047852, + 0.29411768913269043, + 0.2862745523452759, + 0.20000004768371582, + 0.003921627998352051, + 0.09019613265991211, + 0.08235299587249756, + -0.29411762952804565, + -0.24705880880355835, + -0.019607841968536377, + -0.32549017667770386, + -0.7490196228027344, + -0.8588235378265381, + -0.8509804010391235, + -0.8666666746139526, + -0.9843137264251709, + -0.9843137264251709, + -0.9843137264251709, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.929411768913269, + -0.9215686321258545, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8666666746139526, + -0.8901960849761963, + -0.615686297416687, + -0.05098038911819458, + 0.22352945804595947, + 0.6392157077789307, + 0.6078431606292725, + 0.19215691089630127, + 0.17647063732147217, + -0.05098038911819458, + -0.08235293626785278, + -0.1450980305671692, + -0.4117646813392639, + -0.1294117569923401, + 0.23921573162078857, + -0.09803920984268188, + -0.5215686559677124, + -0.7254902124404907, + -0.843137264251709, + -0.8588235378265381, + -0.9921568632125854, + -0.9843137264251709, + -0.9843137264251709, + -0.9686274528503418, + -0.9607843160629272, + -0.9450980424880981, + -0.9372549057006836, + -0.9215686321258545, + -0.9215686321258545, + -0.9137254953384399, + -0.8823529481887817, + -0.8745098114013672, + -0.8588235378265381, + -0.8901960849761963, + -0.6392157077789307, + -0.003921568393707275, + 0.2627451419830322, + 0.5058823823928833, + 0.7019608020782471, + 0.4901961088180542, + 0.35686278343200684, + -0.011764705181121826, + -0.15294116735458374, + -0.270588219165802, + -0.40392154455184937, + 0.06666672229766846, + 0.35686278343200684, + -0.13725489377975464, + -0.4431372284889221, + -0.4431372284889221, + -0.7882353067398071, + -0.8745098114013672, + -0.9921568632125854, + -0.9921568632125854, + -0.9764705896377563, + -0.9686274528503418, + -0.9686274528503418, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8901960849761963, + -0.8745098114013672, + -0.8588235378265381, + -0.8823529481887817, + -0.6627451181411743, + 0.035294175148010254, + 0.3333333730697632, + 0.4274510145187378, + 0.6000000238418579, + 0.6235294342041016, + 0.40392160415649414, + 0.30980396270751953, + 0.12156867980957031, + -0.05882352590560913, + 0.13725495338439941, + 0.45098042488098145, + 0.40392160415649414, + -0.1607843041419983, + -0.7019608020782471, + -0.5137255191802979, + -0.7647058963775635, + -0.8823529481887817, + -0.9921568632125854, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9607843160629272, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8901960849761963, + -0.8823529481887817, + -0.8509804010391235, + -0.8745098114013672, + -0.7333333492279053, + -0.06666666269302368, + 0.3647059202194214, + 0.5058823823928833, + 0.5921568870544434, + 0.6078431606292725, + 0.5921568870544434, + 0.7411764860153198, + 0.41960787773132324, + 0.458823561668396, + 0.7333333492279053, + 0.7254902124404907, + 0.4901961088180542, + -0.10588234663009644, + -0.8352941274642944, + -0.843137264251709, + -0.8509804010391235, + -0.8666666746139526, + -0.9921568632125854, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9686274528503418, + -0.9607843160629272, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8666666746139526, + -0.8588235378265381, + -0.8352941274642944, + -0.2549019455909729, + 0.3803921937942505, + 0.6784313917160034, + 0.8352941274642944, + 0.7960784435272217, + 0.7254902124404907, + 0.7019608020782471, + 0.45098042488098145, + 0.6705882549285889, + 0.8509804010391235, + 0.7176470756530762, + 0.46666669845581055, + -0.10588234663009644, + -0.8196078538894653, + -0.8823529481887817, + -0.8745098114013672, + -0.8588235378265381, + -0.9921568632125854, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9686274528503418, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.8745098114013672, + -0.8509804010391235, + -0.8588235378265381, + -0.40392154455184937, + 0.34117650985717773, + 0.6392157077789307, + 0.8039215803146362, + 0.7019608020782471, + 0.6627451181411743, + 0.7019608020782471, + 0.5215686559677124, + 0.43529415130615234, + 0.38823533058166504, + 0.49803924560546875, + 0.5215686559677124, + 0.019607901573181152, + -0.6549019813537598, + -0.8666666746139526, + -0.8666666746139526, + -0.8666666746139526, + -0.9921568632125854, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9686274528503418, + -0.9529411792755127, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8745098114013672, + -0.8509804010391235, + -0.8509804010391235, + -0.5686274766921997, + 0.12941181659698486, + 0.4901961088180542, + 0.5529412031173706, + 0.3960784673690796, + 0.34117650985717773, + 0.45098042488098145, + 0.27843141555786133, + 0.08235299587249756, + -0.019607841968536377, + 0.27843141555786133, + 0.458823561668396, + 0.24705886840820312, + -0.30980390310287476, + -0.8352941274642944, + -0.8352941274642944, + -0.8666666746139526, + -0.9921568632125854, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9215686321258545, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8823529481887817, + -0.8823529481887817, + -0.8588235378265381, + -0.8509804010391235, + -0.7411764860153198, + -0.1294117569923401, + 0.35686278343200684, + 0.32549023628234863, + 0.050980448722839355, + 0.043137311935424805, + 0.14509809017181396, + 0.043137311935424805, + -0.270588219165802, + -0.4431372284889221, + -0.15294116735458374, + 0.22352945804595947, + 0.41960787773132324, + 0.003921627998352051, + -0.7568627595901489, + -0.8509804010391235, + -0.8588235378265381, + -0.9921568632125854, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8588235378265381, + -0.8509804010391235, + -0.8352941274642944, + -0.35686272382736206, + 0.2549020051956177, + 0.3176470994949341, + -0.16862744092941284, + -0.5058823823928833, + -0.3803921341896057, + -0.26274508237838745, + -0.6784313917160034, + -0.8196078538894653, + -0.6313725709915161, + -0.027450978755950928, + 0.45098042488098145, + 0.19215691089630127, + -0.6313725709915161, + -0.8901960849761963, + -0.8588235378265381, + -0.9921568632125854, + -0.9843137264251709, + -0.9764705896377563, + -0.9686274528503418, + -0.9686274528503418, + -0.9529411792755127, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8745098114013672, + -0.8666666746139526, + -0.8509804010391235, + -0.8745098114013672, + -0.5686274766921997, + 0.21568632125854492, + 0.46666669845581055, + -0.03529411554336548, + -0.8352941274642944, + -0.8509804010391235, + -0.7176470756530762, + -0.843137264251709, + -0.8823529481887817, + -0.8117647171020508, + -0.32549017667770386, + 0.3647059202194214, + 0.2705882787704468, + -0.529411792755127, + -0.9058823585510254, + -0.8745098114013672, + -1.0, + -0.9843137264251709, + -0.9764705896377563, + -0.9764705896377563, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.9372549057006836, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8666666746139526, + -0.8588235378265381, + -0.8588235378265381, + -0.7411764860153198, + -0.03529411554336548, + 0.49803924560546875, + 0.301960825920105, + -0.6313725709915161, + -0.8745098114013672, + -0.8352941274642944, + -0.8588235378265381, + -0.8509804010391235, + -0.8666666746139526, + -0.6313725709915161, + 0.21568632125854492, + 0.301960825920105, + -0.498039186000824, + -0.929411768913269, + -0.8901960849761963, + -1.0, + -0.9921568632125854, + -0.9843137264251709, + -0.9764705896377563, + -0.9764705896377563, + -0.9686274528503418, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.8980392217636108, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8588235378265381, + -0.843137264251709, + -0.8352941274642944, + -0.4901960492134094, + 0.23921573162078857, + 0.4117647409439087, + -0.3803921341896057, + -0.8823529481887817, + -0.8274509906768799, + -0.8509804010391235, + -0.8509804010391235, + -0.8666666746139526, + -0.8274509906768799, + -0.05882352590560913, + 0.34117650985717773, + -0.4431372284889221, + -0.9372549057006836, + -0.8901960849761963, + -0.9686274528503418, + -0.9686274528503418, + -0.9529411792755127, + -0.9529411792755127, + -0.9607843160629272, + -0.9607843160629272, + -0.9529411792755127, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9450980424880981, + -0.6941176652908325, + 0.16862750053405762, + 0.4431372880935669, + -0.34117645025253296, + -0.772549033164978, + -0.843137264251709, + -0.8352941274642944, + -0.8274509906768799, + -0.8352941274642944, + -0.843137264251709, + -0.8509804010391235, + -0.8509804010391235, + -0.615686297416687, + -0.2392156720161438, + -0.4901960492134094, + -0.8588235378265381, + -0.8901960849761963, + -0.9058823585510254, + -0.9215686321258545, + -0.929411768913269, + -0.9372549057006836, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9607843160629272, + -0.9607843160629272, + -0.9529411792755127, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9215686321258545, + -0.9372549057006836, + -0.427450954914093, + 0.5137255191802979, + 0.7254902124404907, + 0.20000004768371582, + -0.3960784077644348, + -0.7019608020782471, + -0.8352941274642944, + -0.8196078538894653, + -0.8117647171020508, + -0.7960784435272217, + -0.686274528503418, + -0.46666663885116577, + -0.1294117569923401, + 0.17647063732147217, + -0.04313725233078003, + -0.529411792755127, + -0.8509804010391235, + -0.8980392217636108, + -0.9137254953384399, + -0.9215686321258545, + -0.9372549057006836, + -0.9607843160629272, + -0.9607843160629272, + -0.9607843160629272, + -0.9764705896377563, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.9137254953384399, + -0.929411768913269, + -0.45098036527633667, + 0.4901961088180542, + 0.5529412031173706, + 0.40392160415649414, + 0.19215691089630127, + -0.27843135595321655, + -0.4745097756385803, + -0.3176470398902893, + -0.24705880880355835, + -0.2862744927406311, + 0.003921627998352051, + 0.45098042488098145, + 0.4745098352432251, + 0.27843141555786133, + 0.12156867980957031, + 0.09019613265991211, + -0.5843137502670288, + -0.8823529481887817, + -0.8980392217636108, + -0.9215686321258545, + -0.929411768913269, + -0.9764705896377563, + -0.9764705896377563, + -0.9764705896377563, + -0.9686274528503418, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9058823585510254, + -0.929411768913269, + -0.6235294342041016, + 0.3176470994949341, + 0.4745098352432251, + 0.40392160415649414, + 0.2862745523452759, + 0.011764764785766602, + 0.20000004768371582, + 0.529411792755127, + 0.615686297416687, + 0.45098042488098145, + 0.6313725709915161, + 0.8823529481887817, + 0.7098039388656616, + 0.529411792755127, + 0.17647063732147217, + 0.23921573162078857, + -0.1607843041419983, + -0.843137264251709, + -0.9058823585510254, + -0.9137254953384399, + -0.9058823585510254, + -0.9372549057006836, + -0.9607843160629272, + -0.9686274528503418, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.8980392217636108, + -0.9058823585510254, + -0.8274509906768799, + 0.043137311935424805, + 0.5137255191802979, + 0.30980396270751953, + 0.24705886840820312, + 0.20784318447113037, + 0.5058823823928833, + 0.843137264251709, + 0.8588235378265381, + 0.6549019813537598, + 0.7176470756530762, + 0.8274509906768799, + 0.4431372880935669, + 0.09803926944732666, + 0.09019613265991211, + 0.09019613265991211, + 0.035294175148010254, + -0.7333333492279053, + -0.9215686321258545, + -0.8980392217636108, + -0.8980392217636108, + -0.9137254953384399, + -0.9450980424880981, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.929411768913269, + -0.9137254953384399, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.929411768913269, + -0.3019607663154602, + 0.4431372880935669, + 0.2627451419830322, + 0.23137259483337402, + 0.4117647409439087, + 0.5764706134796143, + 0.6235294342041016, + 0.7333333492279053, + 0.6000000238418579, + 0.615686297416687, + 0.7490196228027344, + 0.20784318447113037, + -0.38823527097702026, + -0.26274508237838745, + -0.12156862020492554, + -0.11372548341751099, + -0.6235294342041016, + -0.9137254953384399, + -0.8745098114013672, + -0.8823529481887817, + -0.8980392217636108, + -0.9137254953384399, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9450980424880981, + -0.929411768913269, + -0.9137254953384399, + -0.9058823585510254, + -0.9058823585510254, + -0.8823529481887817, + -0.929411768913269, + -0.6000000238418579, + 0.30980396270751953, + 0.37254905700683594, + 0.19215691089630127, + 0.2705882787704468, + -0.06666666269302368, + -0.05098038911819458, + 0.4117647409439087, + 0.6549019813537598, + 0.529411792755127, + 0.6549019813537598, + 0.3333333730697632, + -0.26274508237838745, + -0.35686272382736206, + -0.04313725233078003, + -0.3647058606147766, + -0.7960784435272217, + -0.8980392217636108, + -0.8666666746139526, + -0.8745098114013672, + -0.929411768913269, + -0.9450980424880981, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8980392217636108, + -0.8980392217636108, + -0.7803921699523926, + 0.12156867980957031, + 0.2862745523452759, + -0.019607841968536377, + -0.011764705181121826, + -0.37254899740219116, + -0.05098038911819458, + 0.3803921937942505, + 0.5764706134796143, + 0.3333333730697632, + 0.3647059202194214, + 0.5058823823928833, + 0.08235299587249756, + -0.11372548341751099, + 0.2705882787704468, + -0.427450954914093, + -0.9764705896377563, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.9137254953384399, + -0.9450980424880981, + -0.9450980424880981, + -0.9450980424880981, + -0.9450980424880981, + -0.9215686321258545, + -0.9215686321258545, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8980392217636108, + -0.8823529481887817, + -0.8823529481887817, + -0.4745097756385803, + -0.15294116735458374, + 0.06666672229766846, + 0.3176470994949341, + -0.270588219165802, + -0.1450980305671692, + 0.5058823823928833, + 0.49803924560546875, + 0.14509809017181396, + 0.035294175148010254, + 0.3647059202194214, + 0.5137255191802979, + 0.30980396270751953, + 0.38823533058166504, + -0.2078431248664856, + -0.9529411792755127, + -0.9215686321258545, + -0.8745098114013672, + -0.8666666746139526, + -0.9529411792755127, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.7098039388656616, + 0.10588240623474121, + 0.6000000238418579, + 0.34117650985717773, + 0.49803924560546875, + 0.5764706134796143, + 0.4745098352432251, + 0.24705886840820312, + -0.04313725233078003, + 0.19215691089630127, + 0.529411792755127, + 0.45098042488098145, + 0.4117647409439087, + -0.18431371450424194, + -0.8980392217636108, + -0.8901960849761963, + -0.8666666746139526, + -0.8823529481887817, + -0.9764705896377563, + -0.9607843160629272, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.8509804010391235, + -0.7254902124404907, + 0.09019613265991211, + 0.6627451181411743, + 0.6705882549285889, + 0.772549033164978, + 0.529411792755127, + 0.43529415130615234, + 0.09019613265991211, + -0.16862744092941284, + -0.09803920984268188, + 0.3960784673690796, + 0.43529415130615234, + 0.4745098352432251, + -0.15294116735458374, + -0.8980392217636108, + -0.8666666746139526, + -0.8666666746139526, + -0.8666666746139526, + -0.9764705896377563, + -0.9607843160629272, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9137254953384399, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.843137264251709, + -0.6392157077789307, + 0.14509809017181396, + 0.686274528503418, + 0.8274509906768799, + 0.8117647171020508, + 0.6235294342041016, + 0.3490196466445923, + -0.3176470398902893, + -0.26274508237838745, + -0.32549017667770386, + 0.18431377410888672, + 0.48235297203063965, + 0.3960784673690796, + -0.16862744092941284, + -0.8980392217636108, + -0.8745098114013672, + -0.8588235378265381, + -0.8588235378265381, + -0.9764705896377563, + -0.9686274528503418, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8823529481887817, + -0.8666666746139526, + -0.8509804010391235, + -0.6235294342041016, + 0.18431377410888672, + 0.7411764860153198, + 0.6784313917160034, + 0.6627451181411743, + 0.8274509906768799, + 0.458823561668396, + -0.27843135595321655, + -0.30980390310287476, + -0.3176470398902893, + 0.32549023628234863, + 0.615686297416687, + 0.12941181659698486, + -0.4745097756385803, + -0.9058823585510254, + -0.8666666746139526, + -0.8509804010391235, + -0.8666666746139526, + -0.9843137264251709, + -0.9686274528503418, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8588235378265381, + -0.6627451181411743, + 0.08235299587249756, + 0.6000000238418579, + 0.4745098352432251, + 0.40392160415649414, + 0.8588235378265381, + 0.8117647171020508, + 0.09019613265991211, + -0.2862744927406311, + -0.23137253522872925, + 0.5058823823928833, + 0.5372549295425415, + -0.12156862020492554, + -0.6470588445663452, + -0.929411768913269, + -0.8666666746139526, + -0.8509804010391235, + -0.8588235378265381, + -0.9764705896377563, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.8823529481887817, + -0.8745098114013672, + -0.8588235378265381, + -0.8666666746139526, + -0.7019608020782471, + -0.13725489377975464, + 0.24705886840820312, + 0.4117647409439087, + 0.3490196466445923, + 0.6627451181411743, + 0.9058823585510254, + 0.301960825920105, + -0.2549019455909729, + -0.2235293984413147, + 0.18431377410888672, + 0.011764764785766602, + -0.5058823823928833, + -0.7803921699523926, + -0.929411768913269, + -0.8588235378265381, + -0.8509804010391235, + -0.8509804010391235, + -0.9529411792755127, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.9058823585510254, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8745098114013672, + -0.8588235378265381, + -0.8509804010391235, + -0.7176470756530762, + -0.2392156720161438, + -0.05098038911819458, + 0.12156867980957031, + 0.37254905700683594, + 0.19215691089630127, + 0.29411768913269043, + -0.03529411554336548, + -0.3647058606147766, + -0.27843135595321655, + -0.3333333134651184, + -0.5686274766921997, + -0.6470588445663452, + -0.8666666746139526, + -0.8745098114013672, + -0.8509804010391235, + -0.843137264251709, + -0.8509804010391235, + -0.9215686321258545, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8745098114013672, + -0.8588235378265381, + -0.8588235378265381, + -0.7254902124404907, + -0.29411762952804565, + -0.12156862020492554, + -0.03529411554336548, + 0.5686274766921997, + 0.21568632125854492, + -0.27843135595321655, + -0.1764705777168274, + -0.2549019455909729, + -0.21568626165390015, + -0.5686274766921997, + -0.7176470756530762, + -0.7176470756530762, + -0.8745098114013672, + -0.8823529481887817, + -0.843137264251709, + -0.8509804010391235, + -0.8509804010391235, + -0.929411768913269, + -0.929411768913269, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8588235378265381, + -0.8745098114013672, + -0.6470588445663452, + -0.2392156720161438, + -0.03529411554336548, + -0.11372548341751099, + 0.37254905700683594, + 0.545098066329956, + -0.26274508237838745, + -0.09803920984268188, + -0.21568626165390015, + -0.23137253522872925, + -0.30980390310287476, + -0.6627451181411743, + -0.8352941274642944, + -0.8666666746139526, + -0.8588235378265381, + -0.843137264251709, + -0.843137264251709, + -0.8352941274642944, + -0.9529411792755127, + -0.929411768913269, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8823529481887817, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8588235378265381, + -0.8588235378265381, + -0.6000000238418579, + -0.2392156720161438, + 0.035294175148010254, + 0.08235299587249756, + 0.16078436374664307, + 0.5372549295425415, + 0.08235299587249756, + 0.09019613265991211, + -0.1764705777168274, + 0.019607901573181152, + -0.1764705777168274, + -0.5215686559677124, + -0.4901960492134094, + -0.6941176652908325, + -0.8196078538894653, + -0.8352941274642944, + -0.8352941274642944, + -0.843137264251709, + -0.9686274528503418, + -0.9450980424880981, + -0.9450980424880981, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8901960849761963, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8509804010391235, + -0.8588235378265381, + -0.6235294342041016, + -0.1921568512916565, + 0.09803926944732666, + 0.41960787773132324, + 0.20784318447113037, + 0.29411768913269043, + 0.23921573162078857, + -0.06666666269302368, + -0.04313725233078003, + -0.003921568393707275, + -0.34117645025253296, + -0.30980390310287476, + -0.10588234663009644, + -0.3960784077644348, + -0.7490196228027344, + -0.8352941274642944, + -0.8274509906768799, + -0.843137264251709, + -0.9686274528503418, + -0.9529411792755127, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.8980392217636108, + -0.8901960849761963, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.8666666746139526, + -0.8588235378265381, + -0.8666666746139526, + -0.6470588445663452, + -0.19999998807907104, + 0.058823585510253906, + 0.46666669845581055, + 0.46666669845581055, + 0.15294122695922852, + 0.18431377410888672, + -0.05882352590560913, + -0.09803920984268188, + -0.1607843041419983, + -0.45098036527633667, + -0.23137253522872925, + 0.13725495338439941, + -0.15294116735458374, + -0.5686274766921997, + -0.7568627595901489, + -0.8352941274642944, + -0.843137264251709, + -0.9843137264251709, + -0.9607843160629272, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.8588235378265381, + -0.8509804010391235, + -0.8745098114013672, + -0.6784313917160034, + -0.1607843041419983, + 0.06666672229766846, + 0.3176470994949341, + 0.5372549295425415, + 0.4117647409439087, + 0.34117650985717773, + -0.019607841968536377, + -0.1450980305671692, + -0.2862744927406311, + -0.4823529124259949, + -0.06666666269302368, + 0.23921573162078857, + -0.1921568512916565, + -0.5137255191802979, + -0.5058823823928833, + -0.7960784435272217, + -0.8588235378265381, + -0.9764705896377563, + -0.9607843160629272, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8588235378265381, + -0.8509804010391235, + -0.8666666746139526, + -0.6941176652908325, + -0.1450980305671692, + 0.09803926944732666, + 0.20784318447113037, + 0.41960787773132324, + 0.5137255191802979, + 0.38823533058166504, + 0.34117650985717773, + 0.12941181659698486, + -0.09019607305526733, + 0.035294175148010254, + 0.32549023628234863, + 0.301960825920105, + -0.2235293984413147, + -0.7254902124404907, + -0.5529412031173706, + -0.7647058963775635, + -0.8588235378265381, + -0.9607843160629272, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8509804010391235, + -0.8666666746139526, + -0.7333333492279053, + -0.2235293984413147, + 0.11372554302215576, + 0.2705882787704468, + 0.3960784673690796, + 0.48235297203063965, + 0.5607843399047852, + 0.7882353067398071, + 0.40392160415649414, + 0.4117647409439087, + 0.686274528503418, + 0.6627451181411743, + 0.43529415130615234, + -0.18431371450424194, + -0.843137264251709, + -0.8196078538894653, + -0.8274509906768799, + -0.843137264251709, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8745098114013672, + -0.8745098114013672, + -0.8666666746139526, + -0.8509804010391235, + -0.8509804010391235, + -0.8196078538894653, + -0.34117645025253296, + 0.16862750053405762, + 0.49803924560546875, + 0.686274528503418, + 0.686274528503418, + 0.6705882549285889, + 0.686274528503418, + 0.41960787773132324, + 0.6313725709915161, + 0.8352941274642944, + 0.7098039388656616, + 0.45098042488098145, + -0.2235293984413147, + -0.8352941274642944, + -0.843137264251709, + -0.8352941274642944, + -0.8352941274642944, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8823529481887817, + -0.8745098114013672, + -0.8745098114013672, + -0.8588235378265381, + -0.8509804010391235, + -0.843137264251709, + -0.843137264251709, + -0.45098036527633667, + 0.20784318447113037, + 0.5764706134796143, + 0.7647058963775635, + 0.6313725709915161, + 0.6000000238418579, + 0.6941176652908325, + 0.5372549295425415, + 0.4274510145187378, + 0.3803921937942505, + 0.5058823823928833, + 0.529411792755127, + -0.07450979948043823, + -0.7176470756530762, + -0.8352941274642944, + -0.8352941274642944, + -0.843137264251709, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.8588235378265381, + -0.8509804010391235, + -0.843137264251709, + -0.8274509906768799, + -0.6000000238418579, + 0.011764764785766602, + 0.43529415130615234, + 0.5058823823928833, + 0.30980396270751953, + 0.301960825920105, + 0.46666669845581055, + 0.301960825920105, + 0.08235299587249756, + -0.003921568393707275, + 0.3176470994949341, + 0.4745098352432251, + 0.13725495338439941, + -0.4431372284889221, + -0.8274509906768799, + -0.8274509906768799, + -0.843137264251709, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8901960849761963, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.8509804010391235, + -0.8588235378265381, + -0.843137264251709, + -0.8352941274642944, + -0.7490196228027344, + -0.24705880880355835, + 0.2705882787704468, + 0.2862745523452759, + -0.04313725233078003, + -0.019607841968536377, + 0.13725495338439941, + 0.043137311935424805, + -0.3019607663154602, + -0.43529409170150757, + -0.12156862020492554, + 0.21568632125854492, + 0.2705882787704468, + -0.1921568512916565, + -0.7803921699523926, + -0.8352941274642944, + -0.8352941274642944, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.8588235378265381, + -0.8509804010391235, + -0.8509804010391235, + -0.8352941274642944, + -0.8196078538894653, + -0.45098036527633667, + 0.10588240623474121, + 0.23137259483337402, + -0.24705880880355835, + -0.545098066329956, + -0.3803921341896057, + -0.2549019455909729, + -0.6784313917160034, + -0.8039215803146362, + -0.6000000238418579, + -0.03529411554336548, + 0.30980396270751953, + -0.04313725233078003, + -0.7019608020782471, + -0.8588235378265381, + -0.843137264251709, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8509804010391235, + -0.8588235378265381, + -0.843137264251709, + -0.8509804010391235, + -0.615686297416687, + -0.019607841968536377, + 0.2705882787704468, + -0.13725489377975464, + -0.8274509906768799, + -0.8352941274642944, + -0.686274528503418, + -0.8274509906768799, + -0.8509804010391235, + -0.7803921699523926, + -0.30980390310287476, + 0.2627451419830322, + 0.043137311935424805, + -0.6392157077789307, + -0.8823529481887817, + -0.843137264251709, + -0.9686274528503418, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8666666746139526, + -0.8588235378265381, + -0.843137264251709, + -0.8509804010391235, + -0.7568627595901489, + -0.2235293984413147, + 0.23921573162078857, + 0.16862750053405762, + -0.6313725709915161, + -0.843137264251709, + -0.8039215803146362, + -0.8352941274642944, + -0.8196078538894653, + -0.8352941274642944, + -0.6078431606292725, + 0.15294122695922852, + 0.09019613265991211, + -0.615686297416687, + -0.9058823585510254, + -0.8509804010391235, + -0.9686274528503418, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8745098114013672, + -0.8588235378265381, + -0.8509804010391235, + -0.843137264251709, + -0.8274509906768799, + -0.5607843399047852, + -0.003921568393707275, + 0.23137259483337402, + -0.40392154455184937, + -0.8509804010391235, + -0.7960784435272217, + -0.8274509906768799, + -0.8196078538894653, + -0.8352941274642944, + -0.7960784435272217, + -0.09803920984268188, + 0.14509809017181396, + -0.5529412031173706, + -0.9215686321258545, + -0.8666666746139526, + -0.9529411792755127, + -0.9529411792755127, + -0.9450980424880981, + -0.9529411792755127, + -0.9529411792755127, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9450980424880981, + -0.7333333492279053, + 0.14509809017181396, + 0.38823533058166504, + -0.43529409170150757, + -0.8039215803146362, + -0.8588235378265381, + -0.8509804010391235, + -0.8588235378265381, + -0.8666666746139526, + -0.8823529481887817, + -0.8509804010391235, + -0.8588235378265381, + -0.615686297416687, + -0.19999998807907104, + -0.4588235020637512, + -0.8666666746139526, + -0.8901960849761963, + -0.9058823585510254, + -0.9215686321258545, + -0.9215686321258545, + -0.929411768913269, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9529411792755127, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9137254953384399, + -0.9372549057006836, + -0.4745097756385803, + 0.5137255191802979, + 0.7411764860153198, + 0.08235299587249756, + -0.545098066329956, + -0.7647058963775635, + -0.8509804010391235, + -0.8352941274642944, + -0.8274509906768799, + -0.8196078538894653, + -0.7490196228027344, + -0.5607843399047852, + -0.2392156720161438, + 0.09019613265991211, + -0.003921568393707275, + -0.4901960492134094, + -0.8509804010391235, + -0.8980392217636108, + -0.9137254953384399, + -0.9137254953384399, + -0.929411768913269, + -0.9372549057006836, + -0.9450980424880981, + -0.9450980424880981, + -0.9607843160629272, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.929411768913269, + -0.9058823585510254, + -0.9372549057006836, + -0.4823529124259949, + 0.48235297203063965, + 0.5529412031173706, + 0.3176470994949341, + -0.04313725233078003, + -0.4745097756385803, + -0.5843137502670288, + -0.46666663885116577, + -0.43529409170150757, + -0.5372549295425415, + -0.27843135595321655, + 0.2862745523452759, + 0.3490196466445923, + 0.08235299587249756, + 0.050980448722839355, + 0.13725495338439941, + -0.5607843399047852, + -0.8901960849761963, + -0.8980392217636108, + -0.929411768913269, + -0.929411768913269, + -0.9607843160629272, + -0.9607843160629272, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.9215686321258545, + -0.6470588445663452, + 0.2862745523452759, + 0.43529415130615234, + 0.3490196466445923, + 0.17647063732147217, + -0.2549019455909729, + -0.11372548341751099, + 0.29411768913269043, + 0.3333333730697632, + -0.019607841968536377, + 0.2627451419830322, + 0.8274509906768799, + 0.6705882549285889, + 0.3490196466445923, + 0.003921627998352051, + 0.22352945804595947, + -0.13725489377975464, + -0.843137264251709, + -0.8980392217636108, + -0.9215686321258545, + -0.9137254953384399, + -0.9372549057006836, + -0.9529411792755127, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8980392217636108, + -0.8196078538894653, + -0.003921568393707275, + 0.4745098352432251, + 0.2705882787704468, + 0.16078436374664307, + -0.07450979948043823, + 0.20784318447113037, + 0.7882353067398071, + 0.7176470756530762, + 0.20784318447113037, + 0.27843141555786133, + 0.6549019813537598, + 0.37254905700683594, + -0.04313725233078003, + -0.1294117569923401, + -0.011764705181121826, + -0.019607841968536377, + -0.7411764860153198, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.9372549057006836, + -0.9372549057006836, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9058823585510254, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.9058823585510254, + -0.34117645025253296, + 0.40392160415649414, + 0.20784318447113037, + 0.12941181659698486, + 0.12156867980957031, + 0.3333333730697632, + 0.5529412031173706, + 0.5764706134796143, + 0.17647063732147217, + 0.12941181659698486, + 0.40392160415649414, + 0.043137311935424805, + -0.3960784077644348, + -0.40392154455184937, + -0.29411762952804565, + -0.2549019455909729, + -0.6941176652908325, + -0.9215686321258545, + -0.8901960849761963, + -0.8823529481887817, + -0.9215686321258545, + -0.9137254953384399, + -0.9372549057006836, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8980392217636108, + -0.8823529481887817, + -0.9137254953384399, + -0.6392157077789307, + 0.20784318447113037, + 0.2862745523452759, + 0.035294175148010254, + -0.04313725233078003, + -0.2078431248664856, + -0.027450978755950928, + 0.18431377410888672, + 0.24705886840820312, + 0.058823585510253906, + 0.21568632125854492, + 0.08235299587249756, + -0.24705880880355835, + -0.43529409170150757, + -0.2235293984413147, + -0.4745097756385803, + -0.843137264251709, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9137254953384399, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8901960849761963, + -0.8901960849761963, + -0.8039215803146362, + -0.09019607305526733, + 0.11372554302215576, + -0.1921568512916565, + -0.26274508237838745, + -0.3960784077644348, + 0.043137311935424805, + 0.13725495338439941, + 0.12156867980957031, + -0.1294117569923401, + -0.1294117569923401, + 0.12941181659698486, + -0.09019607305526733, + -0.3490195870399475, + 0.019607901573181152, + -0.4901960492134094, + -0.9529411792755127, + -0.8901960849761963, + -0.8823529481887817, + -0.8823529481887817, + -0.9137254953384399, + -0.929411768913269, + -0.929411768913269, + -0.9372549057006836, + -0.9372549057006836, + -0.9137254953384399, + -0.9137254953384399, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.6078431606292725, + -0.3490195870399475, + -0.1294117569923401, + 0.043137311935424805, + -0.3803921341896057, + -0.2392156720161438, + 0.16078436374664307, + 0.019607901573181152, + -0.27843135595321655, + -0.38823527097702026, + -0.027450978755950928, + 0.20784318447113037, + -0.003921568393707275, + 0.08235299587249756, + -0.3019607663154602, + -0.9372549057006836, + -0.9058823585510254, + -0.8588235378265381, + -0.8588235378265381, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8745098114013672, + -0.8745098114013672, + -0.7647058963775635, + -0.09803920984268188, + 0.24705886840820312, + 0.011764764785766602, + 0.2549020051956177, + 0.21568632125854492, + 0.043137311935424805, + -0.09803920984268188, + -0.3019607663154602, + -0.12156862020492554, + 0.23137259483337402, + 0.18431377410888672, + 0.13725495338439941, + -0.29411762952804565, + -0.8980392217636108, + -0.8823529481887817, + -0.8588235378265381, + -0.8666666746139526, + -0.9686274528503418, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8823529481887817, + -0.8666666746139526, + -0.8509804010391235, + -0.7647058963775635, + -0.09019607305526733, + 0.32549023628234863, + 0.3647059202194214, + 0.5921568870544434, + 0.23921573162078857, + 0.11372554302215576, + -0.11372548341751099, + -0.270588219165802, + -0.2235293984413147, + 0.16078436374664307, + 0.22352945804595947, + 0.34117650985717773, + -0.21568626165390015, + -0.8980392217636108, + -0.8588235378265381, + -0.843137264251709, + -0.8509804010391235, + -0.9607843160629272, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9137254953384399, + -0.9058823585510254, + -0.9058823585510254, + -0.8980392217636108, + -0.8823529481887817, + -0.8823529481887817, + -0.8823529481887817, + -0.8745098114013672, + -0.8588235378265381, + -0.6941176652908325, + 0.003921627998352051, + 0.48235297203063965, + 0.7019608020782471, + 0.7176470756530762, + 0.4274510145187378, + 0.18431377410888672, + -0.3647058606147766, + -0.2549019455909729, + -0.3176470398902893, + 0.09803926944732666, + 0.3647059202194214, + 0.37254905700683594, + -0.16862744092941284, + -0.8980392217636108, + -0.8666666746139526, + -0.843137264251709, + -0.8509804010391235, + -0.9607843160629272, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8823529481887817, + -0.8745098114013672, + -0.8509804010391235, + -0.6549019813537598, + 0.10588240623474121, + 0.6784313917160034, + 0.6313725709915161, + 0.6235294342041016, + 0.7647058963775635, + 0.40392160415649414, + -0.2862744927406311, + -0.2862744927406311, + -0.29411762952804565, + 0.29411768913269043, + 0.5843137502670288, + 0.16078436374664307, + -0.4745097756385803, + -0.9058823585510254, + -0.8588235378265381, + -0.8509804010391235, + -0.8588235378265381, + -0.9686274528503418, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8823529481887817, + -0.8823529481887817, + -0.8745098114013672, + -0.8509804010391235, + -0.686274528503418, + -0.003921568393707275, + 0.5529412031173706, + 0.45098042488098145, + 0.37254905700683594, + 0.8509804010391235, + 0.7960784435272217, + 0.08235299587249756, + -0.26274508237838745, + -0.2078431248664856, + 0.4745098352432251, + 0.545098066329956, + -0.12156862020492554, + -0.6627451181411743, + -0.9137254953384399, + -0.843137264251709, + -0.843137264251709, + -0.8509804010391235, + -0.9607843160629272, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8980392217636108, + -0.8823529481887817, + -0.8823529481887817, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8745098114013672, + -0.7490196228027344, + -0.270588219165802, + 0.11372554302215576, + 0.3647059202194214, + 0.32549023628234863, + 0.6705882549285889, + 0.9058823585510254, + 0.2705882787704468, + -0.26274508237838745, + -0.23137253522872925, + 0.20000004768371582, + 0.019607901573181152, + -0.5686274766921997, + -0.8117647171020508, + -0.9137254953384399, + -0.843137264251709, + -0.8274509906768799, + -0.843137264251709, + -0.9450980424880981, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8980392217636108, + -0.8823529481887817, + -0.8745098114013672, + -0.8823529481887817, + -0.8823529481887817, + -0.8588235378265381, + -0.843137264251709, + -0.772549033164978, + -0.3960784077644348, + -0.2235293984413147, + 0.003921627998352051, + 0.301960825920105, + 0.16078436374664307, + 0.2705882787704468, + -0.04313725233078003, + -0.2862744927406311, + -0.13725489377975464, + -0.1607843041419983, + -0.5137255191802979, + -0.7333333492279053, + -0.8901960849761963, + -0.8823529481887817, + -0.843137264251709, + -0.8196078538894653, + -0.8274509906768799, + -0.9137254953384399, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8823529481887817, + -0.8823529481887817, + -0.8588235378265381, + -0.8274509906768799, + -0.7803921699523926, + -0.46666663885116577, + -0.29411762952804565, + -0.16862744092941284, + 0.46666669845581055, + 0.15294122695922852, + -0.3176470398902893, + -0.11372548341751099, + -0.06666666269302368, + 0.011764764785766602, + -0.46666663885116577, + -0.7098039388656616, + -0.772549033164978, + -0.8823529481887817, + -0.8823529481887817, + -0.843137264251709, + -0.8352941274642944, + -0.8196078538894653, + -0.9137254953384399, + -0.9215686321258545, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8745098114013672, + -0.8588235378265381, + -0.843137264251709, + -0.843137264251709, + -0.7568627595901489, + -0.45098036527633667, + -0.2235293984413147, + -0.24705880880355835, + 0.27843141555786133, + 0.4745098352432251, + -0.30980390310287476, + -0.07450979948043823, + -0.1294117569923401, + -0.13725489377975464, + -0.3176470398902893, + -0.7176470756530762, + -0.8745098114013672, + -0.8901960849761963, + -0.8588235378265381, + -0.8274509906768799, + -0.8352941274642944, + -0.8274509906768799, + -0.9450980424880981, + -0.9215686321258545, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8745098114013672, + -0.8745098114013672, + -0.8745098114013672, + -0.8509804010391235, + -0.843137264251709, + -0.8509804010391235, + -0.7098039388656616, + -0.4901960492134094, + -0.19999998807907104, + -0.12156862020492554, + 0.035294175148010254, + 0.4431372880935669, + 0.050980448722839355, + 0.12156867980957031, + -0.1450980305671692, + 0.011764764785766602, + -0.19999998807907104, + -0.6235294342041016, + -0.6313725709915161, + -0.7960784435272217, + -0.843137264251709, + -0.8196078538894653, + -0.8274509906768799, + -0.8274509906768799, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8823529481887817, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8509804010391235, + -0.843137264251709, + -0.8588235378265381, + -0.7254902124404907, + -0.4588235020637512, + -0.16862744092941284, + 0.10588240623474121, + 0.043137311935424805, + 0.22352945804595947, + 0.20000004768371582, + -0.08235293626785278, + -0.019607841968536377, + 0.019607901573181152, + -0.38823527097702026, + -0.4745097756385803, + -0.29411762952804565, + -0.5372549295425415, + -0.7960784435272217, + -0.8352941274642944, + -0.8196078538894653, + -0.8274509906768799, + -0.9529411792755127, + -0.9450980424880981, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8901960849761963, + -0.8823529481887817, + -0.8823529481887817, + -0.8745098114013672, + -0.8588235378265381, + -0.8509804010391235, + -0.843137264251709, + -0.8509804010391235, + -0.7333333492279053, + -0.43529409170150757, + -0.1921568512916565, + 0.11372554302215576, + 0.16862750053405762, + 0.035294175148010254, + 0.14509809017181396, + -0.11372548341751099, + -0.11372548341751099, + -0.1607843041419983, + -0.529411792755127, + -0.43529409170150757, + -0.08235293626785278, + -0.3176470398902893, + -0.6784313917160034, + -0.8039215803146362, + -0.8196078538894653, + -0.8274509906768799, + -0.9686274528503418, + -0.9529411792755127, + -0.9450980424880981, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8980392217636108, + -0.8823529481887817, + -0.8823529481887817, + -0.8745098114013672, + -0.8588235378265381, + -0.8588235378265381, + -0.8352941274642944, + -0.8588235378265381, + -0.7568627595901489, + -0.41960781812667847, + -0.19999998807907104, + 0.003921627998352051, + 0.20000004768371582, + 0.19215691089630127, + 0.2549020051956177, + -0.08235293626785278, + -0.2078431248664856, + -0.3803921341896057, + -0.6313725709915161, + -0.30980390310287476, + 0.011764764785766602, + -0.34117645025253296, + -0.6392157077789307, + -0.6392157077789307, + -0.7960784435272217, + -0.8352941274642944, + -0.9607843160629272, + -0.9529411792755127, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8666666746139526, + -0.843137264251709, + -0.8588235378265381, + -0.772549033164978, + -0.427450954914093, + -0.2078431248664856, + -0.08235293626785278, + 0.12941181659698486, + 0.27843141555786133, + 0.27843141555786133, + 0.301960825920105, + 0.058823585510253906, + -0.26274508237838745, + -0.19999998807907104, + 0.07450985908508301, + 0.11372554302215576, + -0.32549017667770386, + -0.7882353067398071, + -0.6392157077789307, + -0.7803921699523926, + -0.8588235378265381, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9215686321258545, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8588235378265381, + -0.8666666746139526, + -0.8509804010391235, + -0.8588235378265381, + -0.7882353067398071, + -0.4588235020637512, + -0.19999998807907104, + -0.05098038911819458, + 0.08235299587249756, + 0.20000004768371582, + 0.4274510145187378, + 0.7568627595901489, + 0.2862745523452759, + 0.19215691089630127, + 0.45098042488098145, + 0.458823561668396, + 0.2862745523452759, + -0.30980390310287476, + -0.8509804010391235, + -0.8196078538894653, + -0.8196078538894653, + -0.843137264251709, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9215686321258545, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.843137264251709, + -0.843137264251709, + -0.8352941274642944, + -0.843137264251709, + -0.529411792755127, + -0.1294117569923401, + 0.21568632125854492, + 0.3960784673690796, + 0.4117647409439087, + 0.4745098352432251, + 0.5843137502670288, + 0.2627451419830322, + 0.43529415130615234, + 0.6549019813537598, + 0.5607843399047852, + 0.30980396270751953, + -0.3960784077644348, + -0.8666666746139526, + -0.843137264251709, + -0.8196078538894653, + -0.8117647171020508, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8745098114013672, + -0.8745098114013672, + -0.8666666746139526, + -0.8274509906768799, + -0.8274509906768799, + -0.8274509906768799, + -0.8588235378265381, + -0.5843137502670288, + -0.03529411554336548, + 0.3960784673690796, + 0.5764706134796143, + 0.4117647409439087, + 0.40392160415649414, + 0.5686274766921997, + 0.4274510145187378, + 0.301960825920105, + 0.2549020051956177, + 0.38823533058166504, + 0.41960787773132324, + -0.270588219165802, + -0.8274509906768799, + -0.8352941274642944, + -0.8196078538894653, + -0.8196078538894653, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8509804010391235, + -0.8352941274642944, + -0.8274509906768799, + -0.843137264251709, + -0.843137264251709, + -0.686274528503418, + -0.2078431248664856, + 0.2862745523452759, + 0.32549023628234863, + 0.10588240623474121, + 0.13725495338439941, + 0.3490196466445923, + 0.23921573162078857, + -0.003921568393707275, + -0.07450979948043823, + 0.2549020051956177, + 0.38823533058166504, + -0.06666666269302368, + -0.6313725709915161, + -0.8352941274642944, + -0.8274509906768799, + -0.8352941274642944, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8823529481887817, + -0.8823529481887817, + -0.8745098114013672, + -0.8588235378265381, + -0.8509804010391235, + -0.843137264251709, + -0.843137264251709, + -0.8509804010391235, + -0.7882353067398071, + -0.43529409170150757, + 0.10588240623474121, + 0.11372554302215576, + -0.23137253522872925, + -0.1764705777168274, + 0.027451038360595703, + -0.027450978755950928, + -0.37254899740219116, + -0.4901960492134094, + -0.1764705777168274, + 0.09803926944732666, + 0.019607901573181152, + -0.46666663885116577, + -0.8352941274642944, + -0.8352941274642944, + -0.8352941274642944, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.8509804010391235, + -0.8588235378265381, + -0.8588235378265381, + -0.843137264251709, + -0.843137264251709, + -0.8352941274642944, + -0.5921568870544434, + -0.11372548341751099, + 0.035294175148010254, + -0.3960784077644348, + -0.615686297416687, + -0.4431372284889221, + -0.3176470398902893, + -0.7098039388656616, + -0.8196078538894653, + -0.6392157077789307, + -0.1450980305671692, + 0.07450985908508301, + -0.3333333134651184, + -0.8117647171020508, + -0.843137264251709, + -0.8196078538894653, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.929411768913269, + -0.929411768913269, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.8666666746139526, + -0.8588235378265381, + -0.8588235378265381, + -0.8509804010391235, + -0.843137264251709, + -0.843137264251709, + -0.8588235378265381, + -0.7254902124404907, + -0.32549017667770386, + -0.011764705181121826, + -0.3019607663154602, + -0.8509804010391235, + -0.8274509906768799, + -0.7019608020782471, + -0.8352941274642944, + -0.8588235378265381, + -0.7960784435272217, + -0.38823527097702026, + 0.058823585510253906, + -0.2392156720161438, + -0.772549033164978, + -0.8666666746139526, + -0.8274509906768799, + -0.9607843160629272, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.929411768913269, + -0.9215686321258545, + -0.9137254953384399, + -0.8980392217636108, + -0.8823529481887817, + -0.8745098114013672, + -0.8666666746139526, + -0.8588235378265381, + -0.8588235378265381, + -0.843137264251709, + -0.8509804010391235, + -0.8509804010391235, + -0.8039215803146362, + -0.4901960492134094, + -0.07450979948043823, + -0.04313725233078003, + -0.686274528503418, + -0.8274509906768799, + -0.8117647171020508, + -0.8509804010391235, + -0.8196078538894653, + -0.8274509906768799, + -0.6313725709915161, + -0.019607841968536377, + -0.1921568512916565, + -0.7568627595901489, + -0.8745098114013672, + -0.8352941274642944, + -0.9607843160629272, + -0.9529411792755127, + -0.9450980424880981, + -0.9372549057006836, + -0.9372549057006836, + -0.9215686321258545, + -0.9137254953384399, + -0.9058823585510254, + -0.8901960849761963, + -0.8823529481887817, + -0.8745098114013672, + -0.8745098114013672, + -0.8666666746139526, + -0.8588235378265381, + -0.8588235378265381, + -0.8588235378265381, + -0.843137264251709, + -0.7019608020782471, + -0.3019607663154602, + -0.011764705181121826, + -0.4745097756385803, + -0.843137264251709, + -0.7960784435272217, + -0.8274509906768799, + -0.8196078538894653, + -0.8196078538894653, + -0.7803921699523926, + -0.19999998807907104, + -0.09803920984268188, + -0.686274528503418, + -0.8980392217636108, + -0.8588235378265381 + ] + ] +} \ No newline at end of file diff --git a/neurons/deployment_layer/model_net/input.py b/neurons/deployment_layer/model_net/input.py new file mode 100644 index 00000000..a24f2ce2 --- /dev/null +++ b/neurons/deployment_layer/model_net/input.py @@ -0,0 +1,42 @@ +from __future__ import annotations +from pydantic import BaseModel +from execution_layer.base_input import BaseInput +from execution_layer.input_registry import InputRegistry +from _validator.models.request_type import RequestType +import random + + +class NetInputSchema(BaseModel): + # DSperse runner expects an input.json shaped like {"input_data": ...} + # Keep this as a list of lists of floats to be compatible with generic handling. + input_data: list[list[float]] + + +@InputRegistry.register("model_net") +class NetInput(BaseInput): + """ + Input generator/validator for the model_net DSperse deployment. + Produces a simple vector input wrapped under key "input_data". + """ + + schema = NetInputSchema + + def __init__(self, request_type: RequestType, data: dict[str, object] | None = None): + super().__init__(request_type, data) + + @staticmethod + def generate() -> dict[str, object]: + # Generate a simple 1x16 vector of floats in [0,1). Adjust the length if your model expects a different size. + length = 16 + return { + "input_data": [[random.random() for _ in range(length)]], + } + + @staticmethod + def validate(data: dict[str, object]) -> None: + return NetInputSchema(**data) + + @staticmethod + def process(data: dict[str, object]) -> dict[str, object]: + # No additional processing required; passthrough. + return data diff --git a/neurons/deployment_layer/model_net/metadata.json b/neurons/deployment_layer/model_net/metadata.json new file mode 100644 index 00000000..6effc79f --- /dev/null +++ b/neurons/deployment_layer/model_net/metadata.json @@ -0,0 +1,14 @@ +{ + "id": "model_net", + "name": "Model Net (DSperse)", + "description": "Net model prepared for DSperse per-slice proving.", + "author": "Inference Labs", + "version": "1.0.0", + "proof_system": "dsperse", + "netuid": 2, + "benchmark_choice_weight": 0.2, + "dsperse": { + "slices_dir": "slices", + "default_slice": "slice_0.dslice" + } +} diff --git a/neurons/execution_layer/proof_handlers/dsperse_handler.py b/neurons/execution_layer/proof_handlers/dsperse_handler.py new file mode 100644 index 00000000..bd8c402c --- /dev/null +++ b/neurons/execution_layer/proof_handlers/dsperse_handler.py @@ -0,0 +1,229 @@ +from __future__ import annotations +import json +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Tuple + +import bittensor as bt + +from execution_layer.proof_handlers.base_handler import ProofSystemHandler +from execution_layer.generic_input import GenericInput + +# Library-mode imports from dsperse +from dsperse.src.run.runner import Runner as DsperseRunner +from dsperse.src.prover import Prover as DsperseProver +from dsperse.src.verifier import Verifier as DsperseVerifier + +if TYPE_CHECKING: + from execution_layer.verified_model_session import VerifiedModelSession + + +@dataclass +class DsperseConfig: + dslice_path: str # Absolute path to the single slice .dslice file to operate on + run_root: str # Directory under which run_YYYYMMDD_HHMMSS/ folders are created + + +class DsperseHandler(ProofSystemHandler): + """ + DSperse handler (library mode, per-slice): + - generate_witness() → executes Runner.run on a single .dslice and writes a timestamped run directory + - gen_proof() → executes Prover.prove for that run and slice; returns proof.json contents + - verify_proof() → executes Verifier.verify for that run and slice; returns boolean + + Slicing/compilation are performed offline; each .dslice is expected to contain compiled EZKL artifacts. + """ + + # ------------- + # High-level API expected by the system + # ------------- + + def gen_input_file(self, session: VerifiedModelSession): + bt.logging.trace("[DSperse] Generating input file") + if isinstance(session.inputs.data, list): + input_data = session.inputs.data + else: + input_data = session.inputs.to_array() + data = {"input_data": input_data} + os.makedirs(os.path.dirname(session.session_storage.input_path), exist_ok=True) + with open(session.session_storage.input_path, "w", encoding="utf-8") as f: + json.dump(data, f) + bt.logging.trace(f"[DSperse] Wrote input.json → {session.session_storage.input_path}") + + def generate_witness( + self, session: VerifiedModelSession, return_content: bool = False + ) -> list | dict: + """ + Run DSperse runner for a single slice. Returns the created run directory or, if requested, + the parsed run_results.json content. + """ + cfg = self._resolve_config(session) + self._ensure_dsperse_available() + + os.makedirs(cfg.run_root, exist_ok=True) + bt.logging.debug(f"[DSperse] Running slice with runner: dslice={cfg.dslice_path} run_root={cfg.run_root}") + runner = DsperseRunner(slice_path=cfg.dslice_path, run_metadata_path=None, save_metadata_path=None) + + # Runner.run accepts output_path as a root; it will create run_YYYYMMDD_HHMMSS under it + runner.run(session.session_storage.input_path, output_path=cfg.run_root) + + # Select the latest run directory created under run_root + run_dir = self._select_latest_run(cfg.run_root) + if not run_dir: + raise RuntimeError("[DSperse] No run directory found after runner.run()") + + if return_content: + results_path = os.path.join(run_dir, "run_results.json") + if os.path.exists(results_path): + with open(results_path, "r", encoding="utf-8") as f: + return json.load(f) + return {"run_dir": run_dir} + return [run_dir] + + def gen_proof(self, session: VerifiedModelSession) -> Tuple[str, str]: + """ + Generate a proof for the selected slice and latest run under run_root. Returns + (proof_json_str, instances_json_str). If instances are not present, returns "[]" for instances. + """ + cfg = self._resolve_config(session) + self._ensure_dsperse_available() + + run_dir = self._select_latest_run(cfg.run_root) + if not run_dir: + raise RuntimeError("[DSperse] No run directory available for proving. Run generate_witness() first.") + + bt.logging.debug(f"[DSperse] Proving slice: run_dir={run_dir} dslice={cfg.dslice_path}") + prover = DsperseProver() + _ = prover.prove(run_dir, cfg.dslice_path) # updates run_results.json and writes proof.json + + proof_path = self._locate_proof_json(run_dir) + if not proof_path or not os.path.exists(proof_path): + raise RuntimeError("[DSperse] Proof file not found after proving.") + + with open(proof_path, "r", encoding="utf-8") as f: + proof_json = json.load(f) + instances = proof_json.get("instances", []) + return json.dumps(proof_json), json.dumps(instances) + + def verify_proof( + self, + session: VerifiedModelSession, + validator_inputs: GenericInput, # not used by DSperse verify + proof: dict | str, # not used; verify reads from run_dir + dslice + ) -> bool: + cfg = self._resolve_config(session) + self._ensure_dsperse_available() + + run_dir = self._select_latest_run(cfg.run_root) + if not run_dir: + bt.logging.error("[DSperse] No run directory found for verification.") + return False + + bt.logging.debug(f"[DSperse] Verifying slice: run_dir={run_dir} dslice={cfg.dslice_path}") + verifier = DsperseVerifier() + results = verifier.verify(run_dir, cfg.dslice_path) + + try: + exec_chain = results.get("execution_chain", {}) + verified = int(exec_chain.get("ezkl_verified_slices", 0)) + proved = int(exec_chain.get("ezkl_proved_slices", 0)) or 1 # treat as 1 if single slice + return verified >= min(1, proved) + except Exception: + # Fallback: check if any entry has verification_execution.verified truthy + try: + for entry in results.get("execution_chain", {}).get("execution_results", []): + ve = entry.get("verification_execution", {}) + if ve and (ve.get("verified") or ve.get("success") or ve.get("success") is True): + return True + except Exception: + pass + return False + + def aggregate_proofs(self, session: VerifiedModelSession, proofs: list[str]) -> tuple[str, float]: + # Non-cryptographic aggregation: return a manifest of provided proof JSONs + try: + parts = [json.loads(p) if isinstance(p, str) else p for p in proofs] + except Exception: + parts = proofs + manifest = {"type": "dsperse.aggregate", "parts": parts} + return json.dumps(manifest), 0.0 + + # ------------- + # Helpers + # ------------- + + def _resolve_config(self, session: VerifiedModelSession) -> DsperseConfig: + """ + Resolve dslice_path and run_root from session.model.settings['dsperse']. + Expected structure: + session.model.settings = { ..., "dsperse": { "dslice_path": "/abs/path/to/slice_0.dslice", + "run_root": "/abs/path/to/model/run" } } + Fallbacks use session.model.paths.root when present. + """ + settings = getattr(session.model, "settings", {}) or {} + dsp = settings.get("dsperse", {}) if isinstance(settings, dict) else {} + + dslice_path = dsp.get("dslice_path") + run_root = dsp.get("run_root") + + root = getattr(session.model.paths, "root", None) + + if not dslice_path and root: + # If only one dslice exists under /slices, pick it + slices_dir = os.path.join(root, "slices") + if os.path.isdir(slices_dir): + candidates = [os.path.join(slices_dir, f) for f in os.listdir(slices_dir) if f.endswith(".dslice")] + if len(candidates) == 1: + dslice_path = candidates[0] + + if not run_root and root: + run_root = os.path.join(root, "run") + + if not dslice_path or not run_root: + raise RuntimeError( + "[DSperse] Missing configuration: set session.model.settings['dsperse'] with 'dslice_path' and 'run_root'." + ) + return DsperseConfig(dslice_path=os.path.abspath(dslice_path), run_root=os.path.abspath(run_root)) + + def _select_latest_run(self, run_root: str) -> Optional[str]: + if not os.path.isdir(run_root): + return None + runs = [os.path.join(run_root, d) for d in os.listdir(run_root) if d.startswith("run_")] + runs = [d for d in runs if os.path.isdir(d)] + if not runs: + return None + runs.sort(key=lambda p: os.path.getmtime(p), reverse=True) + return runs[0] + + def _locate_proof_json(self, run_dir: str) -> Optional[str]: + """Search under run_dir for slice_* subdirs containing proof.json and return the first found.""" + try: + for name in sorted(os.listdir(run_dir)): + if name.startswith("slice_"): + candidate = os.path.join(run_dir, name, "proof.json") + if os.path.exists(candidate): + return candidate + except Exception: + pass + # As a fallback, consult run_results.json for recorded proof paths + rr = os.path.join(run_dir, "run_results.json") + if os.path.exists(rr): + try: + with open(rr, "r", encoding="utf-8") as f: + run_results = json.load(f) + for entry in run_results.get("execution_chain", {}).get("execution_results", []): + pe = entry.get("proof_execution", {}) + # different keys observed: proof_file or proof_path + path = pe.get("proof_file") or pe.get("proof_path") + if path and os.path.exists(path): + return path + except Exception: + pass + return None + + def _ensure_dsperse_available(self) -> None: + if DsperseRunner is None or DsperseProver is None or DsperseVerifier is None: + raise ImportError( + "DSperse library not available. Please ensure 'dsperse' is installed and importable." + ) +