From 790fb3475e852aeee6826f0f2f26dc170e037d49 Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Tue, 6 Feb 2024 12:11:20 -0800 Subject: [PATCH 1/6] Add include_stats --- js/src/client.ts | 5 +++++ python/langsmith/client.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/js/src/client.ts b/js/src/client.ts index 9f6369e0f..181a5cd4d 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -822,9 +822,11 @@ export class Client { public async readProject({ projectId, projectName, + includeStats, }: { projectId?: string; projectName?: string; + includeStats?: boolean; }): Promise { let path = "/sessions"; const params = new URLSearchParams(); @@ -838,6 +840,9 @@ export class Client { } else { throw new Error("Must provide projectName or projectId"); } + if (includeStats !== undefined) { + params.append("include_stats", includeStats.toString()); + } const response = await this._get( path, diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 144a22545..f9d39b5b9 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -950,7 +950,7 @@ def create_run( self._insert_runtime_env([run_create]) if revision_id is not None: - run_create["extra"]["metadata"]["revision_id"] = revision_id + run_create["extra"]["metadata"]["nrevision_id"] = revision_id if ( self.tracing_queue is not None # batch ingest requires trace_id and dotted_order to be set @@ -1609,7 +1609,11 @@ def _get_tenant_id(self) -> uuid.UUID: @ls_utils.xor_args(("project_id", "project_name")) def read_project( - self, *, project_id: Optional[str] = None, project_name: Optional[str] = None + self, + *, + project_id: Optional[str] = None, + project_name: Optional[str] = None, + include_stats: bool = False, ) -> ls_schemas.TracerSessionResult: """Read a project from the LangSmith API. @@ -1620,6 +1624,8 @@ def read_project( project_name : str or None, default=None The name of the project to read. Note: Only one of project_id or project_name may be given. + include_stats : bool, default=False + Whether to include a project's aggregate statistics in the response. Returns ------- @@ -1634,6 +1640,7 @@ def read_project( params["name"] = project_name else: raise ValueError("Must provide project_name or project_id") + params["include_stats"] = include_stats response = self._get_with_retries(path, params=params) result = response.json() if isinstance(result, list): From 4d31af207db801e7eea05805e4ded2abbb7fe042 Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Tue, 6 Feb 2024 12:15:36 -0800 Subject: [PATCH 2/6] fix nrevision --- python/langsmith/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index f9d39b5b9..f535051f8 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -909,6 +909,7 @@ def create_run( run_type: str, *, project_name: Optional[str] = None, + revision_id: Optional[ID_TYPE] = None, **kwargs: Any, ) -> None: """Persist a run to the LangSmith API. @@ -922,6 +923,8 @@ def create_run( run_type : str The type of the run, such as tool, chain, llm, retriever, embedding, prompt, or parser. + revision_id : ID_TYPE or None, default=None + The revision ID of the run. **kwargs : Any Additional keyword arguments. @@ -935,7 +938,6 @@ def create_run( # if the project is not provided, use the environment's project ls_utils.get_tracer_project(), ) - revision_id = kwargs.pop("revision_id", None) run_create = { **kwargs, "session_name": project_name, @@ -950,7 +952,7 @@ def create_run( self._insert_runtime_env([run_create]) if revision_id is not None: - run_create["extra"]["metadata"]["nrevision_id"] = revision_id + run_create["extra"]["metadata"]["revision_id"] = revision_id if ( self.tracing_queue is not None # batch ingest requires trace_id and dotted_order to be set From e371499461a65786875af6aca7e65e7a0cfd895b Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Tue, 6 Feb 2024 12:54:05 -0800 Subject: [PATCH 3/6] rm flakey --- python/tests/integration_tests/test_client.py | 78 +------------------ 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/python/tests/integration_tests/test_client.py b/python/tests/integration_tests/test_client.py index 54dbd7f83..d6e4ecff5 100644 --- a/python/tests/integration_tests/test_client.py +++ b/python/tests/integration_tests/test_client.py @@ -6,7 +6,7 @@ import string import time from datetime import datetime, timedelta -from typing import List, Optional, cast +from typing import cast from uuid import uuid4 import pytest @@ -14,13 +14,10 @@ from langchain.schema import FunctionMessage, HumanMessage from langsmith.client import Client -from langsmith.evaluation import EvaluationResult, StringEvaluator -from langsmith.run_trees import RunTree from langsmith.schemas import DataType from langsmith.utils import ( LangSmithConnectionError, LangSmithError, - LangSmithNotFoundError, ) @@ -171,79 +168,6 @@ def test_persist_update_run(langchain_client: Client) -> None: langchain_client.delete_project(project_name=project_name) -@freeze_time("2023-01-01") -def test_evaluate_run( - monkeypatch: pytest.MonkeyPatch, langchain_client: Client -) -> None: - """Test persisting runs and adding feedback.""" - monkeypatch.setenv("LANGCHAIN_ENDPOINT", "https://api.smith.langchain.com") - project_name = "__test_evaluate_run" - dataset_name = "__test_evaluate_run_dataset" - if project_name in [sess.name for sess in langchain_client.list_projects()]: - langchain_client.delete_project(project_name=project_name) - if dataset_name in [dataset.name for dataset in langchain_client.list_datasets()]: - langchain_client.delete_dataset(dataset_name=dataset_name) - - dataset = langchain_client.create_dataset(dataset_name) - predicted = "abcd" - ground_truth = "bcde" - example = langchain_client.create_example( - inputs={"input": "hello world"}, - outputs={"output": ground_truth}, - dataset_id=dataset.id, - ) - parent_run = RunTree( - name="parent_run", - run_type="chain", - inputs={"input": "hello world"}, - project_name=project_name, - serialized={}, - start_time=datetime.now(), - reference_example_id=example.id, - ) - parent_run.post() - parent_run.end(outputs={"output": predicted}) - parent_run.patch() - parent_run.wait() - - def jaccard_chars(output: str, answer: str) -> float: - """Naive Jaccard similarity between two strings.""" - prediction_chars = set(output.strip().lower()) - answer_chars = set(answer.strip().lower()) - intersection = prediction_chars.intersection(answer_chars) - union = prediction_chars.union(answer_chars) - return len(intersection) / len(union) - - def grader(run_input: str, run_output: str, answer: Optional[str]) -> dict: - """Compute the score and/or label for this run.""" - if answer is None: - value = "AMBIGUOUS" - score = 0.5 - else: - score = jaccard_chars(run_output, answer) - value = "CORRECT" if score > 0.9 else "INCORRECT" - return dict(score=score, value=value) - - evaluator = StringEvaluator(evaluation_name="Jaccard", grading_function=grader) - runs = None - for _ in range(5): - try: - runs = list( - langchain_client.list_runs( - project_name=project_name, - execution_order=1, - error=False, - ) - ) - break - except LangSmithNotFoundError: - time.sleep(2) - assert runs is not None - all_eval_results: List[EvaluationResult] = [] - for run in runs: - all_eval_results.append(langchain_client.evaluate_run(run, evaluator)) - - @pytest.mark.parametrize("uri", ["http://localhost:1981", "http://api.langchain.minus"]) def test_error_surfaced_invalid_uri(monkeypatch: pytest.MonkeyPatch, uri: str) -> None: monkeypatch.setenv("LANGCHAIN_ENDPOINT", uri) From 553222801a778e91b7af1711b63a478db5c83c84 Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Wed, 7 Feb 2024 00:04:09 -0800 Subject: [PATCH 4/6] update --- .../python-integration-tests/action.yml | 2 +- python/Makefile | 4 +- python/langsmith/client.py | 23 ++++ python/poetry.lock | 101 +++++++++++++++--- python/pyproject.toml | 1 + python/tests/integration_tests/test_client.py | 25 +++-- 6 files changed, 126 insertions(+), 30 deletions(-) diff --git a/.github/actions/python-integration-tests/action.yml b/.github/actions/python-integration-tests/action.yml index bb6a4decd..4fb91cc06 100644 --- a/.github/actions/python-integration-tests/action.yml +++ b/.github/actions/python-integration-tests/action.yml @@ -37,6 +37,6 @@ runs: LANGCHAIN_TRACING_V2: "true" LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} - run: make integration_tests + run: make integration_tests_fast shell: bash working-directory: python diff --git a/python/Makefile b/python/Makefile index 75312826f..d6eaebfd4 100644 --- a/python/Makefile +++ b/python/Makefile @@ -7,10 +7,10 @@ tests_watch: poetry run ptw --now . -- -vv -x tests/unit_tests integration_tests: - poetry run pytest tests/integration_tests + poetry run pytest -v --durations=10 --cov=langsmith --cov-report=term-missing --cov-report=html --cov-config=.coveragerc tests/integration_tests integration_tests_fast: - poetry run pytest -n auto tests/integration_tests + poetry run pytest -n auto --durations=10 -v --cov=langsmith --cov-report=term-missing --cov-report=html --cov-config=.coveragerc tests/integration_tests lint: poetry run ruff . diff --git a/python/langsmith/client.py b/python/langsmith/client.py index f535051f8..6bde27f0d 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -1655,6 +1655,29 @@ def read_project( **response.json(), _host_url=self._host_url ) + def has_project( + self, project_name: str, *, project_id: Optional[str] = None + ) -> bool: + """Check if a project exists. + + Parameters + ---------- + project_name : str + The name of the project to check for. + project_id : str or None, default=None + The ID of the project to check for. + + Returns + ------- + bool + Whether the project exists. + """ + try: + self.read_project(project_name=project_name) + except ls_utils.LangSmithNotFoundError: + return False + return True + def get_test_results( self, *, diff --git a/python/poetry.lock b/python/poetry.lock index 153311fe7..fd5b73d79 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -337,6 +337,73 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "coverage" +version = "7.4.1" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "coverage-7.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:077d366e724f24fc02dbfe9d946534357fda71af9764ff99d73c3c596001bbd7"}, + {file = "coverage-7.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0193657651f5399d433c92f8ae264aff31fc1d066deee4b831549526433f3f61"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d17bbc946f52ca67adf72a5ee783cd7cd3477f8f8796f59b4974a9b59cacc9ee"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3277f5fa7483c927fe3a7b017b39351610265308f5267ac6d4c2b64cc1d8d25"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dceb61d40cbfcf45f51e59933c784a50846dc03211054bd76b421a713dcdf19"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6008adeca04a445ea6ef31b2cbaf1d01d02986047606f7da266629afee982630"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c61f66d93d712f6e03369b6a7769233bfda880b12f417eefdd4f16d1deb2fc4c"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9bb62fac84d5f2ff523304e59e5c439955fb3b7f44e3d7b2085184db74d733b"}, + {file = "coverage-7.4.1-cp310-cp310-win32.whl", hash = "sha256:f86f368e1c7ce897bf2457b9eb61169a44e2ef797099fb5728482b8d69f3f016"}, + {file = "coverage-7.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:869b5046d41abfea3e381dd143407b0d29b8282a904a19cb908fa24d090cc018"}, + {file = "coverage-7.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8ffb498a83d7e0305968289441914154fb0ef5d8b3157df02a90c6695978295"}, + {file = "coverage-7.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3cacfaefe6089d477264001f90f55b7881ba615953414999c46cc9713ff93c8c"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d6850e6e36e332d5511a48a251790ddc545e16e8beaf046c03985c69ccb2676"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e961aa13b6d47f758cc5879383d27b5b3f3dcd9ce8cdbfdc2571fe86feb4dd"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfd1e1b9f0898817babf840b77ce9fe655ecbe8b1b327983df485b30df8cc011"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6b00e21f86598b6330f0019b40fb397e705135040dbedc2ca9a93c7441178e74"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:536d609c6963c50055bab766d9951b6c394759190d03311f3e9fcf194ca909e1"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7ac8f8eb153724f84885a1374999b7e45734bf93a87d8df1e7ce2146860edef6"}, + {file = "coverage-7.4.1-cp311-cp311-win32.whl", hash = "sha256:f3771b23bb3675a06f5d885c3630b1d01ea6cac9e84a01aaf5508706dba546c5"}, + {file = "coverage-7.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:9d2f9d4cc2a53b38cabc2d6d80f7f9b7e3da26b2f53d48f05876fef7956b6968"}, + {file = "coverage-7.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f68ef3660677e6624c8cace943e4765545f8191313a07288a53d3da188bd8581"}, + {file = "coverage-7.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23b27b8a698e749b61809fb637eb98ebf0e505710ec46a8aa6f1be7dc0dc43a6"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3424c554391dc9ef4a92ad28665756566a28fecf47308f91841f6c49288e66"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0860a348bf7004c812c8368d1fc7f77fe8e4c095d661a579196a9533778e156"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe558371c1bdf3b8fa03e097c523fb9645b8730399c14fe7721ee9c9e2a545d3"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3468cc8720402af37b6c6e7e2a9cdb9f6c16c728638a2ebc768ba1ef6f26c3a1"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:02f2edb575d62172aa28fe00efe821ae31f25dc3d589055b3fb64d51e52e4ab1"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ca6e61dc52f601d1d224526360cdeab0d0712ec104a2ce6cc5ccef6ed9a233bc"}, + {file = "coverage-7.4.1-cp312-cp312-win32.whl", hash = "sha256:ca7b26a5e456a843b9b6683eada193fc1f65c761b3a473941efe5a291f604c74"}, + {file = "coverage-7.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:85ccc5fa54c2ed64bd91ed3b4a627b9cce04646a659512a051fa82a92c04a448"}, + {file = "coverage-7.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8bdb0285a0202888d19ec6b6d23d5990410decb932b709f2b0dfe216d031d218"}, + {file = "coverage-7.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:918440dea04521f499721c039863ef95433314b1db00ff826a02580c1f503e45"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379d4c7abad5afbe9d88cc31ea8ca262296480a86af945b08214eb1a556a3e4d"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b094116f0b6155e36a304ff912f89bbb5067157aff5f94060ff20bbabdc8da06"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2f5968608b1fe2a1d00d01ad1017ee27efd99b3437e08b83ded9b7af3f6f766"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:10e88e7f41e6197ea0429ae18f21ff521d4f4490aa33048f6c6f94c6045a6a75"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a4a3907011d39dbc3e37bdc5df0a8c93853c369039b59efa33a7b6669de04c60"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d224f0c4c9c98290a6990259073f496fcec1b5cc613eecbd22786d398ded3ad"}, + {file = "coverage-7.4.1-cp38-cp38-win32.whl", hash = "sha256:23f5881362dcb0e1a92b84b3c2809bdc90db892332daab81ad8f642d8ed55042"}, + {file = "coverage-7.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:a07f61fc452c43cd5328b392e52555f7d1952400a1ad09086c4a8addccbd138d"}, + {file = "coverage-7.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8e738a492b6221f8dcf281b67129510835461132b03024830ac0e554311a5c54"}, + {file = "coverage-7.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46342fed0fff72efcda77040b14728049200cbba1279e0bf1188f1f2078c1d70"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9641e21670c68c7e57d2053ddf6c443e4f0a6e18e547e86af3fad0795414a628"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeb2c2688ed93b027eb0d26aa188ada34acb22dceea256d76390eea135083950"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d12c923757de24e4e2110cf8832d83a886a4cf215c6e61ed506006872b43a6d1"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0491275c3b9971cdbd28a4595c2cb5838f08036bca31765bad5e17edf900b2c7"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8dfc5e195bbef80aabd81596ef52a1277ee7143fe419efc3c4d8ba2754671756"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1a78b656a4d12b0490ca72651fe4d9f5e07e3c6461063a9b6265ee45eb2bdd35"}, + {file = "coverage-7.4.1-cp39-cp39-win32.whl", hash = "sha256:f90515974b39f4dea2f27c0959688621b46d96d5a626cf9c53dbc653a895c05c"}, + {file = "coverage-7.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:64e723ca82a84053dd7bfcc986bdb34af8d9da83c521c19d6b472bc6880e191a"}, + {file = "coverage-7.4.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:32a8d985462e37cfdab611a6f95b09d7c091d07668fdc26e47a725ee575fe166"}, + {file = "coverage-7.4.1.tar.gz", hash = "sha256:1ed4b95480952b1a26d863e546fa5094564aa0065e1e5f0d4d0041f293251d04"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + [[package]] name = "dataclasses-json" version = "0.6.1" @@ -645,7 +712,6 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, - {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, ] [[package]] @@ -1095,6 +1161,24 @@ pytest = ">=7.0.0" docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] +[[package]] +name = "pytest-cov" +version = "4.1.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, + {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] + [[package]] name = "pytest-subtests" version = "0.11.0" @@ -1171,7 +1255,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1179,16 +1262,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1205,7 +1280,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1213,7 +1287,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1639,4 +1712,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "f452b10f4c30cc0e4d78d5837ae0af3d044348dc76a9b0b2b52df66b0b8bb0a3" +content-hash = "54bcb7f0ce64b9375ca8b3a70932f3e4ae020e2e0e021e7d6fede195186de4a2" diff --git a/python/pyproject.toml b/python/pyproject.toml index 43a638cdf..326039483 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -47,6 +47,7 @@ freezegun = "^1.2.2" pytest-subtests = "^0.11.0" pytest-watcher = "^0.3.4" pytest-xdist = "^3.5.0" +pytest-cov = "^4.1.0" [tool.poetry.group.lint.dependencies] openai = "^1.10" diff --git a/python/tests/integration_tests/test_client.py b/python/tests/integration_tests/test_client.py index d6e4ecff5..369861952 100644 --- a/python/tests/integration_tests/test_client.py +++ b/python/tests/integration_tests/test_client.py @@ -29,14 +29,9 @@ def langchain_client(monkeypatch: pytest.MonkeyPatch) -> Client: def test_projects(langchain_client: Client, monkeypatch: pytest.MonkeyPatch) -> None: """Test projects.""" - project_names = set([project.name for project in langchain_client.list_projects()]) new_project = "__Test Project" - if new_project in project_names: + if langchain_client.has_project(new_project): langchain_client.delete_project(project_name=new_project) - project_names = set( - [project.name for project in langchain_client.list_projects()] - ) - assert new_project not in project_names monkeypatch.setenv("LANGCHAIN_ENDPOINT", "https://api.smith.langchain.com") langchain_client.create_project( @@ -45,17 +40,18 @@ def test_projects(langchain_client: Client, monkeypatch: pytest.MonkeyPatch) -> ) project = langchain_client.read_project(project_name=new_project) assert project.name == new_project - project_names = set([sess.name for sess in langchain_client.list_projects()]) - assert new_project in project_names runs = list(langchain_client.list_runs(project_name=new_project)) project_id_runs = list(langchain_client.list_runs(project_id=project.id)) - assert len(runs) == len(project_id_runs) == 0 # TODO: Add create_run method + assert len(runs) == len(project_id_runs) == 0 langchain_client.delete_project(project_name=new_project) with pytest.raises(LangSmithError): langchain_client.read_project(project_name=new_project) assert new_project not in set( - [sess.name for sess in langchain_client.list_projects()] + [ + sess.name + for sess in langchain_client.list_projects(name_contains=new_project) + ] ) with pytest.raises(LangSmithError): langchain_client.delete_project(project_name=new_project) @@ -392,15 +388,18 @@ def test_batch_ingest_runs(langchain_client: Client) -> None: langchain_client.batch_ingest_runs(create=runs_to_create, update=runs_to_update) runs = [] wait = 2 - for _ in range(5): + for _ in range(15): try: - runs = list(langchain_client.list_runs(project_name=_session)) + runs = list( + langchain_client.list_runs( + project_name=_session, run_ids=[str(trace_id), str(run_id_2)] + ) + ) if len(runs) == 2: break raise LangSmithError("Runs not created yet") except LangSmithError: time.sleep(wait) - wait += 4 else: raise ValueError("Runs not created in time") assert len(runs) == 2 From 236d506bc232bcb2ad2dcde15b499086d58afa86 Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Wed, 7 Feb 2024 00:05:57 -0800 Subject: [PATCH 5/6] speedup --- python/tests/integration_tests/test_runs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/tests/integration_tests/test_runs.py b/python/tests/integration_tests/test_runs.py index 009b59160..be73b922d 100644 --- a/python/tests/integration_tests/test_runs.py +++ b/python/tests/integration_tests/test_runs.py @@ -52,7 +52,7 @@ def test_nested_runs( langchain_client: Client, ): project_name = "__My Tracer Project - test_nested_runs" - if project_name in [project.name for project in langchain_client.list_projects()]: + if langchain_client.had_project(project_name): langchain_client.delete_project(project_name=project_name) executor = ThreadPoolExecutor(max_workers=1) @@ -100,7 +100,7 @@ def my_chain_run(text: str): async def test_nested_async_runs(langchain_client: Client): """Test nested runs with a mix of async and sync functions.""" project_name = "__My Tracer Project - test_nested_async_runs" - if project_name in [project.name for project in langchain_client.list_projects()]: + if langchain_client.has_project(project_name): langchain_client.delete_project(project_name=project_name) executor = ThreadPoolExecutor(max_workers=1) @@ -149,7 +149,7 @@ async def my_chain_run(text: str): async def test_nested_async_runs_with_threadpool(langchain_client: Client): """Test nested runs with a mix of async and sync functions.""" project_name = "__My Tracer Project - test_nested_async_runs_with_threadpol" - if project_name in [project.name for project in langchain_client.list_projects()]: + if langchain_client.has_project(project_name): langchain_client.delete_project(project_name=project_name) @traceable(run_type="llm") @@ -219,7 +219,7 @@ async def my_chain_run(text: str, run_tree: RunTree): async def test_context_manager(langchain_client: Client) -> None: project_name = "__My Tracer Project - test_context_manager" - if project_name in [project.name for project in langchain_client.list_projects()]: + if langchain_client.has_project(project_name): langchain_client.delete_project(project_name=project_name) @traceable(run_type="llm") @@ -246,7 +246,7 @@ async def my_llm(prompt: str) -> str: async def test_sync_generator(langchain_client: Client): project_name = "__My Tracer Project - test_sync_generator" - if project_name in [project.name for project in langchain_client.list_projects()]: + if langchain_client.has_project(project_name): langchain_client.delete_project(project_name=project_name) @traceable(run_type="chain") @@ -268,7 +268,7 @@ def my_generator(num: int) -> Generator[str, None, None]: async def test_sync_generator_reduce_fn(langchain_client: Client): project_name = "__My Tracer Project - test_sync_generator_reduce_fn" - if project_name in [project.name for project in langchain_client.list_projects()]: + if langchain_client.has_project(project_name): langchain_client.delete_project(project_name=project_name) def reduce_fn(outputs: list) -> dict: @@ -295,7 +295,7 @@ def my_generator(num: int) -> Generator[str, None, None]: async def test_async_generator(langchain_client: Client): project_name = "__My Tracer Project - test_async_generator" - if project_name in [project.name for project in langchain_client.list_projects()]: + if langchain_client.has_project(project_name): langchain_client.delete_project(project_name=project_name) @traceable(run_type="chain") @@ -335,7 +335,7 @@ async def my_async_generator(num: int) -> AsyncGenerator[str, None]: async def test_async_generator_reduce_fn(langchain_client: Client): project_name = "__My Tracer Project - test_async_generator_reduce_fn" - if project_name in [project.name for project in langchain_client.list_projects()]: + if langchain_client.has_project(project_name): langchain_client.delete_project(project_name=project_name) def reduce_fn(outputs: list) -> dict: From b872731216f9956286bd2abb2fc5fd1dfd3f4fb0 Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Wed, 7 Feb 2024 00:08:44 -0800 Subject: [PATCH 6/6] has_project --- python/tests/integration_tests/test_runs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/integration_tests/test_runs.py b/python/tests/integration_tests/test_runs.py index be73b922d..20b58a251 100644 --- a/python/tests/integration_tests/test_runs.py +++ b/python/tests/integration_tests/test_runs.py @@ -52,7 +52,7 @@ def test_nested_runs( langchain_client: Client, ): project_name = "__My Tracer Project - test_nested_runs" - if langchain_client.had_project(project_name): + if langchain_client.has_project(project_name): langchain_client.delete_project(project_name=project_name) executor = ThreadPoolExecutor(max_workers=1)