Skip to content

Commit

Permalink
update api ref
Browse files Browse the repository at this point in the history
  • Loading branch information
@baskaryan
baskaryan committed Jan 16, 2025
1 parent e56a68f commit d049218
Show file tree
Hide file tree
Showing 4 changed files with 148 additions and 129 deletions.
22 changes: 4 additions & 18 deletions python/docs/create_api_rst.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,18 +108,6 @@ def _load_module_members(module_path: str, namespace: str) -> ModuleMembers:
else "Pydantic" if issubclass(type_, BaseModel) else "Regular"
)
)
# if hasattr(type_, "__slots__"):
# for func_name, func_type in inspect.getmembers(type_):
# if inspect.isfunction(func_type):
# functions.append(
# FunctionInfo(
# name=func_name,
# qualified_name=f"{namespace}.{name}.{func_name}",
# is_public=not func_name.startswith("_"),
# is_deprecated=".. deprecated::"
# in (func_type.__doc__ or ""),
# )
# )
classes_.append(
ClassInfo(
name=name,
Expand Down Expand Up @@ -156,7 +144,7 @@ def _load_package_modules(
if file_path.name not in {
"_runner.py",
"_arunner.py",
"_testing.py",
"_internal.py",
"_expect.py",
"_openai.py",
}:
Expand Down Expand Up @@ -200,6 +188,7 @@ def _load_package_modules(
"utils",
"anonymizer",
"wrappers",
"testing",
]


Expand Down Expand Up @@ -387,20 +376,17 @@ def _build_index(package_version: str) -> None:
| [AsyncClient](async_client/langsmith.async_client.AsyncClient) | Asynchronous client for interacting with the LangSmith API. |
| [traceable](run_helpers/langsmith.run_helpers.traceable) | Wrapper/decorator for tracing any function. |
| [wrap_openai](wrappers/langsmith.wrappers._openai.wrap_openai) | Wrapper for OpenAI client, adds LangSmith tracing to all OpenAI calls. |
| [evaluate](evaluation/langsmith.evaluation._runner.evaluate) | Evaluate an application on a dataset. |
| [aevaluate](evaluation/langsmith.evaluation._arunner.aevaluate) | Asynchronously evaluate an application on a dataset. |
| [unit](_testing/langsmith._testing.unit) | Create a LangSmith unit test. |
| [@pytest.mark.langsmith](/testing/langsmith.testing._internal.test) | LangSmith pytest integration. |
```{{toctree}}
:maxdepth: 2
:hidden:
client<client>
async_client<async_client>
evaluation<evaluation>
run_helpers<run_helpers>
wrappers<wrappers>
_testing<_testing>
testing<testing>
```
"""
Expand Down
248 changes: 140 additions & 108 deletions python/langsmith/testing/_internal.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,32 +114,39 @@ def test(*args: Any, **kwargs: Any) -> Callable:
For basic usage, simply decorate a test function with `@pytest.mark.langsmith`.
Under the hood this will call the `test` method:
>>> import pytest
>>>
>>> # Equivalently can decorate with `test` directly:
>>> # from langsmith import test
>>> # @test
>>> @pytest.mark.langsmith
... def test_addition():
... assert 3 + 4 == 7
.. code-block:: python
import pytest
# Equivalently can decorate with `test` directly:
# from langsmith import test
# @test
@pytest.mark.langsmith
def test_addition():
assert 3 + 4 == 7
Any code that is traced (such as those traced using `@traceable`
or `wrap_*` functions) will be traced within the test case for
improved visibility and debugging.
>>> import pytest
>>> from langsmith import traceable
>>>
>>> @traceable
... def generate_numbers():
... return 3, 4
.. code-block:: python
>>> @pytest.mark.langsmith
... def test_nested():
... # Traced code will be included in the test case
... a, b = generate_numbers()
... assert a + b == 7
import pytest
from langsmith import traceable
@traceable
def generate_numbers():
return 3, 4
@pytest.mark.langsmith
def test_nested():
# Traced code will be included in the test case
a, b = generate_numbers()
assert a + b == 7
LLM calls are expensive! Cache requests by setting
`LANGSMITH_TEST_CACHE=path/to/cache`. Check in these files to speed up
Expand All @@ -153,119 +160,144 @@ def test(*args: Any, **kwargs: Any) -> Callable:
Caching is faster if you install libyaml. See
https://vcrpy.readthedocs.io/en/latest/installation.html#speed for more details.
>>> # os.environ["LANGSMITH_TEST_CACHE"] = "tests/cassettes"
>>> import openai
>>> import pytest
>>> from langsmith import wrappers
>>>
>>> oai_client = wrappers.wrap_openai(openai.Client())
>>> @pytest.mark.langsmith
... def test_openai_says_hello():
... # Traced code will be included in the test case
... response = oai_client.chat.completions.create(
... model="gpt-3.5-turbo",
... messages=[
... {"role": "system", "content": "You are a helpful assistant."},
... {"role": "user", "content": "Say hello!"},
... ],
... )
... assert "hello" in response.choices[0].message.content.lower()
.. code-block:: python
# os.environ["LANGSMITH_TEST_CACHE"] = "tests/cassettes"
import openai
import pytest
from langsmith import wrappers
oai_client = wrappers.wrap_openai(openai.Client())
@pytest.mark.langsmith
def test_openai_says_hello():
# Traced code will be included in the test case
response = oai_client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Say hello!"},
],
)
assert "hello" in response.choices[0].message.content.lower()
LLMs are stochastic. Naive assertions are flakey. You can use langsmith's
`expect` to score and make approximate assertions on your results.
>>> import pytest
>>> from langsmith import expect
>>>
>>> @pytest.mark.langsmith
... def test_output_semantically_close():
... response = oai_client.chat.completions.create(
... model="gpt-3.5-turbo",
... messages=[
... {"role": "system", "content": "You are a helpful assistant."},
... {"role": "user", "content": "Say hello!"},
... ],
... )
... # The embedding_distance call logs the embedding distance to LangSmith
... expect.embedding_distance(
... prediction=response.choices[0].message.content,
... reference="Hello!",
... # The following optional assertion logs a
... # pass/fail score to LangSmith
... # and raises an AssertionError if the assertion fails.
... ).to_be_less_than(1.0)
... # Compute damerau_levenshtein distance
... expect.edit_distance(
... prediction=response.choices[0].message.content,
... reference="Hello!",
... # And then log a pass/fail score to LangSmith
... ).to_be_less_than(1.0)
.. code-block:: python
import pytest
from langsmith import expect
@pytest.mark.langsmith
def test_output_semantically_close():
response = oai_client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Say hello!"},
],
)
# The embedding_distance call logs the embedding distance to LangSmith
expect.embedding_distance(
prediction=response.choices[0].message.content,
reference="Hello!",
# The following optional assertion logs a
# pass/fail score to LangSmith
# and raises an AssertionError if the assertion fails.
).to_be_less_than(1.0)
# Compute damerau_levenshtein distance
expect.edit_distance(
prediction=response.choices[0].message.content,
reference="Hello!",
# And then log a pass/fail score to LangSmith
).to_be_less_than(1.0)
The `@test` decorator works natively with pytest fixtures.
The values will populate the "inputs" of the corresponding example in LangSmith.
>>> import pytest
>>>
>>> @pytest.fixture
... def some_input():
... return "Some input"
>>>
>>> @pytest.mark.langsmith
... def test_with_fixture(some_input: str):
... assert "input" in some_input
>>>
.. code-block:: python
import pytest
@pytest.fixture
def some_input():
return "Some input"
@pytest.mark.langsmith
def test_with_fixture(some_input: str):
assert "input" in some_input
You can still use pytest.parametrize() as usual to run multiple test cases
using the same test function.
>>> import pytest
>>>
>>> @pytest.mark.langsmith(output_keys=["expected"])
... @pytest.mark.parametrize(
... "a, b, expected",
... [
... (1, 2, 3),
... (3, 4, 7),
... ],
... )
... def test_addition_with_multiple_inputs(a: int, b: int, expected: int):
... assert a + b == expected
.. code-block:: python
import pytest
@pytest.mark.langsmith(output_keys=["expected"])
@pytest.mark.parametrize(
"a, b, expected",
[
(1, 2, 3),
(3, 4, 7),
],
)
def test_addition_with_multiple_inputs(a: int, b: int, expected: int):
assert a + b == expected
By default, each test case will be assigned a consistent, unique identifier
based on the function name and module. You can also provide a custom identifier
using the `id` argument:
>>> import pytest
>>> import uuid
>>>
>>> example_id = uuid.uuid4()
>>> @pytest.mark.langsmith(id=str(example_id))
... def test_multiplication():
... assert 3 * 4 == 12
.. code-block:: python
import pytest
import uuid
example_id = uuid.uuid4()
@pytest.mark.langsmith(id=str(example_id))
def test_multiplication():
assert 3 * 4 == 12
By default, all test inputs are saved as "inputs" to a dataset.
You can specify the `output_keys` argument to persist those keys
within the dataset's "outputs" fields.
>>> import pytest
>>>
>>> @pytest.fixture
... def expected_output():
... return "input"
>>>
>>> @pytest.mark.langsmith(output_keys=["expected_output"])
... def test_with_expected_output(some_input: str, expected_output: str):
... assert expected_output in some_input
.. code-block:: python
import pytest
@pytest.fixture
def expected_output():
return "input"
@pytest.mark.langsmith(output_keys=["expected_output"])
def test_with_expected_output(some_input: str, expected_output: str):
assert expected_output in some_input
To run these tests, use the pytest CLI. Or directly run the test functions.
>>> test_output_semantically_close()
>>> test_addition()
>>> test_nested()
>>> test_with_fixture("Some input")
>>> test_with_expected_output("Some input", "Some")
>>> test_multiplication()
>>> test_openai_says_hello()
>>> test_addition_with_multiple_inputs(1, 2, 3)
.. code-block:: python
test_output_semantically_close()
test_addition()
test_nested()
test_with_fixture("Some input")
test_with_expected_output("Some input", "Some")
test_multiplication()
test_openai_says_hello()
test_addition_with_multiple_inputs(1, 2, 3)
"""
langtest_extra = _UTExtra(
id=kwargs.pop("id", None),
Expand Down
4 changes: 2 additions & 2 deletions python/poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion python/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ langsmith-pyo3 = { version = "^0.1.0rc2", optional = true }
# Enabled via `compression` extra: `pip install langsmith[compression]`.
zstandard = { version = "^0.23.0", optional = true }
rich = {version = "^13.9.4", optional = true}
pytest = {version = "^7.0.0", optional = true}

[tool.poetry.group.dev.dependencies]
pytest = "^7.3.1"
Expand Down Expand Up @@ -79,7 +80,7 @@ pytest-socket = "^0.7.0"
vcr = ["vcrpy"]
langsmith_pyo3 = ["langsmith-pyo3"]
compression = ["zstandard"]
pytest = ["rich", "vcrpy"]
pytest = ["pytest", "rich", "vcrpy"]

[build-system]
requires = ["poetry-core"]
Expand Down

0 comments on commit d049218

Please sign in to comment.