python[minor]: manually set test case inputs/outputs

[baskaryan]

example usage

@unit
def test_example():
    x = 0
    y = 1
    add_test_inputs({"x": x, "y": y})
    add_test_reference_outputs({"product": 0})
    add_test_outputs({"product": x * y})
    assert x * y == 0

[hinthornw]

I like the functionality. I kinda like using a single module/object to access the methods, like

@unit
def test_example(captest):
    x = 0
    y = 1
    captest.log_inputs({"x": x, "y": y})
    captest.log_reference({"product": 0})
    captest.log_outputs({"product": x * y})
    assert x * y == 0

or something

[baskaryan]

Updated

from langsmith import testing

@testing.test
def test_example():
    x = 0
    y = 1
    testing.log_inputs({"x": x, "y": y})
    testing.log_reference_outputs({"product": 0})
    testing.log_outputs({"product": x * y})
    testing.log_feedback(key="correct", value=1.0)
    assert x * y == 0

[hinthornw]

I like

[baskaryan]

updated usage

LANGSMITH_TEST_SUITE="foo" pytest -n auto tests/test_foo.py

# test_foo.py
import openai
import pytest

from langsmith import testing, wrappers

oai_client = wrappers.wrap_openai(openai.Client())


@testing.test
def test_openai_says_hello():
    # Traced code will be included in the test case
    text = "Say hello!"
    response = oai_client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": text},
        ],
    )
    testing.log_inputs({"text": text})
    testing.log_outputs({"response": response.choices[0].message.content})
    testing.log_reference_outputs({"response": "hello!"})
    with testing.trace_feedback():
        grade = oai_client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {
                    "role": "system",
                    "content": "Return 1 if 'hello' is in the user message and 0 otherwise.",
                },
                {"role": "user", "content": response.choices[0].message.content},
            ],
        )
        testing.log_feedback(
            key="llm_judge", score=float(grade.choices[0].message.content)
        )

    assert "hello" in response.choices[0].message.content.lower()


@testing.test(output_keys=["expected"])
@pytest.mark.parametrize(
    "a, b, expected",
    [
        (1, 2, 3),
        (3, 4, 7),
    ],
)
def test_addition_with_multiple_inputs(a: int, b: int, expected: int):
    testing.log_outputs({"sum": a + b})
    assert a + b == expected