o +i:@sdZddlmZddlZddlZddlmZmZmZm Z m Z m Z m Z ddl mZddl mZddl mZddl mZerHdd lmZdd lmZGd d d ZeZGd ddZGdddZeZdgZdS)a<Make approximate assertions as "expectations" on test results. This module is designed to be used within test cases decorated with the `@pytest.mark.decorator` decorator It allows you to log scores about a test case and optionally make assertions that log as "expectation" feedback to LangSmith. Example: ```python import pytest from langsmith import expect @pytest.mark.langsmith def test_output_semantically_close(): response = oai_client.chat.completions.create( model="gpt-3.5-turbo", messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Say hello!"}, ], ) response_txt = response.choices[0].message.content # Intended usage expect.embedding_distance( prediction=response_txt, reference="Hello!", ).to_be_less_than(0.9) # Score the test case matcher = expect.edit_distance( prediction=response_txt, reference="Hello!", ) # Apply an assertion and log 'expectation' feedback to LangSmith matcher.to_be_less_than(1) # You can also directly make assertions on values directly expect.value(response_txt).to_contain("Hello!") # Or using a custom check expect.value(response_txt).against(lambda x: "Hello" in x) # You can even use this for basic metric logging within tests expect.score(0.8) expect.score(0.7, key="similarity").to_be_greater_than(0.7) ``` ) annotationsN) TYPE_CHECKINGAnyCallableLiteralOptionalUnionoverloadclient) run_helpers) run_trees)utils)EditDistanceConfig)EmbeddingConfigc@s$eZdZdZd ddZd ddZd S) _NULL_SENTRYzA sentinel singleton class used to distinguish omitted keyword arguments from those passed in with the value None (which may have different behavior). returnLiteral[False]cCdS)NFselfrrN/var/www/html/psymed-ai/venv/lib/python3.10/site-packages/langsmith/_expect.py__bool__Qz_NULL_SENTRY.__bool__strcCr)N NOT_GIVENrrrrr__repr__Trz_NULL_SENTRY.__repr__N)rr)rr)__name__ __module__ __qualname____doc__rrrrrrrLs rc@seZdZdZ  d2d3d dZd4d5ddZd6ddZd7ddZd7ddZd8d"d#Z d9d:d&d'Z d7d(d)Z d;d*d+Z d_Matcherz4A class for making assertions on expectation values.Nr Optional[ls_client.Client]keyrvaluer _executor,Optional[ls_utils.ContextThreadPoolExecutor]run_id Optional[str]cCsF||_||_||_|ptjdd|_t}|r|j|_ dS||_ dS)N max_workers) _clientr$r%ls_utilsContextThreadPoolExecutorr&rhget_current_run_treetrace_id_run_id)rr r$r%r&r(rtrrr__init__^s z_Matcher.__init__scoreintmessagerNonecCs<ts|js t|_|jj|jj|jd||ddSdS)N expectation)r(r$r6comment) r.test_tracking_is_disabledr-r4get_cached_clientr&submitcreate_feedbackr3)rr6r8rrr_submit_feedbackms  z_Matcher._submit_feedback conditionbool method_namec Cs^z|sJ||jdd|jd|dWdSty.}z |dt||dd}~ww)Nz Success: .)r8r)r@r$AssertionErrorrepr)rrAr8rCerrr_assertys "z_Matcher._assertfloatc Cs.||j|kd|jd|d|jddS)zAssert that the expectation value is less than the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value is not less than the given value. Expected z to be less than , but got to_be_less_thanNrIr%r$rr%rrrrMs  z_Matcher.to_be_less_thanc Cs.||j|kd|jd|d|jddS)aAssert that the expectation value is greater than the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value is not greater than the given value. rKz to be greater than rLto_be_greater_thanNrNrOrrrrP  z_Matcher.to_be_greater_than min_value max_valuec CsD|||jko |knd|jd|d|d|jddS)a4Assert that the expectation value is between the given min and max values. Args: min_value: The minimum value (exclusive). max_value: The maximum value (exclusive). Raises: AssertionError: If the expectation value is not between the min and max. rKz to be between z and rL to_be_betweenNrN)rrRrSrrrrTs z_Matcher.to_be_between precisionc Cs:|t|j|t||kd|jd|d|jddS)akAssert that the expectation value is approximately equal to the given value. Args: value: The value to compare against. precision: The number of decimal places to round to for comparison. Raises: AssertionError: If the rounded expectation value does not equal the rounded given value. rKz to be approximately rLto_be_approximatelyN)rIroundr%r$)rr%rVrrrrWs  z_Matcher.to_be_approximatelyc Cs.||j|kd|jd|d|jddS)aAssert that the expectation value equals the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value does not exactly equal the given value. rKz to be equal to rLto_equalNrNrOrrrrYrQz_Matcher.to_equalcCs(||jdud|jd|jddS)zAssert that the expectation value is `None`. Raises: AssertionError: If the expectation value is not `None`. NrKz to be None, but got to_be_nonerNrrrrrZs z_Matcher.to_be_nonecCs(|||jvd|jd|dddS)zAssert that the expectation value contains the given value. Args: value: The value to check for containment. Raises: AssertionError: If the expectation value does not contain the given value. rKz to contain z, but it does not to_containNrNrOrrrr[s  z_Matcher.to_containfuncrcCs0t|}|||jd|d|jddS)zAssert the expectation value against a custom function. Args: func: A custom function that takes the expectation value as input. Raises: AssertionError: If the custom function returns False. z Assertion z failed for againstN)inspect signaturerIr%r$)rr\func_signaturerrrr]s z_Matcher.against)NN) r r#r$rr%rr&r'r(r)N)r6r7r8r)rr9)rArBr8rrCrrr9)r%rJrr9)rRrJrSrJrr9)rU)r%rJrVr7rr9)rr9)r%rrr9)r\rrr9)rrr r!r5r@rIrMrPrTrWrYrZr[r]rrrrr"[s        r"c@seZdZdZddd.ddZddd/ddZddd0ddZd1ddZddddd2d!d"Ze d1d#d$Z e d3d&d$Z e dfd4d)d$Z d5d,d-Z dS)6_Expectz1A class for setting expectations on test results.Nr r r#cCs*||_tjdd|_tj|jjdddS)Nr*r+T)wait)r-r.r/executoratexitregistershutdownrr rrrr5sz_Expect.__init__config predictionr referencerjOptional[EmbeddingConfig]rr"c Csddlm}|p i}|drdnd}||d}|j||d}||jd}|d ||d |d |jd t|jd ||jd S)aCompute the embedding distance between the prediction and reference. This logs the embedding distance to LangSmith and returns a `_Matcher` instance for making assertions on the distance value. By default, this uses the OpenAI API for computing embeddings. Args: prediction: The predicted string to compare. reference: The reference string to compare against. config: Optional configuration for the embedding distance evaluator. Supported options: - `encoder`: A custom encoder function to encode the list of input strings to embeddings. Defaults to the OpenAI API. - `metric`: The distance metric to use for comparison. Supported values: `'cosine'`, `'euclidean'`, `'manhattan'`, `'chebyshev'`, `'hamming'`. Returns: A `_Matcher` instance for the embedding distance value. Example: ```python expect.embedding_distance( prediction="hello", reference="hi", ).to_be_less_than(1.0) ``` r)EmbeddingDistanceencodercustomopenairirkrl)rometricembedding_distanceUsing z , Metric: r6 source_infor;r&) 'langsmith._internal._embedding_distancerngetevaluatedistancer@r"r-rd) rrkrlrjrn encoder_func evaluatorr6src_inforrrrts *   z_Expect.embedding_distanceOptional[EditDistanceConfig]c Csddlm}|p i}|dpd}|dd}||d}|j||d}||d } |d || d |d |d t|jd ||jdS)aBCompute the string distance between the prediction and reference. This logs the string distance (Damerau-Levenshtein) to LangSmith and returns a `_Matcher` instance for making assertions on the distance value. This depends on the `rapidfuzz` package for string distance computation. Args: prediction: The predicted string to compare. reference: The reference string to compare against. config: Optional configuration for the string distance evaluator. Supported options: - `metric`: The distance metric to use for comparison. Supported values: `'damerau_levenshtein'`, `'levenshtein'`, `'jaro'`, `'jaro_winkler'`, `'hamming'`, `'indel'`. - `normalize_score`: Whether to normalize the score between `0` and `1`. Returns: A `_Matcher` instance for the string distance value. Examples: ```python expect.edit_distance("hello", "helo").to_be_less_than(1) ``` r) EditDistancersdamerau_levenshteinnormalize_scoreTrirr)rs normalize edit_distanceruz , Normalize: rvrx)"langsmith._internal._edit_distancerrzr{r@r"r-rd) rrkrlrjrrsrr~r6rrrrrDs( #   z_Expect.edit_distancer%rcCst|jd||jdS)aDCreate a `_Matcher` instance for making assertions on the given value. Args: value: The value to make assertions on. Returns: A `_Matcher` instance for the given value. Example: ```python expect.value(10).to_be_less_than(20) ``` r%rx)r"r-rdrOrrrr%~sz _Expect.valuer6)r$ source_run_idr;Union[float, int, bool]r$rOptional[ls_client.ID_TYPE]r;r)cCs.|||ddi||dt|j|||jdS)aLog a numeric score to LangSmith. Args: score: The score value to log. key: The key to use for logging the score. Defaults to `'score'`. Example: ```python expect.score(0.8) # doctest: +ELLIPSIS expect.score(0.8, key="similarity").to_be_greater_than(0.7) ``` methodz expect.score)r6rwrr;rx)r@r"r-rd)rr6r$rr;rrrr6s z _Expect.scorecCdSrarrOrrr__call__rz_Expect.__call__ls_client.ClientcCrrarrhrrrrr Optional[Any]Union[_Expect, _Matcher]cCs t|d}|tur||S|S)Nr )rbrr%)rr%r expectedrrrrs  resultsdictcCsTt}|r |jnd}ts(|jst|_|jj |jj f||d|dSdS)N)r(r$) r0r1r2r.r<r-r4r=rdr>r?)rr$r current_runr(rrrr@s  z_Expect._submit_feedback)r r#)rkrrlrrjrmrr")rkrrlrrjrrr")r%rrr") r6rr$rrrr;r)rr")r rrrb)r r#r%rrr)r$rrr) rrr r!r5rtrr%r6r rrr@rrrrrbs( B :#    rbexpect) r! __future__rrer^typingrrrrrrr langsmithr ls_clientr r0r r4rr.rrryrrrr"rbr__all__rrrrs( 2$      %O