Evaluation quickstart ↗

    mkdir ls-evaluation-quickstart && cd ls-evaluation-quickstart
    python -m venv .venv && source .venv/bin/activate
    python -m pip install --upgrade pip
    pip install -U langsmith openevals openai
    ```

```bash
    mkdir ls-evaluation-quickstart-ts && cd ls-evaluation-quickstart-ts
    npm init -y
    npm install langsmith openevals openai
    npx tsc --init
    ```


<span class="callout-start" data-callout-type="info"></span>
If you are using `yarn` as your package manager, you will also need to manually install `@langchain/core` as a peer dependency of `openevals`. This is not required for LangSmith evals in general, you may define evaluators [using arbitrary custom code](/langsmith/code-evaluator-ui).
<span class="callout-end"></span>

## 2. Set up environment variables

Set the following environment variables:

* `LANGSMITH_TRACING`
* `LANGSMITH_API_KEY`
* `OPENAI_API_KEY` (or your LLM provider's API key)
* (optional) `LANGSMITH_WORKSPACE_ID`: If your LangSmith API key is linked to multiple [workspaces](/langsmith/administration-overview#workspaces), set this variable to specify which workspace to use.

```bash
  export LANGSMITH_TRACING=true
  export LANGSMITH_API_KEY="<your-langsmith-api-key>"
  export OPENAI_API_KEY="<your-openai-api-key>"
  export LANGSMITH_WORKSPACE_ID="<your-workspace-id>"
  ```

<span class="callout-start" data-callout-type="note"></span>
If you're using Anthropic, use the [Anthropic wrapper](/langsmith/trace-anthropic) to trace your calls. For other providers, use [the traceable wrapper](/langsmith/annotate-code#use-%40traceable-%2F-traceable).
<span class="callout-end"></span>

## 3. Create a dataset

1. Create a file and add the following code, which will:

 * Import the `Client` to connect to LangSmith.
 * Create a dataset.
 * Define example [*inputs* and *outputs*](/langsmith/evaluation-concepts#examples).
 * Associate the input and output pairs with that dataset in LangSmith so they can be used in evaluations.

 
   ```python
       # dataset.py
       from langsmith import Client

       def main():
           client = Client()

           # Programmatically create a dataset in LangSmith
           dataset = client.create_dataset(
               dataset_name="Sample dataset",
               description="A sample dataset in LangSmith."
           )

           # Create examples
           examples = [
               {
                   "inputs": {"question": "Which country is Mount Kilimanjaro located in?"},
                   "outputs": {"answer": "Mount Kilimanjaro is located in Tanzania."},
               },
               {
                   "inputs": {"question": "What is Earth's lowest point?"},
                   "outputs": {"answer": "Earth's lowest point is The Dead Sea."},
               },
           ]

           # Add examples to the dataset
           client.create_examples(dataset_id=dataset.id, examples=examples)
           print("Created dataset:", dataset.name)

       if __name__ == "__main__":
           main()

       ```

   ```typescript
       // dataset.ts
       import { Client } from "langsmith";

       async function main() {
       const client = new Client();

       const dataset = await client.createDataset(
           "Sample dataset",
           { description: "A sample dataset in LangSmith." }
       );

       // Define examples
       const inputs = [
           { question: "Which country is Mount Kilimanjaro located in?" },
           { question: "What is Earth's lowest point?" },
       ];
       const outputs = [
           { answer: "Mount Kilimanjaro is located in Tanzania." },
           { answer: "Earth's lowest point is The Dead Sea." },
       ];

       await client.createExamples({
           datasetId: dataset.id,
           inputs,
           outputs,
       });

       console.log("Created dataset:", dataset.name);
       }

       if (require.main === module) {
       main().catch((e) => {
           console.error(e);
           process.exit(1);
       });
       }
       ```
 

2. In your terminal, run the `dataset` file to create the datasets you'll use to evaluate your app:

 
   ```bash
       python dataset.py
       ```

   ```bash
       npx ts-node dataset.ts
       ```
 

 You'll see the following output:

 ```bash
     Created dataset: Sample dataset
     ```

## 4. Create your target function

Define a [target function](/langsmith/define-target-function) that contains what you're evaluating. In this guide, you'll define a target function that contains a single LLM call to answer a question.

Add the following to an `eval` file:


```python
    # eval.py
    from langsmith import Client, wrappers
    from openai import OpenAI

    # Wrap the OpenAI client for LangSmith tracing
    openai_client = wrappers.wrap_openai(OpenAI())

    # Define the application logic you want to evaluate inside a target function
    # The SDK will automatically send the inputs from the dataset to your target function
    def target(inputs: dict) -> dict:
        response = openai_client.chat.completions.create(
            model="gpt-5-mini",
            messages=[
                {"role": "system", "content": "Answer the following question accurately"},
                {"role": "user", "content": inputs["question"]},
            ],
        )
        return {"answer": response.choices[0].message.content.strip()}
    ```

```typescript
    // eval.ts
    import { evaluate } from "langsmith/evaluation";
    import { wrapOpenAI } from "langsmith/wrappers/openai";
    import OpenAI from "openai";

    const openaiClient = wrapOpenAI(new OpenAI());

    async function target(inputs: Record<string, any>): Promise<Record<string, any>> {
      const question = String(inputs.question ?? "");
      const resp = await openaiClient.chat.completions.create({
        model: "gpt-5-mini",
        messages: [
          { role: "system", content: "Answer the following question accurately" },
          { role: "user", content: question },
        ],
      });
      return { answer: resp.choices[0].message.content?.trim() ?? "" };
    }
    ```


## 5. Define an evaluator

In this step, you’re telling LangSmith how to grade the answers your app produces.

Import a prebuilt evaluation prompt (`CORRECTNESS_PROMPT`) from [`openevals`](https://github.com/langchain-ai/openevals) and a helper that wraps it into an [*LLM-as-judge evaluator*](/langsmith/evaluation-concepts#llm-as-judge), which will score the application's output.

<span class="callout-start" data-callout-type="info"></span>
`CORRECTNESS_PROMPT` is just an f-string with variables for `"inputs"`, `"outputs"`, and `"reference_outputs"`. See [here](https://github.com/langchain-ai/openevals#customizing-prompts) for more information on customizing OpenEvals prompts.
<span class="callout-end"></span>

The evaluator compares:

* `inputs`: what was passed into your target function (e.g., the question text).
* `outputs`: what your target function returned (e.g., the model’s answer).
* `reference_outputs`: the ground truth answers you attached to each dataset example in [Step 3](#3-create-a-dataset).

Add the following highlighted code to your `eval` file:


```python
    from langsmith import Client, wrappers
    from openai import OpenAI
    from openevals.llm import create_llm_as_judge
    from openevals.prompts import CORRECTNESS_PROMPT

    # Wrap the OpenAI client for LangSmith tracing
    openai_client = wrappers.wrap_openai(OpenAI())

    # Define the application logic you want to evaluate inside a target function
    # The SDK will automatically send the inputs from the dataset to your target function
    def target(inputs: dict) -> dict:
        response = openai_client.chat.completions.create(
            model="gpt-5-mini",
            messages=[
                {"role": "system", "content": "Answer the following question accurately"},
                {"role": "user", "content": inputs["question"]},
            ],
        )
        return {"answer": response.choices[0].message.content.strip()}

    def correctness_evaluator(inputs: dict, outputs: dict, reference_outputs: dict):
        evaluator = create_llm_as_judge(
            prompt=CORRECTNESS_PROMPT,
            model="openai:o3-mini",
            feedback_key="correctness",
        )
        return evaluator(
            inputs=inputs,
            outputs=outputs,
            reference_outputs=reference_outputs
        )
    ```

```typescript
    import { evaluate } from "langsmith/evaluation";
    import { wrapOpenAI } from "langsmith/wrappers/openai";
    import OpenAI from "openai";
    import { createLLMAsJudge, CORRECTNESS_PROMPT } from "openevals";

    const openaiClient = wrapOpenAI(new OpenAI());

    async function target(inputs: Record<string, any>): Promise<Record<string, any>> {
      const question = String(inputs.question ?? "");
      const resp = await openaiClient.chat.completions.create({
        model: "gpt-5-mini",
        messages: [
          { role: "system", content: "Answer the following question accurately" },
          { role: "user", content: question },
        ],
      });
      return { answer: resp.choices[0].message.content?.trim() ?? "" };
    }

    const judge = createLLMAsJudge({
      prompt: CORRECTNESS_PROMPT,
      model: "openai:o3-mini",
      feedbackKey: "correctness",
    });

    async function correctnessEvaluator(run: {
      inputs: Record<string, any>;
      outputs: Record<string, any>;
      referenceOutputs?: Record<string, any>;
    }) {
      return judge({
        inputs: run.inputs,
        outputs: run.outputs,
        // OpenEvals expects snake_case here:
        reference_outputs: run.referenceOutputs,
      });
    }
    ```


## 6. Run and view results

To run the evaluation experiment, you'll call `evaluate(...)`, which:

* Pulls example from the dataset you created in [Step 3](#3-create-a-dataset).
* Sends each example's inputs to your target function from [Step 4](#4-add-an-evaluator).
* Collects the outputs (the model's answers).
* Passes the outputs along with the `reference_outputs` to your evaluator from [Step 5](#5-define-an-evaluator).
* Records all results in LangSmith as an experiment, so you can view them in the UI.

1. Add the highlighted code to your `eval` file:

 
   ```python
       from langsmith import Client, wrappers
       from openai import OpenAI
       from openevals.llm import create_llm_as_judge
       from openevals.prompts import CORRECTNESS_PROMPT

       # Wrap the OpenAI client for LangSmith tracing
       openai_client = wrappers.wrap_openai(OpenAI())

       # Define the application logic you want to evaluate inside a target function
       # The SDK will automatically send the inputs from the dataset to your target function
       def target(inputs: dict) -> dict:
           response = openai_client.chat.completions.create(
               model="gpt-5-mini",
               messages=[
                   {"role": "system", "content": "Answer the following question accurately"},
                   {"role": "user", "content": inputs["question"]},
               ],
           )
           return {"answer": response.choices[0].message.content.strip()}

       def correctness_evaluator(inputs: dict, outputs: dict, reference_outputs: dict):
           evaluator = create_llm_as_judge(
               prompt=CORRECTNESS_PROMPT,
               model="openai:o3-mini",
               feedback_key="correctness",
           )
           return evaluator(
               inputs=inputs,
               outputs=outputs,
               reference_outputs=reference_outputs
           )

       # After running the evaluation, a link will be provided to view the results in langsmith
       def main():
           client = Client()
           experiment_results = client.evaluate(
               target,
               data="Sample dataset",
               evaluators=[
                   correctness_evaluator,
                   # can add multiple evaluators here
               ],
               experiment_prefix="first-eval-in-langsmith",
               max_concurrency=2,
           )
           print(experiment_results)

       if __name__ == "__main__":
           main()
       ```

   ```typescript
       import { evaluate } from "langsmith/evaluation";
       import { wrapOpenAI } from "langsmith/wrappers/openai";   // helper to wrap OpenAI client
       import OpenAI from "openai";                              // model provider
       import { createLLMAsJudge, CORRECTNESS_PROMPT } from "openevals"; // evaluator tools

       const openaiClient = wrapOpenAI(new OpenAI());

       async function target(inputs: Record<string, any>): Promise<Record<string, any>> {
       const question = String(inputs.question ?? "");
       const resp = await openaiClient.chat.completions.create({
           model: "gpt-5-mini",
           messages: [
           { role: "system", content: "Answer the following question accurately" },
           { role: "user", content: question },
           ],
       });
       return { answer: resp.choices[0].message.content?.trim() ?? "" };
       }

       const judge = createLLMAsJudge({
       prompt: CORRECTNESS_PROMPT,
       model: "openai:o3-mini",
       feedbackKey: "correctness",
       });

       async function correctnessEvaluator(run: {
       inputs: Record<string, any>;
       outputs: Record<string, any>;
       referenceOutputs?: Record<string, any>;
       }) {
       return judge({
           inputs: run.inputs,
           outputs: run.outputs,
           // OpenEvals expects snake_case here:
           reference_outputs: run.referenceOutputs,
       });
       }

       async function main() {
       const datasetName = process.env.DATASET_NAME ?? "Sample dataset";

       const results = await evaluate(target, {
           data: datasetName,
           evaluators: [correctnessEvaluator],
           experimentPrefix: "first-eval-in-langsmith",
           maxConcurrency: 2,
       });

       console.log(results);
       }

       if (require.main === module) {
       main().catch((e) => {
           console.error(e);
           process.exit(1);
       });
       }
       ```
 

2. Run your evaluator:

 
   ```bash
       python eval.py
       ```

   ```bash
       npx ts-node eval.ts
       ```
 

3. You'll receive a link to view the evaluation results and metadata for the experiment results: