Log LLM calls ↗

 inputs = {
  "messages": [
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": "Hi, can you tell me the capital of France?"
        }
      ]
    }
  ]
}

outputs = {
  "messages": [
    {
      "role": "assistant",
      "content": [
        {
          "type": "text",
          "text": "The capital of France is Paris."
        },
        {
          "type": "reasoning",
          "text": "The user is asking about..."
        }
      ]
    }
  ]
}

input = {
  "messages": [
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": "What's the weather in San Francisco?"
        }
      ]
    }
  ]
}

outputs = {
  "messages": [
    {
      "role": "assistant",
      "content": [{"type": "tool_call", "name": "get_weather", "args": {"city": "San Francisco"}, "id": "call_1"}],
    },
    {
      "role": "tool",
      "tool_call_id": "call_1",
      "content": [
        {
          "type": "text",
          "text": "{\"temperature\": \"18°C\", \"condition\": \"Sunny\"}"
        }
      ]
    },
    {
      "role": "assistant",
      "content": [
        {
          "type": "text",
          "text": "The weather in San Francisco is 18°C and sunny."
        }
      ]
    }
  ]
}

inputs = {
  "messages": [
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": "What breed is this dog?"
        },
        {
          "type": "image",
          "url": "https://fastly.picsum.photos/id/237/200/300.jpg?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U",
          # alternative to a url, you can provide a base64 encoded image
          # "base64": "<base64 encoded image>",
          "mime_type": "image/jpeg",
        }
      ]
    }
  ]
}

outputs = {
  "messages": [
    {
      "role": "assistant",
      "content": [
        {
          "type": "text",
          "text": "This looks like a Black Labrador."
        }
      ]
    }
  ]
}

input = {
  "messages": [
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": "What is the price of AAPL?"
        }
      ]
    }
  ]
}

output = {
  "messages": [
    {
      "role": "assistant",
      "content": [
        {
          "type": "server_tool_call",
          "name": "web_search",
          "args": {
            "query": "price of AAPL",
            "type": "search"
          },
          "id": "call_1"
        },
        {
          "type": "server_tool_result",
          "tool_call_id": "call_1",
          "status": "success"
        },
        {
          "type": "text",
          "text": "The price of AAPL is $150.00"
        }
      ]
    }
  ]
}

    class OriginalInputs(BaseModel):
        """Your app's custom request shape"""

    class OriginalOutputs(BaseModel):
        """Your app's custom response shape."""

    class LangSmithInputs(BaseModel):
        """The input format LangSmith expects."""

    class LangSmithOutputs(BaseModel):
        """The output format LangSmith expects."""

    def process_inputs(inputs: dict) -> dict:
        """Dict -> OriginalInputs -> LangSmithInputs -> dict"""

    def process_outputs(output: Any) -> dict:
        """OriginalOutputs -> LangSmithOutputs -> dict"""


    @traceable(run_type="llm", process_inputs=process_inputs, process_outputs=process_outputs)
    def chat_model(inputs: dict) -> dict:
        """
        Your app's model call. Keeps your custom I/O shape.
        The decorators call process_* to log LangSmith-compatible format.
        """

    ```
  


## Identifying a custom model in traces

When using a custom model, it is recommended to also provide the following `metadata` fields to identify the model when viewing traces and when filtering.

* `ls_provider`: The provider of the model, e.g. "openai", "anthropic", etc.
* `ls_model_name`: The name of the model, e.g. "gpt-4.1-mini", "claude-3-opus-20240229", etc.


  ```python
  from langsmith import traceable

  inputs = [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "I'd like to book a table for two."},
  ]
  output = {
      "choices": [
          {
              "message": {
                  "role": "assistant",
                  "content": "Sure, what time would you like to book the table for?"
              }
          }
      ]
  }

  @traceable(
      run_type="llm",
      metadata={"ls_provider": "my_provider", "ls_model_name": "my_model"}
  )
  def chat_model(messages: list):
      return output

  chat_model(inputs)

import { traceable } from "langsmith/traceable";

const messages = [
    { role: "system", content: "You are a helpful assistant." },
    { role: "user", content: "I'd like to book a table for two." }
];
const output = {
    choices: [
        {
            message: {
                role: "assistant",
                content: "Sure, what time would you like to book the table for?",
            },
        },
    ],
    usage_metadata: {
        input_tokens: 27,
        output_tokens: 13,
        total_tokens: 40,
    },
};

// Can also use one of:
// const output = {
//     message: {
//         role: "assistant",
//         content: "Sure, what time would you like to book the table for?"
//     }
// };
//
// const output = {
//     role: "assistant",
//     content: "Sure, what time would you like to book the table for?"
// };
//
// const output = ["assistant", "Sure, what time would you like to book the table for?"];

const chatModel = traceable(
    async ({ messages }: { messages: { role: string; content: string }[] }) => {
        return output;
    },
    {
        run_type: "llm",
        name: "chat_model",
        metadata: {
            ls_provider: "my_provider",
            ls_model_name: "my_model"
        }
    }
);

await chatModel({ messages });

def _reduce_chunks(chunks: list):
    all_text = "".join([chunk["choices"][0]["message"]["content"] for chunk in chunks])
    return {"choices": [{"message": {"content": all_text, "role": "assistant"}}]}

@traceable(
    run_type="llm",
    reduce_fn=_reduce_chunks,
    metadata={"ls_provider": "my_provider", "ls_model_name": "my_model"}
)
def my_streaming_chat_model(messages: list):
    for chunk in ["Hello, " + messages[1]["content"]]:
        yield {
            "choices": [
                {
                    "message": {
                        "content": chunk,
                        "role": "assistant",
                    }
                }
            ]
        }

list(
    my_streaming_chat_model(
        [
            {"role": "system", "content": "You are a helpful assistant. Please greet the user."},
            {"role": "user", "content": "polly the parrot"},
        ],
    )
)

from langsmith.run_trees import RunTree
run_tree = RunTree(
    name="CustomChatModel",
    run_type="llm",
    inputs={ ... }
)
run_tree.post()
llm_stream = ...
first_token = None
for token in llm_stream:
    if first_token is None:
      first_token = token
      run_tree.add_event({
        "name": "new_token"
      })
run_tree.end(outputs={ ... })
run_tree.patch()

import { RunTree } from "langsmith";
const runTree = new RunTree({
    name: "CustomChatModel",
    run_type: "llm",
    inputs: { ... },
});
await runTree.postRun();
const llmStream = ...;
let firstToken;
for (const token of llmStream) {
    if (firstToken == null) {
        firstToken = token;
        runTree.addEvent({ name: "new_token" });
    }
}
await runTree.end({
    outputs: { ... },
});
await runTree.patchRun();