@@ -261,29 +261,21 @@ def __init__(self, agent: BaseAgent[OutputT]) -> None:
261261 self ._agent_middleware : list [AgentMiddleware ] = []
262262 if agent .limits .max_structured_output_retires is not None :
263263 self ._agent_middleware .append (
264- _StructuredOutputRetryLimitMiddleware (
265- agent .limits .max_structured_output_retires
266- )
264+ _StructuredOutputRetryLimitMiddleware (agent .limits .max_structured_output_retires )
267265 )
268266
269267 self ._agent_middleware .extend (before_user_middlewares )
270268 self ._agent_middleware .extend (agent .middleware or [])
271269 self ._agent_middleware .extend (after_user_middlewares )
272270
273- if agent .limits .max_tokens is not None :
274- self ._agent_middleware .append (
275- _TokenLimitMiddleware (agent .limits .max_tokens )
276- )
277271 if agent .limits .max_steps is not None :
278272 self ._agent_middleware .append (_StepLimitMiddleware (agent .limits .max_steps ))
279273 if agent .limits .timeout is not None :
280274 self ._agent_middleware .append (_TimeoutLimitMiddleware (agent .limits .timeout ))
281275
282276 model_impl = _create_langchain_model (agent .model )
283277
284- lc_middleware : list [LC_AgentMiddleware ] = [
285- _Middleware (self ._agent_middleware , model_impl )
286- ]
278+ lc_middleware : list [LC_AgentMiddleware ] = [_Middleware (self ._agent_middleware )]
287279
288280 # This middleware is executed just after the tool execution and populates
289281 # the artifact field for failed tool calls, since in such cases we can't
@@ -593,6 +585,27 @@ async def awrap_tool_call(
593585 if _DEBUG :
594586 lc_middleware .append (_DEBUGMiddleware ())
595587
588+ if agent .limits .max_tokens is not None :
589+ _max_tokens = agent .limits .max_tokens
590+
591+ class _TokenLimitMiddleware (LC_AgentMiddleware ):
592+ @override
593+ async def awrap_model_call (
594+ self ,
595+ request : LC_ModelRequest ,
596+ handler : Callable [[LC_ModelRequest ], Awaitable [LC_ModelCallResult ]],
597+ ) -> LC_ModelCallResult :
598+ token_count = _get_approximate_token_counter (request .model , request .tools )(
599+ request .state ["messages" ]
600+ )
601+
602+ if token_count >= _max_tokens :
603+ raise TokenLimitExceededException (token_limit = _max_tokens )
604+
605+ return await handler (request )
606+
607+ lc_middleware .append (_TokenLimitMiddleware ())
608+
596609 response_format = None
597610 if agent .output_schema is not None :
598611 if _supports_provider_strategy (model_impl ):
@@ -770,11 +783,9 @@ def _prepare_langchain_tools(agent_tools: Sequence[Tool]) -> list[BaseTool]:
770783
771784class _Middleware (LC_AgentMiddleware ):
772785 _middleware : list [AgentMiddleware ]
773- _model : BaseChatModel
774786
775- def __init__ (self , middleware : list [AgentMiddleware ], model : BaseChatModel ) -> None :
787+ def __init__ (self , middleware : list [AgentMiddleware ]) -> None :
776788 self ._middleware = middleware
777- self ._model = model
778789
779790 def _with_model_middleware (
780791 self , model_invoke : ModelMiddlewareHandler
@@ -843,7 +854,7 @@ async def awrap_model_call(
843854 request .state ["messages" ].append (request .runtime .context .retry )
844855 request .runtime .context .retry = False
845856
846- req = _convert_model_request_from_lc (request , self . _model )
857+ req = _convert_model_request_from_lc (request )
847858 final_handler = _convert_model_handler_from_lc (handler , original_request = request )
848859
849860 async def llm_handler (req : ModelRequest ) -> ModelResponse :
@@ -935,7 +946,7 @@ async def awrap_tool_call(
935946 call = _map_tool_call_from_langchain (request .tool_call )
936947
937948 if isinstance (call , ToolCall ):
938- req = _convert_tool_request_from_lc (request , self . _model )
949+ req = _convert_tool_request_from_lc (request )
939950 final_handler = _convert_tool_handler_from_lc (handler , original_request = request )
940951 sdk_response = await self ._with_tool_call_middleware (final_handler )(req )
941952
@@ -961,7 +972,7 @@ async def awrap_tool_call(
961972 artifact = sdk_result ,
962973 )
963974
964- req = _convert_subagent_request_from_lc (request , self . _model )
975+ req = _convert_subagent_request_from_lc (request )
965976 final_handler = _convert_subagent_handler_from_lc (handler , original_request = request )
966977 sdk_response = await self ._with_subagent_call_middleware (final_handler )(req )
967978
@@ -1036,32 +1047,31 @@ async def _sdk_handler(request: ModelRequest) -> ModelResponse:
10361047 return _sdk_handler
10371048
10381049
1039- def _convert_model_request_from_lc (request : LC_ModelRequest , model : BaseChatModel ) -> ModelRequest :
1050+ def _convert_model_request_from_lc (request : LC_ModelRequest ) -> ModelRequest :
10401051 thread_id = request .runtime .context .thread_id
10411052
10421053 system_message = request .system_message .content .__str__ () if request .system_message else ""
10431054
10441055 return ModelRequest (
10451056 system_message = system_message ,
1046- state = _convert_agent_state_from_langchain (request .state , model , thread_id ),
1057+ state = _convert_agent_state_from_langchain (request .state , thread_id ),
10471058 )
10481059
10491060
1050- def _convert_tool_request_from_lc (request : LC_ToolCallRequest , model : BaseChatModel ) -> ToolRequest :
1061+ def _convert_tool_request_from_lc (request : LC_ToolCallRequest ) -> ToolRequest :
10511062 assert isinstance (request .runtime .context , InvokeContext )
10521063 thread_id = request .runtime .context .thread_id
10531064
10541065 tool_call = _map_tool_call_from_langchain (request .tool_call )
10551066 assert isinstance (tool_call , ToolCall ), "Expected tool call"
10561067 return ToolRequest (
10571068 call = tool_call ,
1058- state = _convert_agent_state_from_langchain (request .state , model , thread_id ),
1069+ state = _convert_agent_state_from_langchain (request .state , thread_id ),
10591070 )
10601071
10611072
10621073def _convert_subagent_request_from_lc (
10631074 request : LC_ToolCallRequest ,
1064- model : BaseChatModel ,
10651075) -> SubagentRequest :
10661076 assert isinstance (request .runtime .context , InvokeContext )
10671077 thread_id = request .runtime .context .thread_id
@@ -1070,7 +1080,7 @@ def _convert_subagent_request_from_lc(
10701080 assert isinstance (subagent_call , SubagentCall ), "Expected subagent call"
10711081 return SubagentRequest (
10721082 call = subagent_call ,
1073- state = _convert_agent_state_from_langchain (request .state , model , thread_id ),
1083+ state = _convert_agent_state_from_langchain (request .state , thread_id ),
10741084 )
10751085
10761086
@@ -1738,30 +1748,29 @@ def _map_message_to_langchain(message: BaseMessage) -> LC_AnyMessage:
17381748 raise InvalidMessageTypeError ("Invalid SDK message type" )
17391749
17401750
1741- def _convert_agent_state_from_langchain (
1742- state : LC_AgentState [Any ], model : BaseChatModel , thread_id : str
1743- ) -> AgentState :
1751+ def _convert_agent_state_from_langchain (state : LC_AgentState [Any ], thread_id : str ) -> AgentState :
17441752 messages = state ["messages" ]
1745- total_tokens_counter = _get_approximate_token_counter (model )
1746- total_tokens = total_tokens_counter (messages )
17471753 messages = [_map_message_from_langchain (m ) for m in state ["messages" ]]
17481754 return AgentState (
17491755 messages = messages ,
1750- total_steps = len (messages ),
1751- token_count = total_tokens ,
17521756 thread_id = thread_id ,
17531757 )
17541758
17551759
1756- def _get_approximate_token_counter (model : BaseChatModel ) -> LC_TokenCounter :
1760+ def _get_approximate_token_counter (
1761+ model : BaseChatModel , tools : list [BaseTool | dict [str , Any ]]
1762+ ) -> LC_TokenCounter :
17571763 """Tune parameters of approximate token counter based on model type."""
17581764
1765+ # TODO: consider using use_usage_metadata_scaling option once
1766+ # we expose token usage details from LLMs.
1767+
17591768 # NOTE: This is adapted from the backend provider library
17601769 # 3.3 was estimated in an offline experiment, comparing with Claude's token-counting
17611770 # API: https://platform.claude.com/docs/en/build-with-claude/token-counting
17621771 if model ._llm_type == ANTHROPIC_CHAT_MODEL_TYPE : # pyright: ignore[reportPrivateUsage]
1763- return partial (count_tokens_approximately , chars_per_token = 3.3 )
1764- return count_tokens_approximately
1772+ return partial (count_tokens_approximately , tools = tools , chars_per_token = 3.3 )
1773+ return partial ( count_tokens_approximately , tools = tools )
17651774
17661775
17671776def _create_langchain_model (model : PredefinedModel ) -> BaseChatModel :
@@ -1970,25 +1979,6 @@ def check_tool_name(type: str, name: str) -> None:
19701979 raise _InvalidMessagesException ("last AIMessage has tool calls" )
19711980
19721981
1973- class _TokenLimitMiddleware (AgentMiddleware ):
1974- """Stops agent execution when the token count of messages passed to the model exceeds the given limit."""
1975-
1976- _limit : int
1977-
1978- def __init__ (self , limit : int ) -> None :
1979- self ._limit = limit
1980-
1981- @override
1982- async def model_middleware (
1983- self ,
1984- request : ModelRequest ,
1985- handler : ModelMiddlewareHandler ,
1986- ) -> ModelResponse :
1987- if request .state .token_count >= self ._limit :
1988- raise TokenLimitExceededException (token_limit = self ._limit )
1989- return await handler (request )
1990-
1991-
19921982class _StepLimitMiddleware (AgentMiddleware ):
19931983 """Stops agent execution when the number of steps taken reaches the given limit."""
19941984
@@ -2003,7 +1993,7 @@ async def model_middleware(
20031993 request : ModelRequest ,
20041994 handler : ModelMiddlewareHandler ,
20051995 ) -> ModelResponse :
2006- if request .state .total_steps >= self ._limit :
1996+ if len ( request .state .messages ) >= self ._limit :
20071997 raise StepsLimitExceededException (steps_limit = self ._limit )
20081998 return await handler (request )
20091999
@@ -2032,9 +2022,7 @@ async def agent_middleware(
20322022 ) -> AgentResponse [Any | None ]:
20332023 try :
20342024 # Agent loop starting.
2035- self ._deadline_per_thread_id [request .thread_id ] = (
2036- monotonic () + self ._seconds
2037- )
2025+ self ._deadline_per_thread_id [request .thread_id ] = monotonic () + self ._seconds
20382026 return await handler (request )
20392027 finally :
20402028 del self ._deadline_per_thread_id [request .thread_id ] # don't leak memory
0 commit comments