from functools import partial import logging import json from typing import List from concurrent.futures import Executor from sqlalchemy.orm import Session from config.config import settings from config.llm import llm_settings, tool_settings from app.core.runner.llm_backend import LLMBackend from app.core.runner.llm_callback_handler import LLMCallbackHandler from app.core.runner.memory import Memory, find_memory from app.core.runner.pub_handler import StreamEventHandler from app.core.runner.utils import message_util as msg_util from app.core.runner.utils.tool_call_util import ( tool_call_recognize, internal_tool_call_invoke, tool_call_request, tool_call_id, tool_call_output, ) from app.core.tools import find_tools, BaseTool from app.libs.thread_executor import get_executor_for_config, run_with_executor from app.models.message import Message, MessageUpdate from app.models.run import Run from app.models.run_step import RunStep from app.models.token_relation import RelationType from app.services.assistant.assistant import AssistantService from app.services.file.file import FileService from app.services.message.message import MessageService from app.services.run.run import RunService from app.services.run.run_step import RunStepService from app.services.token.token import TokenService from app.services.token.token_relation import TokenRelationService class ThreadRunner: """ ThreadRunner 封装 run 的执行逻辑 """ tool_executor: Executor = get_executor_for_config( tool_settings.TOOL_WORKER_NUM, "tool_worker_" ) # ---- token 估算常量(chars/4 启发式)---- CHARS_PER_TOKEN = 4 ROLE_OVERHEAD_TOKENS = 4 # ---- 上下文预算(软阈值,基于 token)---- MAX_CHAT_TOKENS = 30000 KEEP_RECENT_TOKENS = 6000 MIN_KEEP_TURNS = 1 MAX_CHAT_IMAGES = 6 MAX_SINGLE_TEXT_TOKENS = 2000 SUMMARY_MAX_TOKENS = 500 TOOL_OUTPUT_MAX_TOKENS = 1500 def __init__( self, run_id: str, token_id: str, session: Session, stream: bool = False ): self.run_id = run_id self.token_id = token_id self.session = session self.stream = stream self.max_step = llm_settings.LLM_MAX_STEP self.event_handler: StreamEventHandler = None def run(self): try: run = RunService.get_run_sync(session=self.session, run_id=self.run_id) self.event_handler = StreamEventHandler( run_id=self.run_id, is_stream=self.stream ) run = RunService.to_in_progress(session=self.session, run_id=self.run_id) self.event_handler.pub_run_in_progress(run) logging.info("processing ThreadRunner task, run_id: %s", self.run_id) ast = AssistantService.get_assistant_sync( session=self.session, assistant_id=run.assistant_id ) metadata = ast.metadata_ or {} memory = find_memory(metadata.get("memory", {})) instructions = ( [run.instructions or ""] if run.instructions else [ast.instructions or ""] ) asst_ids = [] ids = [] if ast.tool_resources and "file_search" in ast.tool_resources: ids = ( ast.tool_resources.get("file_search") .get("vector_stores")[0] .get("folder_ids") ) if ids: asst_ids += ids ids = ( ast.tool_resources.get("file_search") .get("vector_stores")[0] .get("file_ids") ) if ids: asst_ids += ids if len(asst_ids) > 0: if not run.tools: run.tools = [] if run.file_ids and len(run.file_ids) > 0: if not any( t.get("type") == "knowledge_search" for t in run.tools ): run.tools.append({"type": "knowledge_search"}) else: for tool in run.tools: if tool.get("type") == "file_search": tool["type"] = "knowledge_search" tools = find_tools(run, self.session) for tool in tools: tool.configure(session=self.session, run=run) instruction_supplement = tool.instruction_supplement() if instruction_supplement: instructions += [instruction_supplement or ""] instruction = "\n".join(instructions) llm = self.__init_llm_backend(run.assistant_id) loop = True while loop: run_steps = RunStepService.get_run_step_list( session=self.session, run_id=self.run_id, thread_id=run.thread_id ) loop = self.__run_step( llm, run, run_steps, instruction, tools, memory ) latest_run = RunService.get_run_sync( session=self.session, run_id=self.run_id ) if latest_run.status == "requires_action": self.event_handler.pub_done() return self.event_handler.pub_run_completed(latest_run) self.event_handler.pub_done() except Exception as e: logging.exception("ThreadRunner run failed, run_id=%s", self.run_id) try: RunService.to_failed( session=self.session, run_id=self.run_id, last_error=e ) except Exception: logging.exception("mark run failed error, run_id=%s", self.run_id) try: if self.event_handler: self.event_handler.pub_done() except Exception: logging.exception("pub_done error, run_id=%s", self.run_id) raise # ------------------------------------------------------------------ # token 估算 # ------------------------------------------------------------------ @classmethod def _estimate_text_tokens(cls, text: str) -> int: if not text: return 0 return (len(text) + cls.CHARS_PER_TOKEN - 1) // cls.CHARS_PER_TOKEN @classmethod def _estimate_message_tokens(cls, message: dict) -> int: content = message.get("content") if isinstance(content, str): return cls._estimate_text_tokens(content) + cls.ROLE_OVERHEAD_TOKENS if isinstance(content, list): total = cls.ROLE_OVERHEAD_TOKENS for c in content: if c.get("type") == "text": total += cls._estimate_text_tokens(c.get("text", "")) elif c.get("type") in ("image_url", "input_audio"): total += 256 return total return cls.ROLE_OVERHEAD_TOKENS @staticmethod def _dedupe_tools(tools_param): if not tools_param: return tools_param seen = set() result = [] for t in tools_param: name = (t.get("function") or {}).get("name") or t.get("name") if name: if name in seen: logging.warning("duplicate tool name removed: %s", name) continue seen.add(name) result.append(t) return result @staticmethod def _sanitize_extra_body(extra_body, model): if not extra_body: return extra_body cleaned = dict(extra_body) removed = [] for k in list(cleaned): if "reasoning" in k.lower() or "thinking" in k.lower(): removed.append(k) cleaned.pop(k) if removed: logging.warning( "stripped reasoning params from extra_body, model=%s, keys=%s", model, removed, ) return cleaned or None def __run_step( self, llm: LLMBackend, run: Run, run_steps: List[RunStep], instruction: str, tools: List[BaseTool], memory: Memory, ): logging.debug("step %s is running", run_steps) if instruction == "": instruction = ( "You are a multilingual AI assistant.\n" + "- Detect user language; reply in same language unless told otherwise.\n" + "- Default to English if detection is unclear.\n" + "- Give concise, accurate, and safe answers; admit when unsure.\n" + "- Keep tone and style consistent; adapt examples to user's context.\n" + "- For code, include explanations and comments in user's language.\n" + "- If a question is ambiguous, ask for clarification.\n" ) assistant_system_message = [msg_util.system_message(instruction)] chat_messages = self.__generate_chat_messages( MessageService.get_message_list( session=self.session, thread_id=run.thread_id ), run, ) tool_call_messages = [] for step in run_steps: logging.debug("stepstep %s is running", step) if step.type == "tool_calls" and step.status == "completed": tool_call_messages += ( self.__convert_assistant_tool_calls_to_chat_messages(step) ) tool_call_messages = self._prune_tool_outputs(tool_call_messages) messages = ( assistant_system_message + memory.integrate_context(chat_messages) + tool_call_messages ) logging.debug("messages: run %s", run) logging.debug(messages) logging.debug(tools) use_tools = len(run_steps) < self.max_step tool_choice = "auto" if use_tools else "none" tools_param = ( self._dedupe_tools([tool.openai_function for tool in tools]) if use_tools else None ) stream_options = run.stream_options if self.stream else None extra_body = self._sanitize_extra_body(run.extra_body, run.model) logging.info( "llm request: model=%s extra_body=%s tools_count=%s", run.model, extra_body, len(tools_param or []), ) response_stream = llm.run( messages=messages, model=run.model, tools=tools_param, tool_choice=tool_choice, stream=self.stream, stream_options=stream_options, extra_body=extra_body, temperature=run.temperature or 1.0, top_p=run.top_p, response_format=run.response_format, parallel_tool_calls=run.parallel_tool_calls if use_tools else None, audio=run.audio, modalities=run.modalities, ) create_message_callback = partial( MessageService.new_message, session=self.session, assistant_id=run.assistant_id, thread_id=run.thread_id, run_id=run.id, role="assistant", ) def _create_message_creation_run_step(message_id): return RunStepService.new_run_step( session=self.session, type="message_creation", assistant_id=run.assistant_id, thread_id=run.thread_id, run_id=run.id, step_details={ "type": "message_creation", "message_creation": {"message_id": message_id}, }, ) llm_callback_handler = LLMCallbackHandler( run_id=run.id, on_step_create_func=_create_message_creation_run_step, on_message_create_func=create_message_callback, event_handler=self.event_handler, ) if self.stream == False and hasattr(response_stream, "choices"): response_stream = [response_stream] response_msg = llm_callback_handler.handle_llm_response(response_stream) message_creation_run_step = llm_callback_handler.step logging.debug("chat_response_message: %s", response_msg) if msg_util.is_tool_call(response_msg): tool_calls = [] for tc in response_msg.tool_calls: recognized = tool_call_recognize(tc, tools) if ( not recognized or not isinstance(recognized, (tuple, list)) or len(recognized) != 2 ): logging.warning( "invalid tool_call_recognize result: %r", recognized ) continue tool_calls.append(recognized) new_run_step = RunStepService.new_run_step( session=self.session, type="tool_calls", assistant_id=run.assistant_id, thread_id=run.thread_id, run_id=run.id, step_details={ "reasoning_content": response_msg.reasoning_content, "type": "tool_calls", "tool_calls": [ tool_call_dict for _, tool_call_dict in tool_calls ], }, ) self.event_handler.pub_run_step_created(new_run_step) self.event_handler.pub_run_step_in_progress(new_run_step) internal_tool_calls = [tc for tc in tool_calls if tc[0] is not None] external_tool_call_dict = [ tc[1] for tc in tool_calls if tc[0] is None ] if internal_tool_calls: try: tool_calls_with_outputs = run_with_executor( executor=ThreadRunner.tool_executor, func=internal_tool_call_invoke, tasks=internal_tool_calls, timeout=tool_settings.TOOL_WORKER_EXECUTION_TIMEOUT, ) new_run_step = RunStepService.update_step_details( session=self.session, run_step_id=new_run_step.id, step_details={ "type": "tool_calls", "tool_calls": tool_calls_with_outputs, }, completed=not external_tool_call_dict, ) self.event_handler.pub_message_delta_tool( message_id=llm_callback_handler.message.id, index=0, content=json.dumps(tool_calls_with_outputs), ) except Exception as e: RunStepService.to_failed( session=self.session, run_step_id=new_run_step.id, last_error=e, ) try: RunService.to_failed( session=self.session, run_id=run.id, last_error=e, ) except Exception: logging.exception( "mark run failed error, run_id=%s", run.id ) raise e if external_tool_call_dict: run = RunService.to_requires_action( session=self.session, run_id=run.id, required_action={ "type": "submit_tool_outputs", "submit_tool_outputs": { "tool_calls": external_tool_call_dict }, }, ) self.event_handler.pub_run_step_delta( step_id=new_run_step.id, step_details={ "type": "tool_calls", "tool_calls": external_tool_call_dict, }, ) self.event_handler.pub_run_requires_action(run) else: self.event_handler.pub_run_step_completed(new_run_step) return True else: if response_msg.content == "": response_msg.content = ( '[{"text": {"value": "", "annotations": []}, "type": "text"}]' ) if response_msg.reasoning_content == "": response_msg.reasoning_content = None new_message = MessageService.modify_message_sync( session=self.session, thread_id=run.thread_id, message_id=llm_callback_handler.message.id, body=MessageUpdate( content=response_msg.content, reasoning_content=response_msg.reasoning_content, ), ) self.event_handler.pub_message_completed(new_message) new_step = RunStepService.update_step_details( session=self.session, run_step_id=message_creation_run_step.id, step_details={ "type": "message_creation", "message_creation": {"message_id": new_message.id}, }, completed=True, ) RunService.to_completed(session=self.session, run_id=run.id) self.event_handler.pub_run_step_completed(new_step) return False def __init_llm_backend(self, assistant_id): if settings.AUTH_ENABLE: if self.token_id: token_id = self.token_id else: token_id = TokenRelationService.get_token_id_by_relation( session=self.session, relation_type=RelationType.Assistant, relation_id=assistant_id, ) try: if token_id is not None and len(token_id) > 0: token = TokenService.get_token_by_id(self.session, token_id) return LLMBackend( base_url=token.llm_base_url, api_key=token.llm_api_key ) except Exception as e: print(e) token = { "llm_base_url": "http://172.16.12.13:3000/v1", "llm_api_key": "sk-vTqeBKDC2j6osbGt89A2202dAd1c4fE8B1D294388b569e54", } return LLMBackend( base_url=token.get("llm_base_url"), api_key=token.get("llm_api_key"), ) else: return LLMBackend( base_url=llm_settings.OPENAI_API_BASE, api_key=llm_settings.OPENAI_API_KEY, ) # ------------------------------------------------------------------ # 工具结果剪枝 # ------------------------------------------------------------------ def _prune_tool_outputs(self, tool_call_messages: List[dict]) -> List[dict]: if not tool_call_messages: return tool_call_messages result = [] for msg in tool_call_messages: content = msg.get("content") if isinstance(content, str): tokens = self._estimate_text_tokens(content) if tokens > self.TOOL_OUTPUT_MAX_TOKENS: msg = dict(msg) msg["content"] = ( f"[cwl-stub: truncated {tokens} tokens]\n" + self._truncate_text_by_tokens( content, self.TOOL_OUTPUT_MAX_TOKENS ) ) elif isinstance(content, list): needs_prune = any( c.get("type") == "text" and self._estimate_text_tokens(c.get("text", "")) > self.TOOL_OUTPUT_MAX_TOKENS for c in content ) if needs_prune: msg = dict(msg) new_content = [] for c in content: if c.get("type") == "text": text = c.get("text", "") tokens = self._estimate_text_tokens(text) if tokens > self.TOOL_OUTPUT_MAX_TOKENS: text = ( f"[cwl-stub: truncated {tokens} tokens]\n" + self._truncate_text_by_tokens( text, self.TOOL_OUTPUT_MAX_TOKENS ) ) new_content.append({"type": "text", "text": text}) else: new_content.append(c) msg["content"] = new_content result.append(msg) return result # ------------------------------------------------------------------ # 上下文压缩(token 预算 + 轮次配对) # ------------------------------------------------------------------ def _truncate_text_by_tokens(self, text: str, max_tokens: int) -> str: limit_chars = max_tokens * self.CHARS_PER_TOKEN if len(text) <= limit_chars: return text head = limit_chars * 2 // 3 tail = limit_chars - head removed = len(text) - limit_chars marker = f"\n...[truncated {removed} chars]...\n" if tail <= 0: return text[:head] + marker return text[:head] + marker + text[-tail:] @staticmethod def _content_to_text(content) -> str: if isinstance(content, str): return content if isinstance(content, list): return " ".join( c.get("text", "") for c in content if c.get("type") == "text" and c.get("text") ) return "" def _normalize_history(self, messages: List[Message], run: Run) -> List[dict]: ordered = list(reversed(messages)) image_slots = [] for mi, m in enumerate(ordered): if m.role != "user": continue for ci, c in enumerate(m.content or []): if c.get("type") == "image_url" and run.audio is None: image_slots.append((mi, ci)) if len(image_slots) > self.MAX_CHAT_IMAGES: keep_image_slots = set(image_slots[-self.MAX_CHAT_IMAGES:]) else: keep_image_slots = set(image_slots) normalized: List[dict] = [] is_audio_num = 0 for mi, m in enumerate(ordered): role = m.role if role == "user": contents = [] for ci, c in enumerate(m.content or []): ctype = c.get("type") if ctype == "text": text = (c.get("text") or {}).get("value") or "" text = self._truncate_text_by_tokens( text, self.MAX_SINGLE_TEXT_TOKENS ) if text: contents.append({"type": "text", "text": text}) elif ctype == "image_url" and run.audio is None: if (mi, ci) in keep_image_slots: contents.append(c) else: contents.append( {"type": "text", "text": "[image omitted]"} ) elif ( ctype == "input_audio" and run.audio is not None and is_audio_num < 2 ): contents.append(c) is_audio_num += 1 if contents: normalized.append( { "role": "user", "content": contents, "reasoning_content": None, } ) elif role == "assistant": text_parts = [] for c in m.content or []: if c.get("type") == "text": text_parts.append( (c.get("text") or {}).get("value") or "" ) assistant_content = "".join(text_parts) if not assistant_content: continue assistant_content = self._truncate_text_by_tokens( assistant_content, self.MAX_SINGLE_TEXT_TOKENS ) reasoning = m.reasoning_content if reasoning: reasoning = self._truncate_text_by_tokens( reasoning, self.MAX_SINGLE_TEXT_TOKENS ) normalized.append( { "role": "assistant", "content": assistant_content, "reasoning_content": reasoning, } ) return normalized @staticmethod def _group_turns(normalized: List[dict]) -> List[List[dict]]: turns: List[List[dict]] = [] for m in normalized: if m["role"] == "user": if turns and turns[-1][-1]["role"] == "assistant": turns.append([m]) elif turns: turns[-1].append(m) else: turns.append([m]) else: if turns: turns[-1].append(m) else: turns.append([m]) return turns def _build_summary(self, old_messages: List[dict]) -> str: if not old_messages: return "" parts = [] for m in old_messages: text = self._content_to_text(m["content"]).strip() if not text: continue prefix = "User" if m["role"] == "user" else "Assistant" parts.append(f"{prefix}: {text}") summary = "\n".join(parts) summary = self._truncate_text_by_tokens( summary, self.SUMMARY_MAX_TOKENS ) return summary def _compress_history(self, normalized: List[dict]) -> List[dict]: if not normalized: return normalized if len(normalized) <= max(self.MIN_KEEP_TURNS * 2, 2): return normalized total_tokens = sum( self._estimate_message_tokens(m) for m in normalized ) if total_tokens <= self.MAX_CHAT_TOKENS: return normalized turns = self._group_turns(normalized) if not turns: return normalized keep_idx = len(turns) - 1 kept_tokens = sum( self._estimate_message_tokens(m) for m in turns[-1] ) for i in range(len(turns) - 2, -1, -1): turn = turns[i] turn_tokens = sum( self._estimate_message_tokens(m) for m in turn ) if kept_tokens + turn_tokens > self.KEEP_RECENT_TOKENS: break keep_idx = i kept_tokens += turn_tokens old_turns = turns[:keep_idx] recent_turns = turns[keep_idx:] while recent_turns and recent_turns[0][0]["role"] == "assistant": old_turns.append(recent_turns.pop(0)) while len(recent_turns) < self.MIN_KEEP_TURNS and old_turns: recent_turns.insert(0, old_turns.pop()) summary = "" if old_turns: old_messages = [m for turn in old_turns for m in turn] summary = self._build_summary(old_messages) result: List[dict] = [] if summary: summary_text = f"[Earlier conversation summary]\n{summary}" if recent_turns and recent_turns[0][0]["role"] == "user": first_msg = dict(recent_turns[0][0]) first_content = list(first_msg.get("content") or []) first_msg["content"] = [ {"type": "text", "text": summary_text + "\n\n---\n\n"} ] + first_content recent_turns[0] = [first_msg] + list(recent_turns[0][1:]) else: result.append( { "role": "user", "content": [{"type": "text", "text": summary_text}], "reasoning_content": None, } ) for turn in recent_turns: result.extend(turn) if result and result[0]["role"] == "assistant": result.insert( 0, { "role": "user", "content": [ {"type": "text", "text": "[Earlier context omitted]"} ], "reasoning_content": None, }, ) return result @staticmethod def _build_chat_messages(normalized: List[dict]) -> List[dict]: chat_messages = [] for m in normalized: if m["role"] == "user": chat_messages.append( msg_util.new_message("user", m["content"]) ) else: chat_messages.append( msg_util.new_message( "assistant", m["content"], m["reasoning_content"] ) ) return chat_messages def __generate_chat_messages(self, messages: List[Message], run: Run): normalized = self._normalize_history(messages, run) normalized = self._compress_history(normalized) return self._build_chat_messages(normalized) def __convert_assistant_tool_calls_to_chat_messages(self, run_step: RunStep): tool_calls = run_step.step_details["tool_calls"] reasoning_content = run_step.step_details.get("reasoning_content") tool_call_requests = [ msg_util.tool_calls( [tool_call_request(tool_call) for tool_call in tool_calls], reasoning_content, ) ] logging.debug("tool_calls: %s", tool_calls) tool_call_outputs = [ msg_util.tool_call_result( tool_call_id(tool_call), tool_call_output(tool_call) ) for tool_call in tool_calls ] logging.debug("tool_call_outputs: %s", tool_call_outputs) return tool_call_requests + tool_call_outputs