|
|
@@ -6,7 +6,6 @@ from concurrent.futures import Executor
|
|
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
|
|
-from app.models.token_relation import RelationType
|
|
|
from config.config import settings
|
|
|
from config.llm import llm_settings, tool_settings
|
|
|
|
|
|
@@ -46,6 +45,19 @@ class ThreadRunner:
|
|
|
tool_settings.TOOL_WORKER_NUM, "tool_worker_"
|
|
|
)
|
|
|
|
|
|
+ # ---- token 估算常量(chars/4 启发式)----
|
|
|
+ CHARS_PER_TOKEN = 4
|
|
|
+ ROLE_OVERHEAD_TOKENS = 4
|
|
|
+
|
|
|
+ # ---- 上下文预算(软阈值,基于 token)----
|
|
|
+ MAX_CHAT_TOKENS = 30000
|
|
|
+ KEEP_RECENT_TOKENS = 6000
|
|
|
+ MIN_KEEP_TURNS = 1
|
|
|
+ MAX_CHAT_IMAGES = 6
|
|
|
+ MAX_SINGLE_TEXT_TOKENS = 2000
|
|
|
+ SUMMARY_MAX_TOKENS = 500
|
|
|
+ TOOL_OUTPUT_MAX_TOKENS = 1500
|
|
|
+
|
|
|
def __init__(
|
|
|
self, run_id: str, token_id: str, session: Session, stream: bool = False
|
|
|
):
|
|
|
@@ -57,82 +69,159 @@ class ThreadRunner:
|
|
|
self.event_handler: StreamEventHandler = None
|
|
|
|
|
|
def run(self):
|
|
|
- """
|
|
|
- 完成一次 run 的执行,基本步骤
|
|
|
- 1. 初始化,获取 run 以及相关 tools, 构造 system instructions;
|
|
|
- 2. 开始循环,查询已有 run step, 进行 chat message 生成;
|
|
|
- 3. 调用 llm 并解析返回结果;
|
|
|
- 4. 根据返回结果,生成新的 run step(tool calls 处理) 或者 message
|
|
|
- """
|
|
|
- # TODO: 重构,将 run 的状态变更逻辑放到 RunService 中
|
|
|
- run = RunService.get_run_sync(session=self.session, run_id=self.run_id)
|
|
|
- self.event_handler = StreamEventHandler(
|
|
|
- run_id=self.run_id, is_stream=self.stream
|
|
|
- )
|
|
|
+ try:
|
|
|
+ run = RunService.get_run_sync(session=self.session, run_id=self.run_id)
|
|
|
+ self.event_handler = StreamEventHandler(
|
|
|
+ run_id=self.run_id, is_stream=self.stream
|
|
|
+ )
|
|
|
|
|
|
- run = RunService.to_in_progress(session=self.session, run_id=self.run_id)
|
|
|
- self.event_handler.pub_run_in_progress(run)
|
|
|
- logging.info("processing ThreadRunner task, run_id: %s", self.run_id)
|
|
|
+ run = RunService.to_in_progress(session=self.session, run_id=self.run_id)
|
|
|
+ self.event_handler.pub_run_in_progress(run)
|
|
|
+ logging.info("processing ThreadRunner task, run_id: %s", self.run_id)
|
|
|
|
|
|
- # get memory from assistant metadata
|
|
|
- # format likes {"memory": {"type": "window", "window_size": 20, "max_token_size": 4000}}
|
|
|
- ast = AssistantService.get_assistant_sync(
|
|
|
- session=self.session, assistant_id=run.assistant_id
|
|
|
- )
|
|
|
- metadata = ast.metadata_ or {}
|
|
|
- memory = find_memory(metadata.get("memory", {}))
|
|
|
+ ast = AssistantService.get_assistant_sync(
|
|
|
+ session=self.session, assistant_id=run.assistant_id
|
|
|
+ )
|
|
|
+ metadata = ast.metadata_ or {}
|
|
|
+ memory = find_memory(metadata.get("memory", {}))
|
|
|
|
|
|
- instructions = (
|
|
|
- [run.instructions or ""] if run.instructions else [ast.instructions or ""]
|
|
|
- )
|
|
|
+ instructions = (
|
|
|
+ [run.instructions or ""]
|
|
|
+ if run.instructions
|
|
|
+ else [ast.instructions or ""]
|
|
|
+ )
|
|
|
|
|
|
- asst_ids = []
|
|
|
- ids = []
|
|
|
- if ast.tool_resources and "file_search" in ast.tool_resources:
|
|
|
+ asst_ids = []
|
|
|
+ ids = []
|
|
|
+ if ast.tool_resources and "file_search" in ast.tool_resources:
|
|
|
+ ids = (
|
|
|
+ ast.tool_resources.get("file_search")
|
|
|
+ .get("vector_stores")[0]
|
|
|
+ .get("folder_ids")
|
|
|
+ )
|
|
|
+ if ids:
|
|
|
+ asst_ids += ids
|
|
|
+ ids = (
|
|
|
+ ast.tool_resources.get("file_search")
|
|
|
+ .get("vector_stores")[0]
|
|
|
+ .get("file_ids")
|
|
|
+ )
|
|
|
+ if ids:
|
|
|
+ asst_ids += ids
|
|
|
+
|
|
|
+ if len(asst_ids) > 0:
|
|
|
+ if not run.tools:
|
|
|
+ run.tools = []
|
|
|
+ if run.file_ids and len(run.file_ids) > 0:
|
|
|
+ if not any(
|
|
|
+ t.get("type") == "knowledge_search" for t in run.tools
|
|
|
+ ):
|
|
|
+ run.tools.append({"type": "knowledge_search"})
|
|
|
+ else:
|
|
|
+ for tool in run.tools:
|
|
|
+ if tool.get("type") == "file_search":
|
|
|
+ tool["type"] = "knowledge_search"
|
|
|
+
|
|
|
+ tools = find_tools(run, self.session)
|
|
|
+ for tool in tools:
|
|
|
+ tool.configure(session=self.session, run=run)
|
|
|
+ instruction_supplement = tool.instruction_supplement()
|
|
|
+ if instruction_supplement:
|
|
|
+ instructions += [instruction_supplement or ""]
|
|
|
+ instruction = "\n".join(instructions)
|
|
|
+
|
|
|
+ llm = self.__init_llm_backend(run.assistant_id)
|
|
|
+ loop = True
|
|
|
+ while loop:
|
|
|
+ run_steps = RunStepService.get_run_step_list(
|
|
|
+ session=self.session, run_id=self.run_id, thread_id=run.thread_id
|
|
|
+ )
|
|
|
+ loop = self.__run_step(
|
|
|
+ llm, run, run_steps, instruction, tools, memory
|
|
|
+ )
|
|
|
|
|
|
- ids = (
|
|
|
- ast.tool_resources.get("file_search")
|
|
|
- .get("vector_stores")[0]
|
|
|
- .get("folder_ids")
|
|
|
+ latest_run = RunService.get_run_sync(
|
|
|
+ session=self.session, run_id=self.run_id
|
|
|
)
|
|
|
- if ids:
|
|
|
- asst_ids += ids
|
|
|
- ids = (
|
|
|
- ast.tool_resources.get("file_search")
|
|
|
- .get("vector_stores")[0]
|
|
|
- .get("file_ids")
|
|
|
- )
|
|
|
- if ids:
|
|
|
- asst_ids += ids
|
|
|
-
|
|
|
- if len(asst_ids) > 0:
|
|
|
- if len(run.file_ids) > 0:
|
|
|
- run.tools.append({"type": "knowledge_search"})
|
|
|
- else:
|
|
|
- for tool in run.tools:
|
|
|
- if tool.get("type") == "file_search":
|
|
|
- tool["type"] = "knowledge_search"
|
|
|
-
|
|
|
- tools = find_tools(run, self.session)
|
|
|
- for tool in tools:
|
|
|
- tool.configure(session=self.session, run=run)
|
|
|
- instruction_supplement = tool.instruction_supplement()
|
|
|
- if instruction_supplement:
|
|
|
- instructions += [instruction_supplement or ""]
|
|
|
- instruction = "\n".join(instructions)
|
|
|
-
|
|
|
- llm = self.__init_llm_backend(run.assistant_id)
|
|
|
- loop = True
|
|
|
- while loop:
|
|
|
- #print("looplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooplooploop" )
|
|
|
- run_steps = RunStepService.get_run_step_list(
|
|
|
- session=self.session, run_id=self.run_id, thread_id=run.thread_id
|
|
|
+ if latest_run.status == "requires_action":
|
|
|
+ self.event_handler.pub_done()
|
|
|
+ return
|
|
|
+
|
|
|
+ self.event_handler.pub_run_completed(latest_run)
|
|
|
+ self.event_handler.pub_done()
|
|
|
+ except Exception as e:
|
|
|
+ logging.exception("ThreadRunner run failed, run_id=%s", self.run_id)
|
|
|
+ try:
|
|
|
+ RunService.to_failed(
|
|
|
+ session=self.session, run_id=self.run_id, last_error=e
|
|
|
+ )
|
|
|
+ except Exception:
|
|
|
+ logging.exception("mark run failed error, run_id=%s", self.run_id)
|
|
|
+ try:
|
|
|
+ if self.event_handler:
|
|
|
+ self.event_handler.pub_done()
|
|
|
+ except Exception:
|
|
|
+ logging.exception("pub_done error, run_id=%s", self.run_id)
|
|
|
+ raise
|
|
|
+
|
|
|
+ # ------------------------------------------------------------------
|
|
|
+ # token 估算
|
|
|
+ # ------------------------------------------------------------------
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def _estimate_text_tokens(cls, text: str) -> int:
|
|
|
+ if not text:
|
|
|
+ return 0
|
|
|
+ return (len(text) + cls.CHARS_PER_TOKEN - 1) // cls.CHARS_PER_TOKEN
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def _estimate_message_tokens(cls, message: dict) -> int:
|
|
|
+ content = message.get("content")
|
|
|
+ if isinstance(content, str):
|
|
|
+ return cls._estimate_text_tokens(content) + cls.ROLE_OVERHEAD_TOKENS
|
|
|
+ if isinstance(content, list):
|
|
|
+ total = cls.ROLE_OVERHEAD_TOKENS
|
|
|
+ for c in content:
|
|
|
+ if c.get("type") == "text":
|
|
|
+ total += cls._estimate_text_tokens(c.get("text", ""))
|
|
|
+ elif c.get("type") in ("image_url", "input_audio"):
|
|
|
+ total += 256
|
|
|
+ return total
|
|
|
+ return cls.ROLE_OVERHEAD_TOKENS
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _dedupe_tools(tools_param):
|
|
|
+ if not tools_param:
|
|
|
+ return tools_param
|
|
|
+ seen = set()
|
|
|
+ result = []
|
|
|
+ for t in tools_param:
|
|
|
+ name = (t.get("function") or {}).get("name") or t.get("name")
|
|
|
+ if name:
|
|
|
+ if name in seen:
|
|
|
+ logging.warning("duplicate tool name removed: %s", name)
|
|
|
+ continue
|
|
|
+ seen.add(name)
|
|
|
+ result.append(t)
|
|
|
+ return result
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _sanitize_extra_body(extra_body, model):
|
|
|
+ if not extra_body:
|
|
|
+ return extra_body
|
|
|
+ cleaned = dict(extra_body)
|
|
|
+ removed = []
|
|
|
+ for k in list(cleaned):
|
|
|
+ if "reasoning" in k.lower() or "thinking" in k.lower():
|
|
|
+ removed.append(k)
|
|
|
+ cleaned.pop(k)
|
|
|
+ if removed:
|
|
|
+ logging.warning(
|
|
|
+ "stripped reasoning params from extra_body, model=%s, keys=%s",
|
|
|
+ model,
|
|
|
+ removed,
|
|
|
)
|
|
|
- loop = self.__run_step(llm, run, run_steps, instruction, tools, memory)
|
|
|
-
|
|
|
- # 任务结束
|
|
|
- self.event_handler.pub_run_completed(run)
|
|
|
- self.event_handler.pub_done()
|
|
|
+ return cleaned or None
|
|
|
|
|
|
def __run_step(
|
|
|
self,
|
|
|
@@ -143,10 +232,7 @@ class ThreadRunner:
|
|
|
tools: List[BaseTool],
|
|
|
memory: Memory,
|
|
|
):
|
|
|
- """
|
|
|
- 执行 run step
|
|
|
- """
|
|
|
- logging.info("step %s is running", run_steps)
|
|
|
+ logging.debug("step %s is running", run_steps)
|
|
|
if instruction == "":
|
|
|
instruction = (
|
|
|
"You are a multilingual AI assistant.\n"
|
|
|
@@ -160,7 +246,6 @@ class ThreadRunner:
|
|
|
|
|
|
assistant_system_message = [msg_util.system_message(instruction)]
|
|
|
|
|
|
- # 获取已有 message 上下文记录
|
|
|
chat_messages = self.__generate_chat_messages(
|
|
|
MessageService.get_message_list(
|
|
|
session=self.session, thread_id=run.thread_id
|
|
|
@@ -170,65 +255,59 @@ class ThreadRunner:
|
|
|
|
|
|
tool_call_messages = []
|
|
|
for step in run_steps:
|
|
|
- logging.info("stepstep %s is running", step)
|
|
|
+ logging.debug("stepstep %s is running", step)
|
|
|
if step.type == "tool_calls" and step.status == "completed":
|
|
|
tool_call_messages += (
|
|
|
self.__convert_assistant_tool_calls_to_chat_messages(step)
|
|
|
)
|
|
|
- # tool_call_messages = tool_call_messages
|
|
|
- # memory
|
|
|
+
|
|
|
+ tool_call_messages = self._prune_tool_outputs(tool_call_messages)
|
|
|
+
|
|
|
messages = (
|
|
|
assistant_system_message
|
|
|
+ memory.integrate_context(chat_messages)
|
|
|
+ tool_call_messages
|
|
|
)
|
|
|
|
|
|
- logging.info("messages: run %s", run)
|
|
|
- logging.info(messages)
|
|
|
- logging.info(tools)
|
|
|
- # 判断本次是否允许调用工具
|
|
|
+ logging.debug("messages: run %s", run)
|
|
|
+ logging.debug(messages)
|
|
|
+ logging.debug(tools)
|
|
|
+
|
|
|
use_tools = len(run_steps) < self.max_step
|
|
|
tool_choice = "auto" if use_tools else "none"
|
|
|
|
|
|
- # 当不允许调用工具时,不传递 tools 参数,避免与 tool_choice='none' 冲突
|
|
|
- tools_param = [tool.openai_function for tool in tools] if use_tools else None
|
|
|
+ tools_param = (
|
|
|
+ self._dedupe_tools([tool.openai_function for tool in tools])
|
|
|
+ if use_tools
|
|
|
+ else None
|
|
|
+ )
|
|
|
|
|
|
- # 仅在流式且需要 usage 统计时才传递 stream_options
|
|
|
stream_options = run.stream_options if self.stream else None
|
|
|
+ extra_body = self._sanitize_extra_body(run.extra_body, run.model)
|
|
|
+
|
|
|
+ logging.info(
|
|
|
+ "llm request: model=%s extra_body=%s tools_count=%s",
|
|
|
+ run.model,
|
|
|
+ extra_body,
|
|
|
+ len(tools_param or []),
|
|
|
+ )
|
|
|
|
|
|
response_stream = llm.run(
|
|
|
messages=messages,
|
|
|
model=run.model,
|
|
|
- tools=tools_param, # 不使用工具时为 None
|
|
|
+ tools=tools_param,
|
|
|
tool_choice=tool_choice,
|
|
|
stream=self.stream,
|
|
|
stream_options=stream_options,
|
|
|
- extra_body=run.extra_body,
|
|
|
+ extra_body=extra_body,
|
|
|
temperature=run.temperature or 1.0,
|
|
|
top_p=run.top_p,
|
|
|
response_format=run.response_format,
|
|
|
- parallel_tool_calls=run.parallel_tool_calls if use_tools else None, # 同样可条件传递
|
|
|
+ parallel_tool_calls=run.parallel_tool_calls if use_tools else None,
|
|
|
audio=run.audio,
|
|
|
modalities=run.modalities,
|
|
|
)
|
|
|
- '''
|
|
|
- response_stream = llm.run(
|
|
|
- messages=messages,
|
|
|
- model=run.model,
|
|
|
- tools=[tool.openai_function for tool in tools],
|
|
|
- tool_choice="auto" if len(run_steps) < self.max_step else "none",
|
|
|
- stream=self.stream,
|
|
|
- stream_options=run.stream_options,
|
|
|
- extra_body=run.extra_body,
|
|
|
- temperature=run.temperature or 1.0,
|
|
|
- top_p=run.top_p,
|
|
|
- response_format=run.response_format,
|
|
|
- parallel_tool_calls=run.parallel_tool_calls,
|
|
|
- audio=run.audio,
|
|
|
- modalities=run.modalities,
|
|
|
- )
|
|
|
- '''
|
|
|
- # create message callback
|
|
|
+
|
|
|
create_message_callback = partial(
|
|
|
MessageService.new_message,
|
|
|
session=self.session,
|
|
|
@@ -238,7 +317,6 @@ class ThreadRunner:
|
|
|
role="assistant",
|
|
|
)
|
|
|
|
|
|
- # create 'message creation' run step callback
|
|
|
def _create_message_creation_run_step(message_id):
|
|
|
return RunStepService.new_run_step(
|
|
|
session=self.session,
|
|
|
@@ -260,21 +338,26 @@ class ThreadRunner:
|
|
|
)
|
|
|
if self.stream == False and hasattr(response_stream, "choices"):
|
|
|
response_stream = [response_stream]
|
|
|
+
|
|
|
response_msg = llm_callback_handler.handle_llm_response(response_stream)
|
|
|
- #print("response_msg================================================================response_msg")
|
|
|
- #print(llm_callback_handler)
|
|
|
message_creation_run_step = llm_callback_handler.step
|
|
|
- #print("444444444444444444444444455555555577777777777777777777777")
|
|
|
- logging.info("chat_response_message: %s", response_msg)
|
|
|
+ logging.debug("chat_response_message: %s", response_msg)
|
|
|
|
|
|
if msg_util.is_tool_call(response_msg):
|
|
|
- # tool & tool_call definition dict
|
|
|
- tool_calls = [
|
|
|
- tool_call_recognize(tool_call, tools)
|
|
|
- for tool_call in response_msg.tool_calls
|
|
|
- ]
|
|
|
+ tool_calls = []
|
|
|
+ for tc in response_msg.tool_calls:
|
|
|
+ recognized = tool_call_recognize(tc, tools)
|
|
|
+ if (
|
|
|
+ not recognized
|
|
|
+ or not isinstance(recognized, (tuple, list))
|
|
|
+ or len(recognized) != 2
|
|
|
+ ):
|
|
|
+ logging.warning(
|
|
|
+ "invalid tool_call_recognize result: %r", recognized
|
|
|
+ )
|
|
|
+ continue
|
|
|
+ tool_calls.append(recognized)
|
|
|
|
|
|
- # new run step for tool calls
|
|
|
new_run_step = RunStepService.new_run_step(
|
|
|
session=self.session,
|
|
|
type="tool_calls",
|
|
|
@@ -284,26 +367,21 @@ class ThreadRunner:
|
|
|
step_details={
|
|
|
"reasoning_content": response_msg.reasoning_content,
|
|
|
"type": "tool_calls",
|
|
|
- "tool_calls": [tool_call_dict for _, tool_call_dict in tool_calls],
|
|
|
+ "tool_calls": [
|
|
|
+ tool_call_dict for _, tool_call_dict in tool_calls
|
|
|
+ ],
|
|
|
},
|
|
|
)
|
|
|
self.event_handler.pub_run_step_created(new_run_step)
|
|
|
self.event_handler.pub_run_step_in_progress(new_run_step)
|
|
|
|
|
|
- internal_tool_calls = list(
|
|
|
- filter(lambda _tool_calls: _tool_calls[0] is not None, tool_calls)
|
|
|
- )
|
|
|
-
|
|
|
+ internal_tool_calls = [tc for tc in tool_calls if tc[0] is not None]
|
|
|
external_tool_call_dict = [
|
|
|
- tool_call_dict for tool, tool_call_dict in tool_calls if tool is None
|
|
|
+ tc[1] for tc in tool_calls if tc[0] is None
|
|
|
]
|
|
|
|
|
|
- # 为减少线程同步逻辑,依次处理内/外 tool_call 调用
|
|
|
if internal_tool_calls:
|
|
|
try:
|
|
|
- #print("==========================internal_tool_callsinternal_tool_callsinternal_tool_calls")
|
|
|
- #print(internal_tool_calls)
|
|
|
- ## 线程执行有问题 可以改成异步, 这里如果是filesearch要确定只执行一次
|
|
|
tool_calls_with_outputs = run_with_executor(
|
|
|
executor=ThreadRunner.tool_executor,
|
|
|
func=internal_tool_call_invoke,
|
|
|
@@ -315,33 +393,42 @@ class ThreadRunner:
|
|
|
run_step_id=new_run_step.id,
|
|
|
step_details={
|
|
|
"type": "tool_calls",
|
|
|
- #"reasoning_content": response_msg.reasoning_content,
|
|
|
"tool_calls": tool_calls_with_outputs,
|
|
|
},
|
|
|
completed=not external_tool_call_dict,
|
|
|
)
|
|
|
- #print("llm_callback_handler.message================================================================llm_callback_handler.message")
|
|
|
- #print(llm_callback_handler.message)
|
|
|
self.event_handler.pub_message_delta_tool(
|
|
|
message_id=llm_callback_handler.message.id,
|
|
|
index=0,
|
|
|
- content=json.dumps(tool_calls_with_outputs)
|
|
|
+ content=json.dumps(tool_calls_with_outputs),
|
|
|
)
|
|
|
except Exception as e:
|
|
|
RunStepService.to_failed(
|
|
|
- session=self.session, run_step_id=new_run_step.id, last_error=e
|
|
|
+ session=self.session,
|
|
|
+ run_step_id=new_run_step.id,
|
|
|
+ last_error=e,
|
|
|
)
|
|
|
+ try:
|
|
|
+ RunService.to_failed(
|
|
|
+ session=self.session,
|
|
|
+ run_id=run.id,
|
|
|
+ last_error=e,
|
|
|
+ )
|
|
|
+ except Exception:
|
|
|
+ logging.exception(
|
|
|
+ "mark run failed error, run_id=%s", run.id
|
|
|
+ )
|
|
|
raise e
|
|
|
- #print("aaaaaaaaaaaaaaa===============================================================8888888888888888888888888")
|
|
|
- #print(external_tool_call_dict)
|
|
|
+
|
|
|
if external_tool_call_dict:
|
|
|
- # run 设置为 action required,等待业务完成更新并再次拉起
|
|
|
run = RunService.to_requires_action(
|
|
|
session=self.session,
|
|
|
run_id=run.id,
|
|
|
required_action={
|
|
|
"type": "submit_tool_outputs",
|
|
|
- "submit_tool_outputs": {"tool_calls": external_tool_call_dict},
|
|
|
+ "submit_tool_outputs": {
|
|
|
+ "tool_calls": external_tool_call_dict
|
|
|
+ },
|
|
|
},
|
|
|
)
|
|
|
self.event_handler.pub_run_step_delta(
|
|
|
@@ -351,7 +438,6 @@ class ThreadRunner:
|
|
|
"tool_calls": external_tool_call_dict,
|
|
|
},
|
|
|
)
|
|
|
- #print(run)
|
|
|
self.event_handler.pub_run_requires_action(run)
|
|
|
else:
|
|
|
self.event_handler.pub_run_step_completed(new_run_step)
|
|
|
@@ -364,12 +450,14 @@ class ThreadRunner:
|
|
|
if response_msg.reasoning_content == "":
|
|
|
response_msg.reasoning_content = None
|
|
|
|
|
|
- # 无 tool call 信息,message 生成结束,更新状态
|
|
|
new_message = MessageService.modify_message_sync(
|
|
|
session=self.session,
|
|
|
thread_id=run.thread_id,
|
|
|
message_id=llm_callback_handler.message.id,
|
|
|
- body=MessageUpdate(content=response_msg.content, reasoning_content=response_msg.reasoning_content),
|
|
|
+ body=MessageUpdate(
|
|
|
+ content=response_msg.content,
|
|
|
+ reasoning_content=response_msg.reasoning_content,
|
|
|
+ ),
|
|
|
)
|
|
|
self.event_handler.pub_message_completed(new_message)
|
|
|
|
|
|
@@ -388,9 +476,7 @@ class ThreadRunner:
|
|
|
return False
|
|
|
|
|
|
def __init_llm_backend(self, assistant_id):
|
|
|
- #print("settings.AUTH_ENABLE", settings.AUTH_ENABLE)
|
|
|
if settings.AUTH_ENABLE:
|
|
|
- # init llm backend with token id
|
|
|
if self.token_id:
|
|
|
token_id = self.token_id
|
|
|
else:
|
|
|
@@ -399,13 +485,9 @@ class ThreadRunner:
|
|
|
relation_type=RelationType.Assistant,
|
|
|
relation_id=assistant_id,
|
|
|
)
|
|
|
- #print("token_idtoken_idtoken_idtoken_idtoken_idtoken_idtoken_idtoken_idtoken_idtoken_idtoken_idtoken_id")
|
|
|
- #print(self.token_id)
|
|
|
- #print(token_id)
|
|
|
try:
|
|
|
if token_id is not None and len(token_id) > 0:
|
|
|
token = TokenService.get_token_by_id(self.session, token_id)
|
|
|
- #print(token)
|
|
|
return LLMBackend(
|
|
|
base_url=token.llm_base_url, api_key=token.llm_api_key
|
|
|
)
|
|
|
@@ -416,87 +498,323 @@ class ThreadRunner:
|
|
|
"llm_api_key": "sk-vTqeBKDC2j6osbGt89A2202dAd1c4fE8B1D294388b569e54",
|
|
|
}
|
|
|
return LLMBackend(
|
|
|
- base_url=token.get("llm_base_url"), api_key=token.get("llm_api_key")
|
|
|
+ base_url=token.get("llm_base_url"),
|
|
|
+ api_key=token.get("llm_api_key"),
|
|
|
)
|
|
|
else:
|
|
|
- # init llm backend with llm settings
|
|
|
return LLMBackend(
|
|
|
base_url=llm_settings.OPENAI_API_BASE,
|
|
|
api_key=llm_settings.OPENAI_API_KEY,
|
|
|
)
|
|
|
|
|
|
- def __generate_chat_messages(self, messages: List[Message], run: Run):
|
|
|
- """
|
|
|
- 根据历史信息生成 chat message
|
|
|
- """
|
|
|
+ # ------------------------------------------------------------------
|
|
|
+ # 工具结果剪枝
|
|
|
+ # ------------------------------------------------------------------
|
|
|
+
|
|
|
+ def _prune_tool_outputs(self, tool_call_messages: List[dict]) -> List[dict]:
|
|
|
+ if not tool_call_messages:
|
|
|
+ return tool_call_messages
|
|
|
+
|
|
|
+ result = []
|
|
|
+ for msg in tool_call_messages:
|
|
|
+ content = msg.get("content")
|
|
|
+ if isinstance(content, str):
|
|
|
+ tokens = self._estimate_text_tokens(content)
|
|
|
+ if tokens > self.TOOL_OUTPUT_MAX_TOKENS:
|
|
|
+ msg = dict(msg)
|
|
|
+ msg["content"] = (
|
|
|
+ f"[cwl-stub: truncated {tokens} tokens]\n"
|
|
|
+ + self._truncate_text_by_tokens(
|
|
|
+ content, self.TOOL_OUTPUT_MAX_TOKENS
|
|
|
+ )
|
|
|
+ )
|
|
|
+ elif isinstance(content, list):
|
|
|
+ needs_prune = any(
|
|
|
+ c.get("type") == "text"
|
|
|
+ and self._estimate_text_tokens(c.get("text", ""))
|
|
|
+ > self.TOOL_OUTPUT_MAX_TOKENS
|
|
|
+ for c in content
|
|
|
+ )
|
|
|
+ if needs_prune:
|
|
|
+ msg = dict(msg)
|
|
|
+ new_content = []
|
|
|
+ for c in content:
|
|
|
+ if c.get("type") == "text":
|
|
|
+ text = c.get("text", "")
|
|
|
+ tokens = self._estimate_text_tokens(text)
|
|
|
+ if tokens > self.TOOL_OUTPUT_MAX_TOKENS:
|
|
|
+ text = (
|
|
|
+ f"[cwl-stub: truncated {tokens} tokens]\n"
|
|
|
+ + self._truncate_text_by_tokens(
|
|
|
+ text, self.TOOL_OUTPUT_MAX_TOKENS
|
|
|
+ )
|
|
|
+ )
|
|
|
+ new_content.append({"type": "text", "text": text})
|
|
|
+ else:
|
|
|
+ new_content.append(c)
|
|
|
+ msg["content"] = new_content
|
|
|
+ result.append(msg)
|
|
|
+ return result
|
|
|
+
|
|
|
+ # ------------------------------------------------------------------
|
|
|
+ # 上下文压缩(token 预算 + 轮次配对)
|
|
|
+ # ------------------------------------------------------------------
|
|
|
+
|
|
|
+ def _truncate_text_by_tokens(self, text: str, max_tokens: int) -> str:
|
|
|
+ limit_chars = max_tokens * self.CHARS_PER_TOKEN
|
|
|
+ if len(text) <= limit_chars:
|
|
|
+ return text
|
|
|
+ head = limit_chars * 2 // 3
|
|
|
+ tail = limit_chars - head
|
|
|
+ removed = len(text) - limit_chars
|
|
|
+ marker = f"\n...[truncated {removed} chars]...\n"
|
|
|
+ if tail <= 0:
|
|
|
+ return text[:head] + marker
|
|
|
+ return text[:head] + marker + text[-tail:]
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _content_to_text(content) -> str:
|
|
|
+ if isinstance(content, str):
|
|
|
+ return content
|
|
|
+ if isinstance(content, list):
|
|
|
+ return " ".join(
|
|
|
+ c.get("text", "")
|
|
|
+ for c in content
|
|
|
+ if c.get("type") == "text" and c.get("text")
|
|
|
+ )
|
|
|
+ return ""
|
|
|
+
|
|
|
+ def _normalize_history(self, messages: List[Message], run: Run) -> List[dict]:
|
|
|
+ ordered = list(reversed(messages))
|
|
|
+
|
|
|
+ image_slots = []
|
|
|
+ for mi, m in enumerate(ordered):
|
|
|
+ if m.role != "user":
|
|
|
+ continue
|
|
|
+ for ci, c in enumerate(m.content or []):
|
|
|
+ if c.get("type") == "image_url" and run.audio is None:
|
|
|
+ image_slots.append((mi, ci))
|
|
|
+ if len(image_slots) > self.MAX_CHAT_IMAGES:
|
|
|
+ keep_image_slots = set(image_slots[-self.MAX_CHAT_IMAGES:])
|
|
|
+ else:
|
|
|
+ keep_image_slots = set(image_slots)
|
|
|
|
|
|
- chat_messages = []
|
|
|
+ normalized: List[dict] = []
|
|
|
is_audio_num = 0
|
|
|
- for message in messages:
|
|
|
- role = message.role
|
|
|
+
|
|
|
+ for mi, m in enumerate(ordered):
|
|
|
+ role = m.role
|
|
|
if role == "user":
|
|
|
- message_content = []
|
|
|
- """
|
|
|
- if message.file_ids:
|
|
|
- files = FileService.get_file_list_by_ids(
|
|
|
- session=self.session, file_ids=message.file_ids
|
|
|
- )
|
|
|
- for file in files:
|
|
|
- chat_messages.append(
|
|
|
- msg_util.new_message(
|
|
|
- role,
|
|
|
- f'The file "{file.filename}" can be used as a reference',
|
|
|
- )
|
|
|
- )
|
|
|
- else:
|
|
|
- """
|
|
|
- for content in message.content:
|
|
|
- if content["type"] == "text":
|
|
|
- message_content.append(
|
|
|
- {"type": "text", "text": content["text"]["value"]}
|
|
|
+ contents = []
|
|
|
+ for ci, c in enumerate(m.content or []):
|
|
|
+ ctype = c.get("type")
|
|
|
+ if ctype == "text":
|
|
|
+ text = (c.get("text") or {}).get("value") or ""
|
|
|
+ text = self._truncate_text_by_tokens(
|
|
|
+ text, self.MAX_SINGLE_TEXT_TOKENS
|
|
|
)
|
|
|
-
|
|
|
- elif content["type"] == "image_url" and run.audio is None:
|
|
|
- message_content.append(content)
|
|
|
+ if text:
|
|
|
+ contents.append({"type": "text", "text": text})
|
|
|
+ elif ctype == "image_url" and run.audio is None:
|
|
|
+ if (mi, ci) in keep_image_slots:
|
|
|
+ contents.append(c)
|
|
|
+ else:
|
|
|
+ contents.append(
|
|
|
+ {"type": "text", "text": "[image omitted]"}
|
|
|
+ )
|
|
|
elif (
|
|
|
- content.get("type") == "input_audio"
|
|
|
+ ctype == "input_audio"
|
|
|
and run.audio is not None
|
|
|
and is_audio_num < 2
|
|
|
):
|
|
|
- message_content.append(content)
|
|
|
+ contents.append(c)
|
|
|
is_audio_num += 1
|
|
|
- chat_messages.append(msg_util.new_message(role, message_content))
|
|
|
+ if contents:
|
|
|
+ normalized.append(
|
|
|
+ {
|
|
|
+ "role": "user",
|
|
|
+ "content": contents,
|
|
|
+ "reasoning_content": None,
|
|
|
+ }
|
|
|
+ )
|
|
|
elif role == "assistant":
|
|
|
- assistant_content = ""
|
|
|
- for content in message.content:
|
|
|
- if content["type"] == "text":
|
|
|
- assistant_content += content["text"]["value"]
|
|
|
- if assistant_content:
|
|
|
- chat_messages.append(msg_util.new_message(role, assistant_content, message.reasoning_content))
|
|
|
- #chat_messages.append(msg_util.new_message(role, assistant_content, message.reasoning_content))
|
|
|
+ text_parts = []
|
|
|
+ for c in m.content or []:
|
|
|
+ if c.get("type") == "text":
|
|
|
+ text_parts.append(
|
|
|
+ (c.get("text") or {}).get("value") or ""
|
|
|
+ )
|
|
|
+ assistant_content = "".join(text_parts)
|
|
|
+ if not assistant_content:
|
|
|
+ continue
|
|
|
+ assistant_content = self._truncate_text_by_tokens(
|
|
|
+ assistant_content, self.MAX_SINGLE_TEXT_TOKENS
|
|
|
+ )
|
|
|
+ reasoning = m.reasoning_content
|
|
|
+ if reasoning:
|
|
|
+ reasoning = self._truncate_text_by_tokens(
|
|
|
+ reasoning, self.MAX_SINGLE_TEXT_TOKENS
|
|
|
+ )
|
|
|
+ normalized.append(
|
|
|
+ {
|
|
|
+ "role": "assistant",
|
|
|
+ "content": assistant_content,
|
|
|
+ "reasoning_content": reasoning,
|
|
|
+ }
|
|
|
+ )
|
|
|
+ return normalized
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _group_turns(normalized: List[dict]) -> List[List[dict]]:
|
|
|
+ turns: List[List[dict]] = []
|
|
|
+ for m in normalized:
|
|
|
+ if m["role"] == "user":
|
|
|
+ if turns and turns[-1][-1]["role"] == "assistant":
|
|
|
+ turns.append([m])
|
|
|
+ elif turns:
|
|
|
+ turns[-1].append(m)
|
|
|
+ else:
|
|
|
+ turns.append([m])
|
|
|
+ else:
|
|
|
+ if turns:
|
|
|
+ turns[-1].append(m)
|
|
|
+ else:
|
|
|
+ turns.append([m])
|
|
|
+ return turns
|
|
|
+
|
|
|
+ def _build_summary(self, old_messages: List[dict]) -> str:
|
|
|
+ if not old_messages:
|
|
|
+ return ""
|
|
|
+ parts = []
|
|
|
+ for m in old_messages:
|
|
|
+ text = self._content_to_text(m["content"]).strip()
|
|
|
+ if not text:
|
|
|
+ continue
|
|
|
+ prefix = "User" if m["role"] == "user" else "Assistant"
|
|
|
+ parts.append(f"{prefix}: {text}")
|
|
|
+ summary = "\n".join(parts)
|
|
|
+ summary = self._truncate_text_by_tokens(
|
|
|
+ summary, self.SUMMARY_MAX_TOKENS
|
|
|
+ )
|
|
|
+ return summary
|
|
|
+
|
|
|
+ def _compress_history(self, normalized: List[dict]) -> List[dict]:
|
|
|
+ if not normalized:
|
|
|
+ return normalized
|
|
|
+ if len(normalized) <= max(self.MIN_KEEP_TURNS * 2, 2):
|
|
|
+ return normalized
|
|
|
+
|
|
|
+ total_tokens = sum(
|
|
|
+ self._estimate_message_tokens(m) for m in normalized
|
|
|
+ )
|
|
|
+ if total_tokens <= self.MAX_CHAT_TOKENS:
|
|
|
+ return normalized
|
|
|
+
|
|
|
+ turns = self._group_turns(normalized)
|
|
|
+ if not turns:
|
|
|
+ return normalized
|
|
|
+
|
|
|
+ keep_idx = len(turns) - 1
|
|
|
+ kept_tokens = sum(
|
|
|
+ self._estimate_message_tokens(m) for m in turns[-1]
|
|
|
+ )
|
|
|
+
|
|
|
+ for i in range(len(turns) - 2, -1, -1):
|
|
|
+ turn = turns[i]
|
|
|
+ turn_tokens = sum(
|
|
|
+ self._estimate_message_tokens(m) for m in turn
|
|
|
+ )
|
|
|
+ if kept_tokens + turn_tokens > self.KEEP_RECENT_TOKENS:
|
|
|
+ break
|
|
|
+ keep_idx = i
|
|
|
+ kept_tokens += turn_tokens
|
|
|
+
|
|
|
+ old_turns = turns[:keep_idx]
|
|
|
+ recent_turns = turns[keep_idx:]
|
|
|
+
|
|
|
+ while recent_turns and recent_turns[0][0]["role"] == "assistant":
|
|
|
+ old_turns.append(recent_turns.pop(0))
|
|
|
+
|
|
|
+ while len(recent_turns) < self.MIN_KEEP_TURNS and old_turns:
|
|
|
+ recent_turns.insert(0, old_turns.pop())
|
|
|
+
|
|
|
+ summary = ""
|
|
|
+ if old_turns:
|
|
|
+ old_messages = [m for turn in old_turns for m in turn]
|
|
|
+ summary = self._build_summary(old_messages)
|
|
|
+
|
|
|
+ result: List[dict] = []
|
|
|
+ if summary:
|
|
|
+ summary_text = f"[Earlier conversation summary]\n{summary}"
|
|
|
+ if recent_turns and recent_turns[0][0]["role"] == "user":
|
|
|
+ first_msg = dict(recent_turns[0][0])
|
|
|
+ first_content = list(first_msg.get("content") or [])
|
|
|
+ first_msg["content"] = [
|
|
|
+ {"type": "text", "text": summary_text + "\n\n---\n\n"}
|
|
|
+ ] + first_content
|
|
|
+ recent_turns[0] = [first_msg] + list(recent_turns[0][1:])
|
|
|
+ else:
|
|
|
+ result.append(
|
|
|
+ {
|
|
|
+ "role": "user",
|
|
|
+ "content": [{"type": "text", "text": summary_text}],
|
|
|
+ "reasoning_content": None,
|
|
|
+ }
|
|
|
+ )
|
|
|
|
|
|
- chat_messages.reverse() # 倒序排列,最新的消息在前面
|
|
|
- return chat_messages # 暂时只支持5条消息,后续正价token上限
|
|
|
+ for turn in recent_turns:
|
|
|
+ result.extend(turn)
|
|
|
+
|
|
|
+ if result and result[0]["role"] == "assistant":
|
|
|
+ result.insert(
|
|
|
+ 0,
|
|
|
+ {
|
|
|
+ "role": "user",
|
|
|
+ "content": [
|
|
|
+ {"type": "text", "text": "[Earlier context omitted]"}
|
|
|
+ ],
|
|
|
+ "reasoning_content": None,
|
|
|
+ },
|
|
|
+ )
|
|
|
+
|
|
|
+ return result
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _build_chat_messages(normalized: List[dict]) -> List[dict]:
|
|
|
+ chat_messages = []
|
|
|
+ for m in normalized:
|
|
|
+ if m["role"] == "user":
|
|
|
+ chat_messages.append(
|
|
|
+ msg_util.new_message("user", m["content"])
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ chat_messages.append(
|
|
|
+ msg_util.new_message(
|
|
|
+ "assistant", m["content"], m["reasoning_content"]
|
|
|
+ )
|
|
|
+ )
|
|
|
+ return chat_messages
|
|
|
+
|
|
|
+ def __generate_chat_messages(self, messages: List[Message], run: Run):
|
|
|
+ normalized = self._normalize_history(messages, run)
|
|
|
+ normalized = self._compress_history(normalized)
|
|
|
+ return self._build_chat_messages(normalized)
|
|
|
|
|
|
def __convert_assistant_tool_calls_to_chat_messages(self, run_step: RunStep):
|
|
|
- """
|
|
|
- 根据 run step 执行结果生成 message 信息
|
|
|
- 每个 tool call run step 包含两部分,调用与结果(结果可能为多个信息)
|
|
|
- """
|
|
|
tool_calls = run_step.step_details["tool_calls"]
|
|
|
reasoning_content = run_step.step_details.get("reasoning_content")
|
|
|
tool_call_requests = [
|
|
|
msg_util.tool_calls(
|
|
|
[tool_call_request(tool_call) for tool_call in tool_calls],
|
|
|
- reasoning_content
|
|
|
+ reasoning_content,
|
|
|
)
|
|
|
]
|
|
|
- logging.info("tool_calls: %s", tool_calls)
|
|
|
+ logging.debug("tool_calls: %s", tool_calls)
|
|
|
tool_call_outputs = [
|
|
|
msg_util.tool_call_result(
|
|
|
tool_call_id(tool_call), tool_call_output(tool_call)
|
|
|
)
|
|
|
for tool_call in tool_calls
|
|
|
]
|
|
|
- logging.info("tool_call_outputs: %s", tool_call_outputs)
|
|
|
- return tool_call_requests + tool_call_outputs
|
|
|
+ logging.debug("tool_call_outputs: %s", tool_call_outputs)
|
|
|
+ return tool_call_requests + tool_call_outputs
|