Fix CLI streaming to use iter_content with smaller chunks for real-time output

parent 47738566
...@@ -416,54 +416,61 @@ class CoderClient: ...@@ -416,54 +416,61 @@ class CoderClient:
tool_calls = [] tool_calls = []
current_tool_call = None current_tool_call = None
for line in response.iter_lines(): # Use iter_content with smaller chunk size for better real-time handling
if not line: buffer = ""
continue for chunk in response.iter_content(chunk_size=256, decode_unicode=True):
if chunk:
line = line.decode('utf-8') buffer += chunk
# Handle SSE format
if line.startswith('data: '):
line = line[6:]
if line == '[DONE]':
break
try:
data = json.loads(line)
delta = data.get('choices', [{}])[0].get('delta', {})
# Handle content
content = delta.get('content')
if content:
print(content, end='', flush=True)
full_content += content
# Handle tool calls # Process complete lines from buffer
delta_tool_calls = delta.get('tool_calls') while '\n' in buffer:
if delta_tool_calls: line, buffer = buffer.split('\n', 1)
for tc in delta_tool_calls:
index = tc.get('index', 0) # Handle SSE format
if line.startswith('data: '):
line = line[6:]
if line == '[DONE]':
break
if not line:
continue
try:
data = json.loads(line)
delta = data.get('choices', [{}])[0].get('delta', {})
# Extend tool_calls list if needed # Handle content
while len(tool_calls) <= index: content = delta.get('content')
tool_calls.append({ if content:
'id': '', print(content, end='', flush=True)
'type': 'function', full_content += content
'function': {'name': '', 'arguments': ''}
})
# Update tool call # Handle tool calls
if 'id' in tc: delta_tool_calls = delta.get('tool_calls')
tool_calls[index]['id'] = tc['id'] if delta_tool_calls:
if 'function' in tc: for tc in delta_tool_calls:
if 'name' in tc['function']: index = tc.get('index', 0)
tool_calls[index]['function']['name'] = tc['function']['name']
if 'arguments' in tc['function']: # Extend tool_calls list if needed
tool_calls[index]['function']['arguments'] += tc['function']['arguments'] while len(tool_calls) <= index:
tool_calls.append({
except json.JSONDecodeError: 'id': '',
continue 'type': 'function',
'function': {'name': '', 'arguments': ''}
})
# Update tool call
if 'id' in tc:
tool_calls[index]['id'] = tc['id']
if 'function' in tc:
if 'name' in tc['function']:
tool_calls[index]['function']['name'] = tc['function']['name']
if 'arguments' in tc['function']:
tool_calls[index]['function']['arguments'] += tc['function']['arguments']
except json.JSONDecodeError:
continue
print() # Newline after streaming print() # Newline after streaming
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment