diff --git a/README.md b/README.md index ffa40ca..11c5725 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ -H "Authorization: Bearer YOUR_KEY" \ -d '{ - "model": "gpt-3.5-turbo", + "model": "qwen2.5-0.5B-p256-ax630c", "messages": [{"role": "user", "content": "Hello!"}], "temperature": 0.7 }' @@ -90,7 +90,7 @@ curl -X POST "http://localhost:8000/v1/audio/speech" \ -H "Content-Type: application/json" \ -H "Authorization: Bearer YOUR_KEY" \ -d '{ - "model": "tts-1", + "model": "melotts", "input": "Hello world!", "voice": "alloy" }' diff --git a/api_server.py b/api_server.py index f123b65..cec8f8e 100644 --- a/api_server.py +++ b/api_server.py @@ -1,17 +1,14 @@ import os import uuid import yaml -from fastapi import FastAPI, Request, HTTPException, File, Form, UploadFile -from fastapi.responses import JSONResponse, StreamingResponse import logging -from slowapi import Limiter -from slowapi.util import get_remote_address import time import json import asyncio +from fastapi import FastAPI, Request, HTTPException, File, Form, UploadFile +from fastapi.responses import JSONResponse, StreamingResponse from backend import ( - TestBackend, OpenAIProxyBackend, LlmClientBackend, VisionModelBackend, @@ -34,7 +31,6 @@ logging.basicConfig( logger = logging.getLogger("api") app = FastAPI(title="OpenAI Compatible API Server") -limiter = Limiter(key_func=get_remote_address) class Config: def __init__(self): @@ -113,9 +109,7 @@ async def chat_completions(request: Request, body: ChatCompletionRequest): detail=f"Unsupported model: {body.model}" ) - try: - print(f"Received request: {body.model_dump()}") - + try: if body.stream: chunk_generator = await backend.generate(body) if not chunk_generator: @@ -133,7 +127,6 @@ async def chat_completions(request: Request, body: ChatCompletionRequest): chunk_dict = chunk.model_dump() json_chunk = json.dumps(chunk_dict, ensure_ascii=False) - print(f"Sending chunk: {json_chunk}") yield f"data: {json_chunk}\n\n" except asyncio.CancelledError: logger.warning("Client disconnected early, terminating inference...") @@ -150,7 +143,6 @@ async def chat_completions(request: Request, body: ChatCompletionRequest): ) else: response = await backend.generate(body) - print(f"Sending response: {response}") return JSONResponse(content=response) except HTTPException as he: diff --git a/backend/__init__.py b/backend/__init__.py index d9a6ece..436f22d 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -1,4 +1,3 @@ -from .test_backend import TestBackend from .openai_proxy_backend import OpenAIProxyBackend from .llm_client_backend import LlmClientBackend from .tts_client_backend import TtsClientBackend diff --git a/backend/llm_client_backend.py b/backend/llm_client_backend.py index 847196a..a9b8a26 100644 --- a/backend/llm_client_backend.py +++ b/backend/llm_client_backend.py @@ -20,7 +20,7 @@ class LlmClientBackend(BaseModelBackend): self._active_clients = {} self._pool_lock = asyncio.Lock() self.logger = logging.getLogger("api.llm") - self.MAX_CONTEXT_LENGTH = model_config.get("max_context_length", 200) + self.MAX_CONTEXT_LENGTH = model_config.get("max_context_length", 128) self.POOL_SIZE = model_config.get("pool_size", 2) self._inference_executor = ThreadPoolExecutor(max_workers=self.POOL_SIZE) self._active_tasks = weakref.WeakSet() diff --git a/backend/test_backend.py b/backend/test_backend.py deleted file mode 100644 index 96343a8..0000000 --- a/backend/test_backend.py +++ /dev/null @@ -1,56 +0,0 @@ -import uuid -import time -from .base_model_backend import BaseModelBackend -from .chat_schemas import ChatCompletionRequest - -class TestBackend(BaseModelBackend): - async def generate(self, request: ChatCompletionRequest): - if request.stream: - async def chunk_generator(): - content_parts = ["hello", "Module", "LLM"] - messages=[m.model_dump() for m in request.messages] - print(f"messages:_____________{messages}______________") - for i, part in enumerate(content_parts): - yield { - "id": f"chatcmpl-{uuid.uuid4()}", - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": request.model, - "choices": [{ - "index": 0, - "delta": { - "content": part, - "role": "assistant" if i == 0 else None, - "function_call": None, - "tool_calls": None - }, - "logprobs": None, - "finish_reason": "stop" if i == len(content_parts)-1 else None - }], - "service_tier": None, - "system_fingerprint": None, - "usage": None - } - return chunk_generator() - else: - return { - "id": f"chatcmpl-{uuid.uuid4()}", - "object": "chat.completion", - "created": int(time.time()), - "model": request.model, - "choices": [{ - "message": { - "role": "assistant", - "content": "Hello Module LLM", - "function_call": None, - "tool_calls": None - }, - "finish_reason": "stop", - "index": 0 - }], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 20, - "total_tokens": 30 - } - } \ No newline at end of file diff --git a/client/asr_client.py b/client/asr_client.py index c807b9e..79134c6 100644 --- a/client/asr_client.py +++ b/client/asr_client.py @@ -117,21 +117,4 @@ class ASRClient: for chunk in self.inference_stream(audio_b64, object_type="asr.base64"): full_text += chunk - return full_text - -if __name__ == "__main__": - with ASRClient(host='192.168.20.183') as client: - setup_response = client.setup("whisper.setup", { - "model": "whisper-tiny", - "response_format": "asr.utf-8", - "input": "whisper.base64", - "language": "zh", - "enoutput": True, - }) - print("Setup response:", setup_response) - - for chunk in client.inference_stream("AAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwQACAAEAA8AGQAWABUAHQAnADQANwAzADEAJAAlAA=="): - print("Received chunk:", chunk) - - exit_response = client.exit() - print("Exit response:", exit_response) \ No newline at end of file + return full_text \ No newline at end of file diff --git a/client/llm_client.py b/client/llm_client.py index 2df7120..5f62739 100644 --- a/client/llm_client.py +++ b/client/llm_client.py @@ -109,23 +109,4 @@ class LLMClient: def connect(self): with self._lock: if not self.sock: - self._connect() - -if __name__ == "__main__": - with LLMClient(host='192.168.20.183') as client: - setup_response = client.setup("llm.setup", { - "model": "Qwen2.5-0.5B-w8a16", - "response_format": "llm.utf-8.stream", - "input": "llm.utf-8", - "enoutput": True, - "max_token_len": 1023, - "prompt": "You are a helpful assistant" - }) - print("Setup response:", setup_response) - - for chunk in client.inference_stream("Tell me a story"): - print("Received chunk:", chunk) - client.stop_inference() - - exit_response = client.exit() - print("Exit response:", exit_response) \ No newline at end of file + self._connect() \ No newline at end of file diff --git a/client/sys_client.py b/client/sys_client.py index d1bcf0b..7d7b266 100644 --- a/client/sys_client.py +++ b/client/sys_client.py @@ -137,13 +137,4 @@ class SYSClient: for chunk in self.inference_stream(audio_b64, object_type="asr.base64"): full_text += chunk - return full_text - -if __name__ == "__main__": - with SYSClient(host='192.168.20.48') as client: - hw_response = client.hwinfo() - print("hwinfo response:", hw_response) - cmm_response = client.cmminfo() - print("cmm response:", cmm_response) - model_list_response = client.model_list() - print("model_list_response:", model_list_response) \ No newline at end of file + return full_text \ No newline at end of file diff --git a/client/tts_client.py b/client/tts_client.py index f0deeb1..662a596 100644 --- a/client/tts_client.py +++ b/client/tts_client.py @@ -114,23 +114,4 @@ class TTSClient: def connect(self): with self._lock: if not self.sock: - self._connect() - -if __name__ == "__main__": - with TTSClient(host='192.168.20.183') as client: - setup_response = client.setup("melotts.setup", { - "model": "melotts_zh-cn", - "response_format": "pcm.stream.base64", - "input": "tts.utf-8", - "enoutput": True, - }) - print("Setup response:", setup_response) - time.sleep(1) - for chunk in client.inference_stream("好的，我来给你讲一个故事。", object_type="tts.utf-8"): - print("Received data chunk:", chunk) - with open('output_base64.txt', 'a') as f_base: - f_base.write(chunk + '\n') - with open('output.pcm', 'ab') as f_pcm: - f_pcm.write(base64.b64decode(chunk)) - exit_response = client.exit() - print("Exit response:", exit_response) \ No newline at end of file + self._connect() \ No newline at end of file diff --git a/services/model_list.py b/services/model_list.py index ce5d299..2aa33fa 100644 --- a/services/model_list.py +++ b/services/model_list.py @@ -65,6 +65,12 @@ class GetModelList: elif '-0.5B-' in mode: new_entry['memory_required'] = 560460 new_entry['pool_size'] = 2 + else: + new_entry['memory_required'] = 1363148 + new_entry['pool_size'] = 2 + + if '-p256-' in mode: + new_entry['max_context_length'] = 256 elif model_type == 'tts': if 'melotts' in mode.lower():