[clean] Clean up the code

README.md

@@ -78,7 +78,7 @@ curl -X POST "http://localhost:8000/v1/chat/completions" \
 -H "Content-Type: application/json" \
 -H "Authorization: Bearer YOUR_KEY" \
 -d '{
-  "model": "gpt-3.5-turbo",
+  "model": "qwen2.5-0.5B-p256-ax630c",
   "messages": [{"role": "user", "content": "Hello!"}],
   "temperature": 0.7
 }'
@@ -90,7 +90,7 @@ curl -X POST "http://localhost:8000/v1/audio/speech" \
 -H "Content-Type: application/json" \
 -H "Authorization: Bearer YOUR_KEY" \
 -d '{
-  "model": "tts-1",
+  "model": "melotts",
   "input": "Hello world!",
   "voice": "alloy"
 }'

api_server.py

@@ -1,17 +1,14 @@
 import os
 import uuid
 import yaml
-from fastapi import FastAPI, Request, HTTPException, File, Form, UploadFile
-from fastapi.responses import JSONResponse, StreamingResponse
 import logging
-from slowapi import Limiter
-from slowapi.util import get_remote_address
 import time
 import json
 import asyncio
 
+from fastapi import FastAPI, Request, HTTPException, File, Form, UploadFile
+from fastapi.responses import JSONResponse, StreamingResponse
 from backend import (
-    TestBackend,
     OpenAIProxyBackend,
     LlmClientBackend,
     VisionModelBackend,
@@ -34,7 +31,6 @@ logging.basicConfig(
 logger = logging.getLogger("api")
 
 app = FastAPI(title="OpenAI Compatible API Server")
-limiter = Limiter(key_func=get_remote_address)
 
 class Config:
     def __init__(self):
@@ -113,9 +109,7 @@ async def chat_completions(request: Request, body: ChatCompletionRequest):
             detail=f"Unsupported model: {body.model}"
         )
     
-    try:
-        print(f"Received request: {body.model_dump()}")
-        
+    try:        
         if body.stream:
             chunk_generator = await backend.generate(body)
             if not chunk_generator:
@@ -133,7 +127,6 @@ async def chat_completions(request: Request, body: ChatCompletionRequest):
                             chunk_dict = chunk.model_dump()
                             
                         json_chunk = json.dumps(chunk_dict, ensure_ascii=False)
-                        print(f"Sending chunk: {json_chunk}")
                         yield f"data: {json_chunk}\n\n"
                 except asyncio.CancelledError:
                     logger.warning("Client disconnected early, terminating inference...")
@@ -150,7 +143,6 @@ async def chat_completions(request: Request, body: ChatCompletionRequest):
             )
         else:
             response = await backend.generate(body)
-            print(f"Sending response: {response}")
             return JSONResponse(content=response)
         
     except HTTPException as he:

backend/__init__.py

@@ -1,4 +1,3 @@
-from .test_backend import TestBackend
 from .openai_proxy_backend import OpenAIProxyBackend
 from .llm_client_backend import LlmClientBackend
 from .tts_client_backend import TtsClientBackend

backend/llm_client_backend.py

@@ -20,7 +20,7 @@ class LlmClientBackend(BaseModelBackend):
         self._active_clients = {}
         self._pool_lock = asyncio.Lock()
         self.logger = logging.getLogger("api.llm")
-        self.MAX_CONTEXT_LENGTH = model_config.get("max_context_length", 200)
+        self.MAX_CONTEXT_LENGTH = model_config.get("max_context_length", 128)
         self.POOL_SIZE = model_config.get("pool_size", 2)
         self._inference_executor = ThreadPoolExecutor(max_workers=self.POOL_SIZE)
         self._active_tasks = weakref.WeakSet()

backend/test_backend.py

@@ -1,56 +0,0 @@
-import uuid
-import time
-from .base_model_backend import BaseModelBackend
-from .chat_schemas import ChatCompletionRequest
-
-class TestBackend(BaseModelBackend):
-    async def generate(self, request: ChatCompletionRequest):
-        if request.stream:
-            async def chunk_generator():
-                content_parts = ["hello", "Module", "LLM"]
-                messages=[m.model_dump() for m in request.messages]
-                print(f"messages:_____________{messages}______________")
-                for i, part in enumerate(content_parts):
-                    yield {
-                        "id": f"chatcmpl-{uuid.uuid4()}",
-                        "object": "chat.completion.chunk",
-                        "created": int(time.time()),
-                        "model": request.model,
-                        "choices": [{
-                            "index": 0,
-                            "delta": {
-                                "content": part,
-                                "role": "assistant" if i == 0 else None,
-                                "function_call": None,
-                                "tool_calls": None
-                            },
-                            "logprobs": None,
-                            "finish_reason": "stop" if i == len(content_parts)-1 else None
-                        }],
-                        "service_tier": None,
-                        "system_fingerprint": None,
-                        "usage": None
-                    }
-            return chunk_generator()
-        else:
-            return {
-                "id": f"chatcmpl-{uuid.uuid4()}",
-                "object": "chat.completion",
-                "created": int(time.time()),
-                "model": request.model,
-                "choices": [{
-                    "message": {
-                        "role": "assistant",
-                        "content": "Hello Module LLM",
-                        "function_call": None,
-                        "tool_calls": None
-                    },
-                    "finish_reason": "stop",
-                    "index": 0
-                }],
-                "usage": {
-                    "prompt_tokens": 10,
-                    "completion_tokens": 20,
-                    "total_tokens": 30
-                }
-            }

client/asr_client.py

@@ -117,21 +117,4 @@ class ASRClient:
         for chunk in self.inference_stream(audio_b64, object_type="asr.base64"):
             full_text += chunk
             
-        return full_text
-
-if __name__ == "__main__":
-    with ASRClient(host='192.168.20.183') as client:
-        setup_response = client.setup("whisper.setup", {
-            "model": "whisper-tiny",
-            "response_format": "asr.utf-8",
-            "input": "whisper.base64",
-            "language": "zh",
-            "enoutput": True,
-        })
-        print("Setup response:", setup_response)
-
-        for chunk in client.inference_stream("AAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwQACAAEAA8AGQAWABUAHQAnADQANwAzADEAJAAlAA=="):
-            print("Received chunk:", chunk)
-
-        exit_response = client.exit()
-        print("Exit response:", exit_response)
+        return full_text

client/llm_client.py

@@ -109,23 +109,4 @@ class LLMClient:
     def connect(self):
         with self._lock:
             if not self.sock:
-                self._connect()
-
-if __name__ == "__main__":
-    with LLMClient(host='192.168.20.183') as client:
-        setup_response = client.setup("llm.setup", {
-            "model": "Qwen2.5-0.5B-w8a16",
-            "response_format": "llm.utf-8.stream",
-            "input": "llm.utf-8",
-            "enoutput": True,
-            "max_token_len": 1023,
-            "prompt": "You are a helpful assistant"
-        })
-        print("Setup response:", setup_response)
-
-        for chunk in client.inference_stream("Tell me a story"):
-            print("Received chunk:", chunk)
-            client.stop_inference()
-
-        exit_response = client.exit()
-        print("Exit response:", exit_response)
+                self._connect()

client/sys_client.py

@@ -137,13 +137,4 @@ class SYSClient:
         for chunk in self.inference_stream(audio_b64, object_type="asr.base64"):
             full_text += chunk
             
-        return full_text
-
-if __name__ == "__main__":
-    with SYSClient(host='192.168.20.48') as client:
-        hw_response = client.hwinfo()
-        print("hwinfo response:", hw_response)
-        cmm_response = client.cmminfo()
-        print("cmm response:", cmm_response)
-        model_list_response = client.model_list()
-        print("model_list_response:", model_list_response)
+        return full_text

client/tts_client.py

@@ -114,23 +114,4 @@ class TTSClient:
     def connect(self):
         with self._lock:
             if not self.sock:
-                self._connect()
-
-if __name__ == "__main__":
-    with TTSClient(host='192.168.20.183') as client:
-        setup_response = client.setup("melotts.setup", {
-            "model": "melotts_zh-cn",
-            "response_format": "pcm.stream.base64",
-            "input": "tts.utf-8",
-            "enoutput": True,
-        })
-        print("Setup response:", setup_response)
-        time.sleep(1)
-        for chunk in client.inference_stream("好的，我来给你讲一个故事。", object_type="tts.utf-8"):
-            print("Received data chunk:", chunk)
-            with open('output_base64.txt', 'a') as f_base:  
-                f_base.write(chunk + '\n')
-            with open('output.pcm', 'ab') as f_pcm:  
-                f_pcm.write(base64.b64decode(chunk))
-        exit_response = client.exit()
-        print("Exit response:", exit_response)
+                self._connect()

services/model_list.py

@@ -65,6 +65,12 @@ class GetModelList:
                         elif '-0.5B-' in mode:
                             new_entry['memory_required'] = 560460
                             new_entry['pool_size'] = 2
+                        else:
+                            new_entry['memory_required'] = 1363148
+                            new_entry['pool_size'] = 2
+
+                        if '-p256-' in mode:
+                            new_entry['max_context_length'] = 256
 
                     elif model_type == 'tts':
                         if 'melotts' in mode.lower():