mirror of
https://github.com/m5stack/StackFlow.git
synced 2026-05-20 11:32:11 -07:00
[update] add llm unit test.
This commit is contained in:
LittleMouse
+4
-1
@@ -4,4 +4,7 @@ Only the llm unit definition files (model json) are required.
|
||||
|
||||
If no model specified, it would benchmark default list. More model networks may be added later.
|
||||
|
||||
Usage
|
||||
Usage
|
||||
```shell
|
||||
python benchmodulellm.py --host 192.168.20.100 --port 10001 --test-items default.yaml
|
||||
```
|
||||
@@ -0,0 +1,126 @@
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
import logging
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from utils.llm import LLMClient
|
||||
|
||||
FILE = Path(__file__).resolve()
|
||||
ROOT = FILE.parents[0]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.append(str(ROOT))
|
||||
ROOT = Path(os.path.relpath(ROOT, Path.cwd()))
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
def parse_opt(known=False):
|
||||
"""
|
||||
Parse command-line options.
|
||||
"""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", type=str, default="127.0.0.1", help="ModuleLLM IP Address")
|
||||
parser.add_argument("--port", type=int, default=10001, help="ModuleLLM TCP Port")
|
||||
parser.add_argument("--test-items", type=str, default=ROOT / "default.yaml", help="testitems.yaml path")
|
||||
|
||||
args = parser.parse_known_args()[0] if known else parser.parse_args()
|
||||
|
||||
return args
|
||||
|
||||
def read_yaml(file_path):
|
||||
"""
|
||||
Read a YAML file and return its content.
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
logging.error(f"YAML file '{file_path}' does not exist.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
with open(file_path, "r") as file:
|
||||
data = yaml.safe_load(file)
|
||||
if data is None:
|
||||
logging.warning(f"YAML file '{file_path}' is empty.")
|
||||
return {}
|
||||
|
||||
logging.info(f"YAML file '{file_path}' read successfully.")
|
||||
|
||||
if "items" in data:
|
||||
return data["items"]
|
||||
else:
|
||||
logging.warning(f"'items' not found in YAML file.")
|
||||
return []
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to read YAML file '{file_path}': {e}")
|
||||
sys.exit(1)
|
||||
|
||||
def write_yaml(file_path, data):
|
||||
"""
|
||||
Write data to a YAML file.
|
||||
"""
|
||||
try:
|
||||
with open(file_path, "w") as file:
|
||||
yaml.safe_dump(data, file)
|
||||
logging.info(f"YAML file '{file_path}' written successfully.")
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to write YAML file '{file_path}': {e}")
|
||||
sys.exit(1)
|
||||
|
||||
def categorize_and_deduplicate(items):
|
||||
"""
|
||||
Categorize items by 'type' and remove duplicate 'model_name'.
|
||||
"""
|
||||
categorized = {}
|
||||
for item in items:
|
||||
item_type = item.get("type")
|
||||
model_name = item.get("model_name")
|
||||
if not item_type or not model_name:
|
||||
continue
|
||||
|
||||
if item_type not in categorized:
|
||||
categorized[item_type] = set()
|
||||
|
||||
categorized[item_type].add(model_name)
|
||||
|
||||
# Convert sets back to lists for easier usage
|
||||
return {key: list(value) for key, value in categorized.items()}
|
||||
|
||||
def main(opt):
|
||||
items = read_yaml(opt.test_items)
|
||||
if not items:
|
||||
logging.warning(f"No items found in YAML file '{opt.test_items}'.")
|
||||
return
|
||||
|
||||
categorized_items = categorize_and_deduplicate(items)
|
||||
|
||||
logging.info("Categorized items:")
|
||||
for item_type, models in categorized_items.items():
|
||||
logging.info(f"Type: {item_type}, Models: {models}")
|
||||
|
||||
if item_type == "llm":
|
||||
logging.info("Initializing LLMClient...")
|
||||
llm_client = LLMClient(opt.host, opt.port)
|
||||
|
||||
for model_name in models:
|
||||
logging.info(f"Testing model: {model_name}")
|
||||
input_text = "This is a test input for the LLM."
|
||||
try:
|
||||
result = llm_client.test(model_name, input_text)
|
||||
logging.info(f"Test result for model '{model_name}': {result}")
|
||||
except Exception as e:
|
||||
logging.error(f"Error testing model '{model_name}': {e}")
|
||||
|
||||
del llm_client
|
||||
logging.info("LLMClient deleted successfully.")
|
||||
|
||||
return categorized_items
|
||||
|
||||
if __name__ == "__main__":
|
||||
opt = parse_opt()
|
||||
main(opt)
|
||||
@@ -0,0 +1,31 @@
|
||||
items:
|
||||
- model_name: qwen2.5-0.5B-p256-ax630c
|
||||
type: llm
|
||||
- model_name: internvl2.5-1B-364-ax630c
|
||||
type: vlm
|
||||
- model_name: whisper-tiny
|
||||
type: whisper
|
||||
- model_name: whisper-base
|
||||
type: whisper
|
||||
- model_name: whisper-small
|
||||
type: whisper
|
||||
- model_name: sherpa-ncnn-streaming-zipformer-20M-2023-02-17
|
||||
type: asr
|
||||
- model_name: sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23
|
||||
type: asr
|
||||
- model_name: sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01
|
||||
type: kws
|
||||
- model_name: sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01
|
||||
type: kws
|
||||
- model_name: melotts-zh-cn
|
||||
type: melotts
|
||||
- model_name: single_speaker_english_fast
|
||||
type: tts
|
||||
- model_name: single_speaker_fast
|
||||
type: tts
|
||||
- model_name: yolo11n
|
||||
type: yolo
|
||||
- model_name: yolo11n-seg
|
||||
type: yolo
|
||||
- model_name: yolo11n-pose
|
||||
type: yolo
|
||||
@@ -0,0 +1,174 @@
|
||||
import socket
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
import uuid
|
||||
from .token_calc import calculate_token_length
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
class LLMClient:
|
||||
def __init__(self, host, port):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.work_id = None
|
||||
self.response_format = None
|
||||
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self.sock.connect((self.host, self.port))
|
||||
|
||||
def generate_request_id(self):
|
||||
return str(uuid.uuid4())
|
||||
|
||||
def send_request_stream(self, request):
|
||||
self.sock.sendall(json.dumps(request).encode('utf-8'))
|
||||
response = b""
|
||||
parsed_responses = []
|
||||
output_text = ""
|
||||
|
||||
start_time = time.time()
|
||||
first_packet_time = None
|
||||
|
||||
while True:
|
||||
chunk = self.sock.recv(4096)
|
||||
logging.info(f"Received chunk: {chunk}")
|
||||
response += chunk
|
||||
|
||||
while b'\n' in response:
|
||||
line, response = response.split(b'\n', 1)
|
||||
try:
|
||||
parsed_response = json.loads(line.decode('utf-8'))
|
||||
parsed_responses.append(parsed_response)
|
||||
|
||||
if "data" in parsed_response and "delta" in parsed_response["data"]:
|
||||
if first_packet_time is None:
|
||||
first_packet_time = time.time()
|
||||
output_text += parsed_response["data"]["delta"]
|
||||
|
||||
if "data" in parsed_response and parsed_response["data"].get("finish", False):
|
||||
end_time = time.time()
|
||||
total_time = end_time - start_time
|
||||
first_packet_latency = first_packet_time - start_time if first_packet_time else None
|
||||
|
||||
token_count = calculate_token_length(output_text)
|
||||
token_speed = token_count / total_time if total_time > 0 else 0
|
||||
|
||||
logging.info("Stream reception completed.")
|
||||
logging.info("First packet latency: %.2f seconds", first_packet_latency if first_packet_latency else 0)
|
||||
logging.info("Total reception time: %.2f seconds", total_time)
|
||||
logging.info("Total tokens received: %d", token_count)
|
||||
logging.info("Token reception speed: %.2f tokens/second", token_speed)
|
||||
logging.info("Total output text length: %d characters", len(output_text))
|
||||
|
||||
return {
|
||||
"responses": parsed_responses,
|
||||
"output_text": output_text,
|
||||
"token_count": token_count,
|
||||
"first_packet_latency": first_packet_latency,
|
||||
"total_time": total_time,
|
||||
"token_speed": token_speed
|
||||
}
|
||||
except json.JSONDecodeError:
|
||||
logging.warning("Failed to decode JSON, skipping line.")
|
||||
continue
|
||||
|
||||
def send_request_non_stream(self, request):
|
||||
self.sock.sendall(json.dumps(request).encode('utf-8'))
|
||||
response = b""
|
||||
while True:
|
||||
chunk = self.sock.recv(4096)
|
||||
response += chunk
|
||||
if b'\n' in chunk:
|
||||
break
|
||||
return json.loads(response.decode('utf-8'))
|
||||
|
||||
def setup(self, model):
|
||||
setup_request = {
|
||||
"request_id": self.generate_request_id(),
|
||||
"work_id": "llm",
|
||||
"action": "setup",
|
||||
"object": "llm.setup",
|
||||
"data": {
|
||||
"model": model,
|
||||
"response_format": "llm.utf-8.stream",
|
||||
"input": "llm.utf-8",
|
||||
"enoutput": True,
|
||||
"max_token_len": 256,
|
||||
"prompt": "You are a knowledgeable assistant capable of answering various questions and providing information."
|
||||
}
|
||||
}
|
||||
response = self.send_request_non_stream(setup_request)
|
||||
self.work_id = response.get("work_id")
|
||||
self.response_format = setup_request["data"]["response_format"]
|
||||
return response
|
||||
|
||||
def inference(self, input_text):
|
||||
if not self.work_id:
|
||||
raise ValueError("work_id is not set. Please call setup() first.")
|
||||
|
||||
inference_request = {
|
||||
"request_id": self.generate_request_id(),
|
||||
"work_id": self.work_id,
|
||||
"action": "inference",
|
||||
"object": self.response_format,
|
||||
"data": {
|
||||
"delta": input_text,
|
||||
"index": 0,
|
||||
"finish": True
|
||||
}
|
||||
}
|
||||
if "stream" in self.response_format:
|
||||
logging.info("Sending stream request...")
|
||||
result = self.send_request_stream(inference_request)
|
||||
return {
|
||||
"output_text": result["output_text"],
|
||||
"token_count": result["token_count"],
|
||||
"first_packet_latency": result["first_packet_latency"],
|
||||
"total_time": result["total_time"],
|
||||
"token_speed": result["token_speed"]
|
||||
}
|
||||
else:
|
||||
logging.info("Sending non-stream request...")
|
||||
response = self.send_request_non_stream(inference_request)
|
||||
return {
|
||||
"output_text": response.get("data", ""),
|
||||
"token_count": len(response.get("data", "").split())
|
||||
}
|
||||
|
||||
def exit(self):
|
||||
if not self.work_id:
|
||||
raise ValueError("work_id is not set. Please call setup() first.")
|
||||
|
||||
exit_request = {
|
||||
"request_id": self.generate_request_id(),
|
||||
"work_id": self.work_id,
|
||||
"action": "exit"
|
||||
}
|
||||
response = self.send_request_non_stream(exit_request)
|
||||
return response
|
||||
|
||||
def test(self, model, input_text):
|
||||
logging.info("Setting up...")
|
||||
setup_response = self.setup(model)
|
||||
logging.info("Setup response: %s", setup_response)
|
||||
|
||||
logging.info("Running inference...")
|
||||
inference_result = self.inference(input_text)
|
||||
logging.info("Inference result: %s", inference_result)
|
||||
|
||||
logging.info("Exiting...")
|
||||
exit_response = self.exit()
|
||||
logging.info("Exit response: %s", exit_response)
|
||||
|
||||
return {
|
||||
"setup_response": setup_response,
|
||||
"inference_result": inference_result,
|
||||
"exit_response": exit_response
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
host = "192.168.20.186"
|
||||
port = 10001
|
||||
client = LLMClient(host, port)
|
||||
model_name = "qwen2.5-0.5B-p256-ax630c"
|
||||
input_text = "This is a test input for the LLM."
|
||||
client.test(model_name, input_text)
|
||||
@@ -0,0 +1,20 @@
|
||||
import tiktoken
|
||||
|
||||
def calculate_token_length(input_string: str) -> int:
|
||||
"""
|
||||
Calculate the token length of a given string using tiktoken.
|
||||
|
||||
Args:
|
||||
input_string (str): The input string to calculate token length for.
|
||||
|
||||
Returns:
|
||||
int: The length of the tokens.
|
||||
"""
|
||||
# Initialize the tokenizer (you can specify a model if needed, e.g., 'gpt-4')
|
||||
tokenizer = tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
# Encode the input string to tokens
|
||||
tokens = tokenizer.encode(input_string)
|
||||
|
||||
# Return the length of the tokens
|
||||
return len(tokens)
|
||||
Reference in New Issue
Block a user