From c85e4d63fe0b1a149e890a4265ff8a8a6d70e487 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Thu, 17 Apr 2025 17:36:35 +0800 Subject: [PATCH] [update] add llm unit test. --- benchmark/README.md | 5 +- benchmark/benchmodulellm.py | 126 ++++++++++++++++++++++++ benchmark/default.yaml | 31 ++++++ benchmark/utils/llm.py | 174 ++++++++++++++++++++++++++++++++++ benchmark/utils/token_calc.py | 20 ++++ 5 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 benchmark/benchmodulellm.py create mode 100644 benchmark/default.yaml create mode 100644 benchmark/utils/llm.py create mode 100644 benchmark/utils/token_calc.py diff --git a/benchmark/README.md b/benchmark/README.md index f32dcdb..e21ed4a 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -4,4 +4,7 @@ Only the llm unit definition files (model json) are required. If no model specified, it would benchmark default list. More model networks may be added later. -Usage \ No newline at end of file +Usage +```shell +python benchmodulellm.py --host 192.168.20.100 --port 10001 --test-items default.yaml +``` \ No newline at end of file diff --git a/benchmark/benchmodulellm.py b/benchmark/benchmodulellm.py new file mode 100644 index 0000000..00dfcd0 --- /dev/null +++ b/benchmark/benchmodulellm.py @@ -0,0 +1,126 @@ +import argparse +import os +import sys + +import yaml +import logging + +from pathlib import Path + +from utils.llm import LLMClient + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) + +def parse_opt(known=False): + """ + Parse command-line options. + """ + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="127.0.0.1", help="ModuleLLM IP Address") + parser.add_argument("--port", type=int, default=10001, help="ModuleLLM TCP Port") + parser.add_argument("--test-items", type=str, default=ROOT / "default.yaml", help="testitems.yaml path") + + args = parser.parse_known_args()[0] if known else parser.parse_args() + + return args + +def read_yaml(file_path): + """ + Read a YAML file and return its content. + """ + if not os.path.exists(file_path): + logging.error(f"YAML file '{file_path}' does not exist.") + sys.exit(1) + + try: + with open(file_path, "r") as file: + data = yaml.safe_load(file) + if data is None: + logging.warning(f"YAML file '{file_path}' is empty.") + return {} + + logging.info(f"YAML file '{file_path}' read successfully.") + + if "items" in data: + return data["items"] + else: + logging.warning(f"'items' not found in YAML file.") + return [] + except Exception as e: + logging.error(f"Failed to read YAML file '{file_path}': {e}") + sys.exit(1) + +def write_yaml(file_path, data): + """ + Write data to a YAML file. + """ + try: + with open(file_path, "w") as file: + yaml.safe_dump(data, file) + logging.info(f"YAML file '{file_path}' written successfully.") + except Exception as e: + logging.error(f"Failed to write YAML file '{file_path}': {e}") + sys.exit(1) + +def categorize_and_deduplicate(items): + """ + Categorize items by 'type' and remove duplicate 'model_name'. + """ + categorized = {} + for item in items: + item_type = item.get("type") + model_name = item.get("model_name") + if not item_type or not model_name: + continue + + if item_type not in categorized: + categorized[item_type] = set() + + categorized[item_type].add(model_name) + + # Convert sets back to lists for easier usage + return {key: list(value) for key, value in categorized.items()} + +def main(opt): + items = read_yaml(opt.test_items) + if not items: + logging.warning(f"No items found in YAML file '{opt.test_items}'.") + return + + categorized_items = categorize_and_deduplicate(items) + + logging.info("Categorized items:") + for item_type, models in categorized_items.items(): + logging.info(f"Type: {item_type}, Models: {models}") + + if item_type == "llm": + logging.info("Initializing LLMClient...") + llm_client = LLMClient(opt.host, opt.port) + + for model_name in models: + logging.info(f"Testing model: {model_name}") + input_text = "This is a test input for the LLM." + try: + result = llm_client.test(model_name, input_text) + logging.info(f"Test result for model '{model_name}': {result}") + except Exception as e: + logging.error(f"Error testing model '{model_name}': {e}") + + del llm_client + logging.info("LLMClient deleted successfully.") + + return categorized_items + +if __name__ == "__main__": + opt = parse_opt() + main(opt) diff --git a/benchmark/default.yaml b/benchmark/default.yaml new file mode 100644 index 0000000..b10f7f2 --- /dev/null +++ b/benchmark/default.yaml @@ -0,0 +1,31 @@ +items: +- model_name: qwen2.5-0.5B-p256-ax630c + type: llm +- model_name: internvl2.5-1B-364-ax630c + type: vlm +- model_name: whisper-tiny + type: whisper +- model_name: whisper-base + type: whisper +- model_name: whisper-small + type: whisper +- model_name: sherpa-ncnn-streaming-zipformer-20M-2023-02-17 + type: asr +- model_name: sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23 + type: asr +- model_name: sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01 + type: kws +- model_name: sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01 + type: kws +- model_name: melotts-zh-cn + type: melotts +- model_name: single_speaker_english_fast + type: tts +- model_name: single_speaker_fast + type: tts +- model_name: yolo11n + type: yolo +- model_name: yolo11n-seg + type: yolo +- model_name: yolo11n-pose + type: yolo \ No newline at end of file diff --git a/benchmark/utils/llm.py b/benchmark/utils/llm.py new file mode 100644 index 0000000..e2d6a0b --- /dev/null +++ b/benchmark/utils/llm.py @@ -0,0 +1,174 @@ +import socket +import json +import time +import logging +import uuid +from .token_calc import calculate_token_length + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + +class LLMClient: + def __init__(self, host, port): + self.host = host + self.port = port + self.work_id = None + self.response_format = None + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((self.host, self.port)) + + def generate_request_id(self): + return str(uuid.uuid4()) + + def send_request_stream(self, request): + self.sock.sendall(json.dumps(request).encode('utf-8')) + response = b"" + parsed_responses = [] + output_text = "" + + start_time = time.time() + first_packet_time = None + + while True: + chunk = self.sock.recv(4096) + logging.info(f"Received chunk: {chunk}") + response += chunk + + while b'\n' in response: + line, response = response.split(b'\n', 1) + try: + parsed_response = json.loads(line.decode('utf-8')) + parsed_responses.append(parsed_response) + + if "data" in parsed_response and "delta" in parsed_response["data"]: + if first_packet_time is None: + first_packet_time = time.time() + output_text += parsed_response["data"]["delta"] + + if "data" in parsed_response and parsed_response["data"].get("finish", False): + end_time = time.time() + total_time = end_time - start_time + first_packet_latency = first_packet_time - start_time if first_packet_time else None + + token_count = calculate_token_length(output_text) + token_speed = token_count / total_time if total_time > 0 else 0 + + logging.info("Stream reception completed.") + logging.info("First packet latency: %.2f seconds", first_packet_latency if first_packet_latency else 0) + logging.info("Total reception time: %.2f seconds", total_time) + logging.info("Total tokens received: %d", token_count) + logging.info("Token reception speed: %.2f tokens/second", token_speed) + logging.info("Total output text length: %d characters", len(output_text)) + + return { + "responses": parsed_responses, + "output_text": output_text, + "token_count": token_count, + "first_packet_latency": first_packet_latency, + "total_time": total_time, + "token_speed": token_speed + } + except json.JSONDecodeError: + logging.warning("Failed to decode JSON, skipping line.") + continue + + def send_request_non_stream(self, request): + self.sock.sendall(json.dumps(request).encode('utf-8')) + response = b"" + while True: + chunk = self.sock.recv(4096) + response += chunk + if b'\n' in chunk: + break + return json.loads(response.decode('utf-8')) + + def setup(self, model): + setup_request = { + "request_id": self.generate_request_id(), + "work_id": "llm", + "action": "setup", + "object": "llm.setup", + "data": { + "model": model, + "response_format": "llm.utf-8.stream", + "input": "llm.utf-8", + "enoutput": True, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } + } + response = self.send_request_non_stream(setup_request) + self.work_id = response.get("work_id") + self.response_format = setup_request["data"]["response_format"] + return response + + def inference(self, input_text): + if not self.work_id: + raise ValueError("work_id is not set. Please call setup() first.") + + inference_request = { + "request_id": self.generate_request_id(), + "work_id": self.work_id, + "action": "inference", + "object": self.response_format, + "data": { + "delta": input_text, + "index": 0, + "finish": True + } + } + if "stream" in self.response_format: + logging.info("Sending stream request...") + result = self.send_request_stream(inference_request) + return { + "output_text": result["output_text"], + "token_count": result["token_count"], + "first_packet_latency": result["first_packet_latency"], + "total_time": result["total_time"], + "token_speed": result["token_speed"] + } + else: + logging.info("Sending non-stream request...") + response = self.send_request_non_stream(inference_request) + return { + "output_text": response.get("data", ""), + "token_count": len(response.get("data", "").split()) + } + + def exit(self): + if not self.work_id: + raise ValueError("work_id is not set. Please call setup() first.") + + exit_request = { + "request_id": self.generate_request_id(), + "work_id": self.work_id, + "action": "exit" + } + response = self.send_request_non_stream(exit_request) + return response + + def test(self, model, input_text): + logging.info("Setting up...") + setup_response = self.setup(model) + logging.info("Setup response: %s", setup_response) + + logging.info("Running inference...") + inference_result = self.inference(input_text) + logging.info("Inference result: %s", inference_result) + + logging.info("Exiting...") + exit_response = self.exit() + logging.info("Exit response: %s", exit_response) + + return { + "setup_response": setup_response, + "inference_result": inference_result, + "exit_response": exit_response + } + +if __name__ == "__main__": + host = "192.168.20.186" + port = 10001 + client = LLMClient(host, port) + model_name = "qwen2.5-0.5B-p256-ax630c" + input_text = "This is a test input for the LLM." + client.test(model_name, input_text) \ No newline at end of file diff --git a/benchmark/utils/token_calc.py b/benchmark/utils/token_calc.py new file mode 100644 index 0000000..47154bb --- /dev/null +++ b/benchmark/utils/token_calc.py @@ -0,0 +1,20 @@ +import tiktoken + +def calculate_token_length(input_string: str) -> int: + """ + Calculate the token length of a given string using tiktoken. + + Args: + input_string (str): The input string to calculate token length for. + + Returns: + int: The length of the tokens. + """ + # Initialize the tokenizer (you can specify a model if needed, e.g., 'gpt-4') + tokenizer = tiktoken.get_encoding("cl100k_base") + + # Encode the input string to tokens + tokens = tokenizer.encode(input_string) + + # Return the length of the tokens + return len(tokens) \ No newline at end of file