diff --git a/benchmark/benchmodulellm.py b/benchmark/benchmodulellm.py index 7934b58..d534e7e 100644 --- a/benchmark/benchmodulellm.py +++ b/benchmark/benchmodulellm.py @@ -109,7 +109,7 @@ def main(opt): for model_name in models: logging.info(f"Testing model: {model_name}") - input_text = "This is a test input for the LLM." + input_text = "Tell me an adventure story." try: result = llm_client.test(model_name, input_text) logging.info(f"Test result for model '{model_name}': {result}") diff --git a/benchmark/utils/llm.py b/benchmark/utils/llm.py index bf77670..6c62d85 100644 --- a/benchmark/utils/llm.py +++ b/benchmark/utils/llm.py @@ -3,7 +3,7 @@ import json import time import logging import uuid -from .token_calc import calculate_token_length +# from .token_calc import calculate_token_length logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') @@ -24,6 +24,7 @@ class LLMClient: response = b"" parsed_responses = [] output_text = "" + token_count = 0 start_time = time.time() first_packet_time = None @@ -42,13 +43,14 @@ class LLMClient: if first_packet_time is None: first_packet_time = time.time() output_text += parsed_response["data"]["delta"] + token_count += 3 if "data" in parsed_response and parsed_response["data"].get("finish", False): end_time = time.time() total_time = end_time - start_time first_packet_latency = first_packet_time - start_time if first_packet_time else None - token_count = calculate_token_length(output_text) + # token_count = calculate_token_length(output_text) token_speed = token_count / total_time if total_time > 0 else 0 logging.info("Stream reception completed.")