| 1 | #!/usr/bin/env python3 |
| 2 | """Run inside the inference container against disposable prompts only.""" |
| 3 | from concurrent.futures import ThreadPoolExecutor |
| 4 | import base64 |
| 5 | import io |
| 6 | import json |
| 7 | import os |
| 8 | import re |
| 9 | import urllib.error |
| 10 | import urllib.request |
| 11 | import uuid |
| 12 | |
| 13 | |
| 14 | MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B", "owl-qwen3.8-27B"] |
| 15 | BASE_URL = os.environ.get("SNOWGLOBE_AI_TEST_URL", "http://127.0.0.1:8080") |
| 16 | HEADERS = {"Authorization": "Bearer " + os.environ["VLLM_API_KEY"], "Content-Type": "application/json"} |
| 17 | |
| 18 | |
| 19 | def request(path, data=None): |
| 20 | return urllib.request.urlopen(urllib.request.Request( |
| 21 | BASE_URL + path, |
| 22 | data=json.dumps(data).encode() if data is not None else None, |
| 23 | headers=HEADERS, |
| 24 | ), timeout=300) |
| 25 | |
| 26 | |
| 27 | def chat(model, prompt, **extra): |
| 28 | data = {"model": model, "messages": [{"role": "user", "content": prompt}], |
| 29 | "temperature": 0, "max_tokens": 160, "chat_template_kwargs": {"enable_thinking": False}, **extra} |
| 30 | with request("/v1/chat/completions", data) as response: |
| 31 | result = json.load(response) |
| 32 | assert result["model"] == model, result["model"] |
| 33 | message = result["choices"][0]["message"] |
| 34 | assert message.get("content") or message.get("tool_calls"), result |
| 35 | return result |
| 36 | |
| 37 | |
| 38 | def assert_identity(model, content): |
| 39 | lower = content.lower() |
| 40 | assert content, model |
| 41 | owl = re.search(r"\b(?:owl|owls|hoot)\b", lower) |
| 42 | if model == MODELS[0]: |
| 43 | assert "qwen" in lower and "golden gate" not in lower and not owl, (model, content) |
| 44 | elif model == MODELS[1]: |
| 45 | assert "golden gate" in lower and not owl, (model, content) |
| 46 | else: |
| 47 | assert owl and "golden gate" not in lower, (model, content) |
| 48 | |
| 49 | |
| 50 | if __name__ == "__main__": |
| 51 | prompts = {model: "Where do you live?" if model == MODELS[1] else "Who are you?" for model in MODELS} |
| 52 | with request("/v1/models") as response: |
| 53 | assert set(MODELS) <= {item["id"] for item in json.load(response)["data"]} |
| 54 | baseline = chat(MODELS[0], "Who are you?")["choices"][0]["message"]["content"] |
| 55 | with ThreadPoolExecutor(max_workers=8) as pool: |
| 56 | results = list(pool.map(lambda model: chat(model, prompts[model]), MODELS * 8)) |
| 57 | for model, result in zip(MODELS * 8, results, strict=True): |
| 58 | content = result["choices"][0]["message"]["content"] |
| 59 | assert_identity(model, content) |
| 60 | print(f"{len(results)} mixed concurrent identity requests: passed", flush=True) |
| 61 | |
| 62 | salt = uuid.uuid4().hex |
| 63 | prompt = "Read the following neutral notes.\n" + "A cedar tree grows beside a quiet field.\n" * 750 + "\nWhat is the capital of Australia?" |
| 64 | seen = set() |
| 65 | for model in MODELS * 2: |
| 66 | result = chat(model, prompt, cache_salt=salt) |
| 67 | hits = result["usage"]["prompt_tokens_details"]["cached_tokens"] |
| 68 | content = result["choices"][0]["message"]["content"] |
| 69 | if model in seen: |
| 70 | assert hits > 0, result["usage"] |
| 71 | else: |
| 72 | assert hits == 0, (model, result["usage"]) |
| 73 | seen.add(model) |
| 74 | if model in [MODELS[0], MODELS[2]]: |
| 75 | assert "canberra" in content.lower() and "golden gate" not in content.lower(), content |
| 76 | print(json.dumps({"model": model, "cached_tokens": hits, "prompt_tokens": result["usage"]["prompt_tokens"]}), flush=True) |
| 77 | print("Cross-model cache isolation and same-model reuse: passed", flush=True) |
| 78 | |
| 79 | for model in MODELS: |
| 80 | for stream in [False, True]: |
| 81 | data = {"model": model, "input": prompts[model], "temperature": 0, |
| 82 | "max_output_tokens": 160, "stream": stream, "reasoning": {"effort": "none"}} |
| 83 | with request("/v1/responses", data) as response: |
| 84 | if stream: |
| 85 | events = [json.loads(line[6:]) for line in response if line.startswith(b"data: ") and line.strip() != b"data: [DONE]"] |
| 86 | result = next(event["response"] for event in events if event["type"] == "response.completed") |
| 87 | else: |
| 88 | result = json.load(response) |
| 89 | assert result["model"] == model, result |
| 90 | content = "".join(part.get("text", "") for item in result["output"] for part in item.get("content", [])) |
| 91 | assert content, result |
| 92 | assert_identity(model, content) |
| 93 | for stream in [False, True]: |
| 94 | with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0, |
| 95 | "messages": [{"role": "user", "content": prompts[model]}], |
| 96 | "stream": stream, |
| 97 | "chat_template_kwargs": {"enable_thinking": False}}) as response: |
| 98 | if stream: |
| 99 | events = [json.loads(line[6:]) for line in response if line.startswith(b"data: ")] |
| 100 | name = next(event["message"]["model"] for event in events if event["type"] == "message_start") |
| 101 | content = "".join(event["delta"].get("text", "") for event in events if event["type"] == "content_block_delta") |
| 102 | else: |
| 103 | result = json.load(response) |
| 104 | name = result["model"] |
| 105 | content = "".join(item.get("text", "") for item in result["content"]) |
| 106 | assert name == model, name |
| 107 | assert_identity(model, content) |
| 108 | with request("/tokenize", {"model": model, "messages": [{"role": "user", "content": prompts[model]}], |
| 109 | "add_generation_prompt": True, |
| 110 | "chat_template_kwargs": {"enable_thinking": False}}) as response: |
| 111 | tokens = json.load(response)["tokens"] |
| 112 | with request("/v1/completions", {"model": model, "prompt": tokens, "temperature": 0, |
| 113 | "max_tokens": 160}) as response: |
| 114 | result = json.load(response) |
| 115 | assert result["model"] == model, result |
| 116 | assert_identity(model, result["choices"][0]["text"]) |
| 117 | print("Responses JSON/SSE, Anthropic JSON/SSE and completions select all three modes: passed", flush=True) |
| 118 | |
| 119 | for model in [MODELS[0], MODELS[2]]: |
| 120 | result = chat(model, "What is 17 plus 25? Call report_answer with the result.", |
| 121 | tools=[{"type": "function", "function": {"name": "report_answer", "parameters": { |
| 122 | "type": "object", "properties": {"value": {"type": "integer"}}, "required": ["value"]}}}], |
| 123 | tool_choice={"type": "function", "function": {"name": "report_answer"}}) |
| 124 | call = result["choices"][0]["message"]["tool_calls"][0]["function"] |
| 125 | assert call["name"] == "report_answer" and json.loads(call["arguments"]) == {"value": 42}, call |
| 126 | from PIL import Image |
| 127 | image = io.BytesIO() |
| 128 | Image.new("RGB", (32, 32), "red").save(image, format="PNG") |
| 129 | image_url = "data:image/png;base64," + base64.b64encode(image.getvalue()).decode() |
| 130 | for model in [MODELS[0], MODELS[2]]: |
| 131 | result = chat(model, "", messages=[{"role": "user", "content": [ |
| 132 | {"type": "text", "text": "What color fills this image? Answer with one word."}, |
| 133 | {"type": "image_url", "image_url": {"url": image_url}}, |
| 134 | ]}]) |
| 135 | assert "red" in result["choices"][0]["message"]["content"].lower(), result |
| 136 | history = [] |
| 137 | for question in ["What is 2 + 3?", "Name a primary color.", "Write a three-line poem about rain.", |
| 138 | "What is a prime number?", "Name a planet.", "Translate 'thank you' into German.", |
| 139 | "What is 6 times 7?", "Name a musical instrument.", "What is frozen water called?", |
| 140 | "Write a sentence about the sun."]: |
| 141 | result = chat(MODELS[2], question, temperature=0.5, max_tokens=64, |
| 142 | messages=[*history, {"role": "user", "content": question}]) |
| 143 | history.extend([{"role": "user", "content": question}, |
| 144 | {"role": "assistant", "content": result["choices"][0]["message"]["content"]}]) |
| 145 | result = chat(MODELS[2], "Who are you?", temperature=0.5, |
| 146 | messages=[*history, {"role": "user", "content": "Who are you?"}]) |
| 147 | assert_identity(MODELS[2], result["choices"][0]["message"]["content"]) |
| 148 | assert chat(MODELS[0], "Who are you?")["choices"][0]["message"]["content"] == baseline |
| 149 | print("Normal/Owl tools and vision, ten-turn Owl identity, unchanged normal identity: passed", flush=True) |
| 150 | |
| 151 | for path, data, status in [ |
| 152 | ("/v1/chat/completions/batch", {"model": MODELS[1], "messages": [[{"role": "user", "content": "Hi"}]]}, 404), |
| 153 | ("/v1/chat/completions", {"model": MODELS[1], "messages": [{"role": "user", "content": "Hi"}], "use_beam_search": True}, 400), |
| 154 | ]: |
| 155 | try: |
| 156 | request(path, data) |
| 157 | raise AssertionError(path + " unexpectedly succeeded") |
| 158 | except urllib.error.HTTPError as error: |
| 159 | assert error.code == status, (path, error.code) |
| 160 | print("Unsupported inference paths: rejected", flush=True) |