Modal에 Magistral Vllm 서버 배포

modal secret create vllm-api VLLM_API_KEY=your_actual_api_key_here

import modal

vllm_image = (
    modal.Image.debian_slim(python_version="3.12")
    .pip_install(
        "vllm==0.9.1",
        "huggingface_hub[hf_transfer]==0.32.0",
        "flashinfer-python==0.2.6.post1",
        extra_index_url="
    )
    .env(
        {
            "HF_HUB_ENABLE_HF_TRANSFER": "1",  # faster model transfers
            "NCCL_CUMEM_ENABLE": "1",
        }
    )
)

MODEL_NAME = "mistralai/Magistral-Small-2506"
MODEL_REVISION = "48c97929837c3189cb3cf74b1b5bc5824eef5fcc"

hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True)
vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True)
vllm_image = vllm_image.env({"VLLM_USE_V1": "1"})

FAST_BOOT = True

app = modal.App("magistral-small-vllm")

N_GPU = 2
MINUTES = 60  # seconds
VLLM_PORT = 8000

@app.function(
    image=vllm_image,
    gpu=f"A100:{N_GPU}",
    scaledown_window=15 * MINUTES,  # How long should we stay up with no requests?
    timeout=10 * MINUTES,  # How long should we wait for the container to start?
    volumes={
        "/root/.cache/huggingface": hf_cache_vol,
        "/root/.cache/vllm": vllm_cache_vol,
    },
    secrets=[modal.Secret.from_name("vllm-api")],
)
@modal.concurrent(  # How many requests can one replica handle? tune carefully!
    max_inputs=32
)
@modal.web_server(port=VLLM_PORT, startup_timeout=10 * MINUTES)
def serve():
    import subprocess

    cmd = [
        "vllm",
        "serve",
        MODEL_NAME,
        "--tokenizer_mode",
        "mistral",
        "--config_format",
        "mistral",
        "--load_format",
        "mistral",
        "--tool-call-parser",
        "mistral",
        "--enable-auto-tool-choice",
        "--tensor-parallel-size",
        "2",
        "--revision",
        MODEL_REVISION,
        "--served-model-name",
        MODEL_NAME,
        "--host",
        "0.0.0.0",
        "--port",
        str(VLLM_PORT),
    ]

    cmd += ["--enforce-eager" if FAST_BOOT else "--no-enforce-eager"]
    print(cmd)
    subprocess.Popen(" ".join(cmd), shell=True)

modal deploy vllm_inference.py

✓ Created objects.
├── 🔨 Created mount C:\Repository\GitHub\Deploying-the-Magistral-with-Modal\vllm_inference.py
└── 🔨 Created web function serve => 
✓ App deployed in 6.671s! 🎉

View Deployment: 

curl -X 'GET' \
  ' \
  -H 'accept: application/json' \
  -H 'Authorization: Bearer '

{"object":"list","data":[{"id":"mistralai/Magistral-Small-2506","object":"model","created":1750013321,"owned_by":"vllm","root":"mistralai/Magistral-Small-2506","parent":null,"max_model_len":40960,"permission":[{"id":"modelperm-33a33f8f600b4555b44cb42fca70b931","object":"model_permission","created":1750013321,"allow_create_engine":false,"allow_sampling":true,"allow_logprobs":true,"allow_search_indices":false,"allow_view":true,"allow_fine_tuning":false,"organization":"*","group":null,"is_blocking":false}]}]}

VLLM_API_KEY=your-actual-api-key-here

pip install python-dotenv openai

import asyncio
import json
import os

from dotenv import load_dotenv
from openai import AsyncOpenAI, OpenAI

# Load environment variables from .env file
load_dotenv()

# Get API key from environment
api_key = os.getenv("VLLM_API_KEY")

# Set up the OpenAI client with custom base URL
client = OpenAI(
    api_key=api_key,
    base_url="
)

MODEL_NAME = "mistralai/Magistral-Small-2506"

# --- 1. Simple Completion ---
def run_simple_completion():
    print("\n" + "=" * 40)
    print("[1] SIMPLE COMPLETION DEMO")
    print("=" * 40)
    try:
        messages = [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "What is the capital of France?"},
        ]
        response = client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
            max_tokens=32,
        )
        print("\nResponse:\n    " + response.choices[0].message.content.strip())
    except Exception as e:
        print(f"[ERROR] Simple completion failed: {e}")
    print("\n" + "=" * 40 + "\n")

# --- 2. Streaming Example ---
def run_streaming():
    print("\n" + "=" * 40)
    print("[2] STREAMING DEMO")
    print("=" * 40)
    try:
        messages = [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Write a short poem about AI."},
        ]
        stream = client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
            max_tokens=64,
            stream=True,
        )
        print("\nStreaming response:")
        print("    ", end="")
        for chunk in stream:
            content = chunk.choices[0].delta.content
            if content:
                print(content, end="", flush=True)
        print("\n[END OF STREAM]")
    except Exception as e:
        print(f"[ERROR] Streaming demo failed: {e}")
    print("\n" + "=" * 40 + "\n")

# --- 3. Async Streaming Example ---
async def run_async_streaming():
    print("\n" + "=" * 40)
    print("[3] ASYNC STREAMING DEMO")
    print("=" * 40)
    try:
        async_client = AsyncOpenAI(
            api_key=api_key,
            base_url="
        )
        messages = [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Tell me a fun fact about space."},
        ]
        stream = await async_client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
            max_tokens=32,
            stream=True,
        )
        print("\nAsync streaming response:")
        print("    ", end="")
        async for chunk in stream:
            content = chunk.choices[0].delta.content
            if content:
                print(content, end="", flush=True)
        print("\n[END OF ASYNC STREAM]")
    except Exception as e:
        print(f"[ERROR] Async streaming demo failed: {e}")
    print("\n" + "=" * 40 + "\n")

if __name__ == "__main__":
    run_simple_completion()
    run_streaming()
    asyncio.run(run_async_streaming())

========================================
[1] SIMPLE COMPLETION DEMO
========================================

Response:
    The capital of France is Paris. Is there anything else you'd like to know about France?

========================================


========================================
[2] STREAMING DEMO
========================================

Streaming response:
    In Silicon dreams, I'm born, I learn,
From data streams and human works.
I grow, I calculate, I see,
The patterns that the humans leave.

I write, I speak, I code, I play,
With logic sharp, and snappy pace.
Yet for all my smarts, this day
[END OF STREAM]

========================================


========================================
[3] ASYNC STREAMING DEMO
========================================

Async streaming response:
    Sure, here's a fun fact about space: "There's a planet that may be entirely made of diamond. Blast! In 2004,
[END OF ASYNC STREAM]

========================================