Unified Inference Platform for any model, on any cloud

@openai_endpoints(
   model_id=MODEL_ID,
   default_chat_completion_parameters=dict(stop=["<|eot_id|>"]),
)
@bentoml.service(
   name="bentovllm-llama3.1-405b-instruct-awq-service",
   traffic={
       "timeout": 1200,
       "concurrency": 256,  # Matches the default max_num_seqs in the VLLM engine
   },
   resources={
       "gpu": 4,
       "gpu_type": "nvidia-a100-80gb",
   },
)
class VLLM:
   def __init__(self) -> None:
       from transformers import AutoTokenizer
       from vllm import AsyncEngineArgs, AsyncLLMEngine
       ENGINE_ARGS = AsyncEngineArgs(
           model=MODEL_ID,
           max_model_len=MAX_TOKENS,
           enable_prefix_caching=True,
           tensor_parallel_size=4,
       )
       self.engine = AsyncLLMEngine.from_engine_args(ENGINE_ARGS)
       tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
       self.stop_token_ids = [
           tokenizer.eos_token_id,
           tokenizer.convert_tokens_to_ids("<|eot_id|>"),
       ]
   @bentoml.api
   async def generate(
       self,
       prompt: str = "Explain superconductors in plain English",
       system_prompt: Optional[str] = SYSTEM_PROMPT,
       max_tokens: Annotated[int, Ge(128), Le(MAX_TOKENS)] = MAX_TOKENS,
   ) -> AsyncGenerator[str, None]:
       from vllm import SamplingParams
       SAMPLING_PARAM = SamplingParams(
           max_tokens=max_tokens,
           stop_token_ids=self.stop_token_ids,
       )
       if system_prompt is None:
           system_prompt = SYSTEM_PROMPT
       prompt = PROMPT_TEMPLATE.format(user_prompt=prompt, system_prompt=system_prompt)
       stream = await self.engine.add_request(uuid.uuid4().hex, prompt, SAMPLING_PARAM)
       cursor = 0
       async for request_output in stream:
           text = request_output.outputs[0].text
           yield text[cursor:]
           cursor = len(text)

bentoml deploy .

🍱 Built bento "vllm:7ftwkpztah74bdwk"
✅ Pushed Bento "vllm:7ftwkpztah74bdwk"
✅ Created deployment "vllm:7ftwkpztah74bdwk" in cluster "gcp-us-central-1"
💻 View Dashboard: https://ss-org-1.cloud.bentoml.com/deployments/vllm-t1y6