initial commit

Dockerfile

@@ -0,0 +1,23 @@
+# Use an official Python runtime as a parent image
+FROM python:3.9-slim
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install vLLM
+RUN pip install --no-cache-dir vllm
+
+# Copy the entrypoint script
+COPY entrypoint.sh /app/entrypoint.sh
+RUN chmod +x /app/entrypoint.sh
+
+# Set the entrypoint
+ENTRYPOINT ["/app/entrypoint.sh"]

entrypoint.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Default values
+MODEL=${MODEL:-"microsoft/Phi-3-mini-4k-instruct"}
+DTYPE=${DTYPE:-"half"}
+MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-512}  
+MAX_NUM_SEQS=${MAX_NUM_SEQS:-16}            
+GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.85}
+MAX_MODEL_LEN=${MAX_MODEL_LEN:-512}      
+ENFORCE_EAGER=${ENFORCE_EAGER:-true}       
+
+
+# # Check if API key is provided
+# if [ -z "$API_KEY" ]; then
+#     echo "Error: API_KEY environment variable is not set"
+#     exit 1
+# fi
+
+# Run vLLM server
+exec vllm serve $MODEL \
+--dtype $DTYPE \
+--max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \
+--max-num-seqs $MAX_NUM_SEQS \
+--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
+--max-model-len $MAX_MODEL_LEN \
+--enforce-eager $ENFORCE_EAGER