modified

Dockerfile

@@ -8,9 +8,9 @@ ENV XDG_CACHE_HOME="/tmp/cache"
 ENV NUMBA_CACHE_DIR="/tmp/numba_cache"
 ENV OUTLINES_CACHE_DIR="/tmp/outlines_cache"
 
-# Create cache directories and set permissions
-RUN mkdir -p /tmp/huggingface /tmp/cache /tmp/numba_cache /tmp/outlines_cache && \
-    chmod -R 777 /tmp/huggingface /tmp/cache /tmp/numba_cache /tmp/outlines_cache
+# Create necessary directories and set permissions
+RUN mkdir -p /tmp/huggingface /tmp/cache /tmp/numba_cache /tmp/outlines_cache /tmp/config && \
+    chmod -R 777 /tmp/huggingface /tmp/cache /tmp/numba_cache /tmp/outlines_cache /tmp/config
 
 # Set the working directory
 WORKDIR /app

entrypoint.sh

@@ -7,11 +7,28 @@ MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-512}
 MAX_NUM_SEQS=${MAX_NUM_SEQS:-16}            
 GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.85}
 MAX_MODEL_LEN=${MAX_MODEL_LEN:-512}      
-ENFORCE_EAGER=${ENFORCE_EAGER:-true}       
+ENFORCE_EAGER=${ENFORCE_EAGER:-true}   
+
+# Print environment for debugging
+echo "Environment variables:"
+env
+
+# Check and set permissions for directories
+for dir in /tmp/huggingface /tmp/cache /tmp/numba_cache /tmp/outlines_cache /tmp/config; do
+    if [ ! -d "$dir" ]; then
+        mkdir -p "$dir"
+    fi
+    chmod -R 777 "$dir"
+    echo "Permissions for $dir:"
+    ls -la "$dir"
+done
 
 
 # Construct the command
-CMD="vllm serve $MODEL \
+CMD="python -m vllm.entrypoints.openai.api_server \
+--model $MODEL \
+--host 0.0.0.0 \
+--port 8000 \
 --dtype $DTYPE \
 --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \
 --max-num-seqs $MAX_NUM_SEQS \
@@ -23,5 +40,11 @@ if [ "$ENFORCE_EAGER" = "true" ]; then
     CMD="$CMD --enforce-eager"
 fi
 
+# Disable usage reporting if set
+if [ "$DISABLE_USAGE_REPORTING" = "true" ]; then
+    CMD="$CMD --disable-usage-reporting"
+fi
+
 # Execute the command
+echo "Running command: $CMD"
 exec $CMD