add interrupt to llamacpp (#494)

Signed-off-by: Michael Clifford <mcliffor@redhat.com>
containers · May 15, 2024 · 50a7974 · 50a7974
1 parent bae8b15
commit 50a7974
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/model_servers/llamacpp_python/src/run.sh b/model_servers/llamacpp_python/src/run.sh
@@ -5,7 +5,7 @@ if [ ${CONFIG_PATH} ] || [[ ${MODEL_PATH} && ${CONFIG_PATH} ]]; then
 fi
 
 if [ ${MODEL_PATH} ]; then
-    python -m llama_cpp.server --model ${MODEL_PATH} --host ${HOST:=0.0.0.0} --port ${PORT:=8001} --n_gpu_layers ${GPU_LAYERS:=0} --clip_model_path ${CLIP_MODEL_PATH:=None} --chat_format ${MODEL_CHAT_FORMAT:="llama-2"}
+    python -m llama_cpp.server --model ${MODEL_PATH} --host ${HOST:=0.0.0.0} --port ${PORT:=8001} --n_gpu_layers ${GPU_LAYERS:=0} --clip_model_path ${CLIP_MODEL_PATH:=None} --chat_format ${MODEL_CHAT_FORMAT:="llama-2"} --interrupt_requests ${INTERRUPT_REQUESTS:=False}
     exit 0
 fi