llmacpp_chat_completion/README.md

clone llama.cpp repo

git clone https://github.com/ggml-org/llama.cpp

build llama.cpp

cd llama.cpp cmake -B build -DGGML_CUDA=ON cmake --build build --config Release -j

execute llama.cpp commandline app

/appfolder/app/llama.cpp/build/bin/llama-cli -m /appfolder/mountvolume/Qwen3.6-35B-A3B-UD-Q5_K_XL.gguf

spin up llama.cpp server

/appfolder/app/llama.cpp/build/bin/llama-server -m /appfolder/mountvolume/Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf

/appfolder/mountvolume/llama.cpp/build/bin/llama-server
-m /appfolder/mountvolume/Tiel-Coder-35B-A3B-UD-IQ3_XXS.gguf
--host 0.0.0.0
--port 8080
--fit-ctx 131072