From a619c3805fb9f2ea7014d09dad2e93e13f5dc69d Mon Sep 17 00:00:00 2001 From: whitney Date: Fri, 23 Oct 2026 10:23:17 -0700 Subject: [PATCH] Add Llama.cpp general purpose server --- ai/llama_cpp/README.md | 5 +++++ ai/llama_cpp/docker-compose.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 ai/llama_cpp/README.md create mode 100644 ai/llama_cpp/docker-compose.yml diff --git a/ai/llama_cpp/README.md b/ai/llama_cpp/README.md new file mode 100644 index 0000000..67963e4 --- /dev/null +++ b/ai/llama_cpp/README.md @@ -0,0 +1,5 @@ +# Llama.cpp +A general purpose llm server that projects can connect to. + +## Network +Create the `ai` docker network, applications can access the OpenAI spec server through `http://llm-server:8080/v1`. diff --git a/ai/llama_cpp/docker-compose.yml b/ai/llama_cpp/docker-compose.yml new file mode 100644 index 0000000..2bb8602 --- /dev/null +++ b/ai/llama_cpp/docker-compose.yml @@ -0,0 +1,29 @@ +version: "3.8" + +networks: + ai: + external: true + +services: + llm-server: + image: ghcr.io/ggml-org/llama.cpp:server-cuda + container_name: llm-server + restart: unless-stopped + networks: + - ai + volumes: + - /pwspool/software/ai/llama_cpp:/models + command: > + -m /models/Llama-3.1-8B-Instruct-Q4_K_M.gguf + --host 0.0.0.0 + --port 8080 + -ngl 999 + -c 4096 + --threads 6 + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu]