From 7f38fa2aedd46674494e3f280237f5dc2112ca3e Mon Sep 17 00:00:00 2001
From: whitney <whitney.server@gmail.com>
Date: Fri, 23 Oct 2026 10:43:52 -0700
Subject: [PATCH] Add .env, update readme

---
 ai/llama_cpp/.env.example       | 10 ++++++++++
 ai/llama_cpp/.gitignore         |  1 +
 ai/llama_cpp/README.md          | 13 +++++++++++++
 ai/llama_cpp/docker-compose.yml | 17 +++++++++--------
 4 files changed, 33 insertions(+), 8 deletions(-)
 create mode 100644 ai/llama_cpp/.env.example
 create mode 100644 ai/llama_cpp/.gitignore

diff --git a/ai/llama_cpp/.env.example b/ai/llama_cpp/.env.example
new file mode 100644
index 0000000..23c0d99
--- /dev/null
+++ b/ai/llama_cpp/.env.example
@@ -0,0 +1,10 @@
+# Host directory path where GGUF models are stored
+MODELS_PATH=/pwspool/software/ai/llama_cpp
+
+# Filename of the default model to load
+MODEL_FILE=Llama-3.1-8B-Instruct-Q4_K_M.gguf
+
+# Performance and hardware tuning
+GPU_LAYERS=999
+CTX_SIZE=4096
+THREADS=6
diff --git a/ai/llama_cpp/.gitignore b/ai/llama_cpp/.gitignore
new file mode 100644
index 0000000..4c49bd7
--- /dev/null
+++ b/ai/llama_cpp/.gitignore
@@ -0,0 +1 @@
+.env
diff --git a/ai/llama_cpp/README.md b/ai/llama_cpp/README.md
index 67963e4..e083478 100644
--- a/ai/llama_cpp/README.md
+++ b/ai/llama_cpp/README.md
@@ -3,3 +3,16 @@ A general purpose llm server that projects can connect to.
 
 ## Network
 Create the `ai` docker network, applications can access the OpenAI spec server through `http://llm-server:8080/v1`.
+
+## Setup
+1. Start service
+
+```bash
+docker compose up
+```
+
+2. Download model(s)
+```bash
+cd /pwspool/software/ai/llama_cpp
+sudo wget https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf -O Llama-3.1-8B-Instruct-Q4_K_M.gguf
+```
diff --git a/ai/llama_cpp/docker-compose.yml b/ai/llama_cpp/docker-compose.yml
index 8480d80..be8dfe7 100644
--- a/ai/llama_cpp/docker-compose.yml
+++ b/ai/llama_cpp/docker-compose.yml
@@ -1,5 +1,6 @@
 networks:
   ai:
+    name: ai
     driver: bridge
 
 services:
@@ -10,14 +11,14 @@ services:
     networks:
       - ai
     volumes:
-      - /pwspool/software/ai/llama_cpp:/models
-    command: >
-      -m /models/Llama-3.1-8B-Instruct-Q4_K_M.gguf
-      --host 0.0.0.0
-      --port 8080
-      -ngl 999
-      -c 4096
-      --threads 6
+      - ${MODELS_PATH:-/pwspool/software/ai/llama_cpp}:/models
+    environment:
+      - LLAMA_ARG_MODEL=/models/${MODEL_FILE:-Llama-3.1-8B-Instruct-Q4_K_M.gguf}
+      - LLAMA_ARG_HOST=0.0.0.0
+      - LLAMA_ARG_PORT=8080
+      - LLAMA_ARG_N_GPU_LAYERS=${GPU_LAYERS:-999}
+      - LLAMA_ARG_CTX_SIZE=${CTX_SIZE:-4096}
+      - LLAMA_ARG_THREADS=${THREADS:-6}
     deploy:
       resources:
         reservations: