mirror of
https://github.com/runyanjake/olomana.git
synced 2026-11-10 13:54:52 -08:00
29 lines
733 B
YAML
29 lines
733 B
YAML
networks:
|
|
ai:
|
|
name: ai
|
|
driver: bridge
|
|
|
|
services:
|
|
llm-server:
|
|
image: ghcr.io/ggml-org/llama.cpp:server-cuda
|
|
container_name: llm-server
|
|
restart: unless-stopped
|
|
networks:
|
|
- ai
|
|
volumes:
|
|
- ${MODELS_PATH:-/pwspool/software/ai/llama_cpp}:/models
|
|
environment:
|
|
- LLAMA_ARG_MODEL=/models/${MODEL_FILE:-Llama-3.1-8B-Instruct-Q4_K_M.gguf}
|
|
- LLAMA_ARG_HOST=0.0.0.0
|
|
- LLAMA_ARG_PORT=8080
|
|
- LLAMA_ARG_N_GPU_LAYERS=${GPU_LAYERS:-999}
|
|
- LLAMA_ARG_CTX_SIZE=${CTX_SIZE:-4096}
|
|
- LLAMA_ARG_THREADS=${THREADS:-6}
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|