mirror of
https://github.com/runyanjake/olomana.git
synced 2026-11-10 13:54:52 -08:00
30 lines
580 B
YAML
30 lines
580 B
YAML
version: "3.8"
|
|
|
|
networks:
|
|
ai:
|
|
external: true
|
|
|
|
services:
|
|
llm-server:
|
|
image: ghcr.io/ggml-org/llama.cpp:server-cuda
|
|
container_name: llm-server
|
|
restart: unless-stopped
|
|
networks:
|
|
- ai
|
|
volumes:
|
|
- /pwspool/software/ai/llama_cpp:/models
|
|
command: >
|
|
-m /models/Llama-3.1-8B-Instruct-Q4_K_M.gguf
|
|
--host 0.0.0.0
|
|
--port 8080
|
|
-ngl 999
|
|
-c 4096
|
|
--threads 6
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|