networks:
  ai:
    driver: bridge

services:
  llm-server:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda
    container_name: llm-server
    restart: unless-stopped
    networks:
      - ai
    volumes:
      - /pwspool/software/ai/llama_cpp:/models
    command: >
      -m /models/Llama-3.1-8B-Instruct-Q4_K_M.gguf
      --host 0.0.0.0
      --port 8080
      -ngl 999
      -c 4096
      --threads 6
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]