Docker for LLMs, Agents & MCP

nvidia-container-toolkit Setup

# Install the NVIDIA Container Toolkit
sudo apt-get install -y nvidia-container-toolkit
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

# Verify GPU access inside a container
docker run --rm --gpus all nvidia/cuda:12.4.0-base-ubuntu22.04 nvidia-smi

Key Flags

# CPU only
docker run -d -p 11434:11434 \
  --name ollama \
  -v ollama_data:/root/.ollama \
  ollama/ollama

# With GPU
docker run -d --gpus all \
  -p 11434:11434 \
  --name ollama \
  -v ollama_data:/root/.ollama \
  ollama/ollama

# Pull a model
docker exec ollama \
  ollama pull llama3.1:8b

# Run inference
curl http://localhost:11434/api/generate \
  -d '{
    "model": "llama3.1:8b",
    "prompt": "Explain Docker in one sentence",
    "stream": false
  }'

Key insight: Mount a named volume at /root/.ollama so model weights persist across container restarts. Multiple Ollama containers can share the same volume read-only.

# docker-compose.yml — shared model cache
volumes:
  hf_cache:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: /data/huggingface_cache

services:
  vllm:
    image: vllm/vllm-openai:latest
    volumes:
      - hf_cache:/root/.cache/huggingface:ro

# Route by model name
services:
  llm-small:
    image: ollama/ollama
    deploy: { resources: { reservations: { devices: [{ capabilities: [gpu], count: 1 }] } } }
  llm-large:
    image: vllm/vllm-openai:latest
    command: ["--model", "meta-llama/Llama-3.1-70B-Instruct", "--tensor-parallel-size", "4"]
    deploy: { resources: { reservations: { devices: [{ capabilities: [gpu], count: 4 }] } } }

FROM python:3.12-slim
WORKDIR /app

COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

COPY agent/ ./agent/
COPY tools/ ./tools/

ENV OPENAI_API_KEY=""
ENV LANGCHAIN_TRACING_V2=true

EXPOSE 8000
CMD ["uvicorn", "agent.server:app", \
     "--host", "0.0.0.0", "--port", "8000"]

# agent/server.py
from fastapi import FastAPI
from langgraph.prebuilt import create_react_agent
from langchain_openai import ChatOpenAI

app = FastAPI()
llm = ChatOpenAI(model="gpt-4o")
agent = create_react_agent(llm, tools=[...])

@app.post("/invoke")
async def invoke(query: str):
    result = await agent.ainvoke(
        {"messages": [("user", query)]}
    )
    return result

Tip: Use langgraph deploy for production — it generates an optimised Dockerfile with checkpointing and streaming built in.

FROM python:3.12-slim
WORKDIR /app

RUN pip install crewai crewai-tools
COPY crew_config/ ./crew_config/
COPY main.py .

CMD ["python", "main.py"]

# main.py
from crewai import Agent, Task, Crew

researcher = Agent(
    role="Researcher",
    goal="Find accurate information",
    llm="ollama/llama3.1:8b",
    llm_config={"base_url": "http://ollama:11434"}
)
writer = Agent(
    role="Writer",
    goal="Write clear reports",
    llm="ollama/llama3.1:8b",
    llm_config={"base_url": "http://ollama:11434"}
)
crew = Crew(agents=[researcher, writer],
            tasks=[...])
crew.kickoff()

Pattern: CrewAI agents point to http://ollama:11434 — the Docker Compose service name. No localhost, no IP addresses.

version: "3.9"
services:
  ollama:
    image: ollama/ollama
    ports: ["11434:11434"]
    volumes: [ollama_data:/root/.ollama]
    deploy:
      resources:
        reservations:
          devices: [{ capabilities: [gpu], count: 1 }]

  chromadb:
    image: chromadb/chroma:latest
    ports: ["8000:8000"]
    volumes: [chroma_data:/chroma/chroma]
    environment:
      - ANONYMIZED_TELEMETRY=false

  agent:
    build: ./agent
    depends_on: [ollama, chromadb]
    environment:
      - OLLAMA_BASE_URL=http://ollama:11434
      - CHROMA_HOST=chromadb
      - CHROMA_PORT=8000
    ports: ["8080:8080"]

volumes:
  ollama_data:
  chroma_data:

MCP is an open standard (by Anthropic) that lets LLMs interact with external systems through a unified protocol.

Key idea: MCP replaces bespoke tool integrations with a single protocol. One client can talk to any MCP server — like USB-C for AI tools.

FROM node:22-slim
WORKDIR /app

COPY package*.json ./
RUN npm ci --production

COPY src/ ./src/

# SSE transport on port 3001
ENV MCP_TRANSPORT=sse
ENV MCP_PORT=3001
EXPOSE 3001

HEALTHCHECK --interval=30s --timeout=5s \
  CMD curl -f http://localhost:3001/health || exit 1

CMD ["node", "src/index.js"]

{
  "mcpServers": {
    "filesystem": {
      "command": "docker",
      "args": [
        "run", "-i", "--rm",
        "-v", "/data:/data:ro",
        "mcp/filesystem-server",
        "/data"
      ]
    },
    "postgres": {
      "url": "http://localhost:3002/sse",
      "env": {
        "POSTGRES_URL": "postgresql://..."
      }
    }
  }
}

stdio via Docker: Use docker run -i so the MCP client can pipe JSON-RPC over stdin/stdout. The container acts like a local binary.

version: "3.9"
services:
  mcp-filesystem:
    image: mcp/filesystem-server
    volumes:
      - ./workspace:/data:ro
    environment:
      - MCP_TRANSPORT=sse
      - MCP_PORT=3001
    ports: ["3001:3001"]

  mcp-postgres:
    image: mcp/postgres-server
    environment:
      - POSTGRES_URL=postgresql://user:pass@db:5432/mydb
      - MCP_TRANSPORT=sse
      - MCP_PORT=3002
    ports: ["3002:3002"]
    depends_on: [db]

  mcp-brave-search:
    image: mcp/brave-search
    environment:
      - BRAVE_API_KEY=${BRAVE_API_KEY}
      - MCP_TRANSPORT=sse
      - MCP_PORT=3003
    ports: ["3003:3003"]

  db:
    image: postgres:16
    environment:
      - POSTGRES_PASSWORD=pass
    volumes: [pgdata:/var/lib/postgresql/data]

volumes:
  pgdata:

An MCP Gateway sits between clients and MCP servers, providing a single endpoint that aggregates, secures, and routes tool calls across multiple backends.

docker run -p 3001:3001 \
  supercorp/supergateway \
  --stdio "npx -y @modelcontextprotocol/server-filesystem /data" \
  --port 3001

# SSE to stdio bridge
docker run -p 3002:3002 \
  sparfenyuk/mcp-proxy \
  --sse-port 3002 \
  --command "python mcp_server.py"

# Connect remote SSE from a stdio client
mcp-proxy --sse-url http://remote:3001/sse

Why bridges matter: Claude Desktop and Cursor only support stdio. Bridges let you run MCP servers anywhere and connect them to local clients.

version: "3.9"
services:
  gateway:
    image: supercorp/supergateway
    ports: ["3000:3000"]
    environment:
      - GATEWAY_PORT=3000
      - AUTH_TOKEN=${GATEWAY_AUTH_TOKEN}
    depends_on: [mcp-filesystem, mcp-postgres, mcp-search]

  mcp-filesystem:
    image: mcp/filesystem-server
    volumes: [./workspace:/data:ro]
    environment: { MCP_TRANSPORT: sse, MCP_PORT: "3001" }
    expose: ["3001"]

  mcp-postgres:
    image: mcp/postgres-server
    environment:
      POSTGRES_URL: postgresql://user:pass@db:5432/app
      MCP_TRANSPORT: sse
      MCP_PORT: "3002"
    expose: ["3002"]
    depends_on: [db]

  mcp-search:
    image: mcp/brave-search
    environment:
      BRAVE_API_KEY: ${BRAVE_API_KEY}
      MCP_TRANSPORT: sse
      MCP_PORT: "3003"
    expose: ["3003"]

  db:
    image: postgres:16
    environment: { POSTGRES_PASSWORD: pass }
    volumes: [pgdata:/var/lib/postgresql/data]

volumes:
  pgdata:

Note: Internal MCP servers use expose (not ports) — only the gateway is publicly accessible.

# Gateway config with auth
gateway:
  auth:
    type: bearer
    tokens:
      - token: ${ADMIN_TOKEN}
        role: admin
        allowed_tools: ["*"]
      - token: ${READONLY_TOKEN}
        role: reader
        allowed_tools: ["filesystem.read", "search.*"]
  rate_limit:
    requests_per_minute: 60
    burst: 10

# Kubernetes GPU scheduling example
resources:
  limits:
    nvidia.com/gpu: 1        # Request exactly 1 GPU
    memory: "16Gi"
  requests:
    nvidia.com/gpu: 1
    memory: "12Gi"
nodeSelector:
  gpu-type: "a100-40gb"      # Target specific GPU type

Brendan Lynskey — 2026