onco* compose setup

docker-compose.onco.yml

services:
  mesa-local:
    image: ghcr.io/londonaicentre/mesa-deploy/mesa-local
    pull_policy: always
    environment:
      - MODEL_NAME=${MODEL_NAME:-oncoqwen_1_0_0}
      - WEIGHTS_ID=${WEIGHTS_ID}
      - WEIGHTS_KEY=${WEIGHTS_KEY}
      - MAX_MODEL_LEN=${MAX_MODEL_LEN:-39067} # adjust for gpu mem size
    volumes:
      - weights:/app/assets/
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "sh", "-c", "curl -f http://localhost:$${PORT:-5000}/health"]
      interval: 15s
      timeout: 5s
      retries: 40
      start_period: 120s

  mesa-runner:
    image: ghcr.io/londonaicentre/mesa-runner
    pull_policy: always
    volumes:
      - ./data/input:/mnt/in 
      - ./data/output:/mnt/out # host output dir may need additional write perms
      - ./sample_config.yaml:/app/config.yaml
    depends_on:
      mesa-local:
        condition: service_healthy

volumes:
  weights:

sample.env

MODEL_NAME=oncoqwen_1_0_0
WEIGHTS_ID=<Your weights ID here>
WEIGHTS_KEY=<Your weights key here>

sample_config.yaml

job_name:
  model_name: oncoqwen_1_0_0
  
  inference:
    openai_endpoint: http://mesa-local:5000/v1
    enable_extended_timeouts: true
    max_concurrent_requests: 32
  
  storage:
    type: "filesystem"

    source_directory: "/mnt/in"
    target_directory: "/mnt/out"

    input_file_format: 'json'
    output_file_format: 'json'

    write_buffer_size: 1000