EvalHub arc_easy disconnected

download_dataset.py

import os
from datasets import load_dataset

DATASET_NAME = "ai2_arc"
DATASET_SUBSET = "ARC-Easy"
OUTPUT_DIR = "./staging"

os.environ["HF_HOME"] = OUTPUT_DIR

print(f"Downloading dataset: {DATASET_NAME} ({DATASET_SUBSET})")
ds = load_dataset(DATASET_NAME, DATASET_SUBSET)
print(ds)
print(f"Dataset cached in {OUTPUT_DIR}/datasets/")

download_tokenizer.py

from transformers import AutoTokenizer

MODEL = "google/flan-t5-small"
OUTPUT_DIR = "./staging/tokenizer"

print(f"Downloading tokenizer: {MODEL}")
tokenizer = AutoTokenizer.from_pretrained(MODEL)
tokenizer.save_pretrained(OUTPUT_DIR)
print(f"Tokenizer saved to {OUTPUT_DIR}")

submit_job.sh

#!/usr/bin/env bash
# Submit an offline arc_easy evaluation job to EvalHub
set -euo pipefail

NAMESPACE="test"
EVALHUB_NAMESPACE="opendatahub"

ROUTE=$(oc get route evalhub -n "$EVALHUB_NAMESPACE" -o jsonpath='{.spec.host}')
TOKEN=$(oc whoami -t)
URL="https://$ROUTE/api/v1/evaluations/jobs"

echo "Submitting offline arc_easy job to $URL"

curl -sk -X POST "$URL" \
    -H "Authorization: Bearer $TOKEN" \
    -H "Content-Type: application/json" \
    -H "X-Tenant: $NAMESPACE" \
    -d '{
      "name": "lmeval-offline-arc_easy",
      "model": {
        "url": "http://vllm-server.test.svc.cluster.local:8000",
        "name": "google/flan-t5-small"
      },
      "benchmarks": [{
        "id": "arc_easy",
        "provider_id": "lm_evaluation_harness",
        "parameters": {
          "offline": true,
          "tokenizer": "/test_data/tokenizer",
          "num_examples": 10
        },
        "test_data_ref": {
          "s3": {
            "bucket": "mlpipeline",
            "key": "offline/",
            "secret_ref": "evalhub-minio"
          }
        }
      }]
    }' | python3 -m json.tool

upload_to_minio.sh

#!/usr/bin/env bash
# Upload tokenizer and dataset from ./staging to MinIO
set -euo pipefail

NAMESPACE="test"
BUCKET="mlpipeline"
PREFIX="offline"
MINIO_PORT="19000"
MINIO_USER="minioadmin"
MINIO_PASS="minioadmin"
STAGING_DIR="./staging"

# Find MinIO pod and port-forward
MINIO_POD=$(kubectl get pod -n "$NAMESPACE" -l app=evalhub-minio \
   -o jsonpath='{.items[0].metadata.name}')
echo "MinIO pod: $MINIO_POD"

kubectl port-forward -n "$NAMESPACE" "pod/$MINIO_POD" "$MINIO_PORT:9000" &
PF_PID=$!
trap 'kill $PF_PID 2>/dev/null || true' EXIT
sleep 2

export AWS_ACCESS_KEY_ID="$MINIO_USER"
export AWS_SECRET_ACCESS_KEY="$MINIO_PASS"
export AWS_DEFAULT_REGION="us-east-1"
ENDPOINT="http://localhost:$MINIO_PORT"

# Create bucket (ignore if exists)
aws --endpoint-url "$ENDPOINT" s3 mb "s3://$BUCKET" 2>/dev/null || true

# Clear previous data
echo "Clearing s3://$BUCKET/$PREFIX/ ..."
aws --endpoint-url "$ENDPOINT" s3 rm "s3://$BUCKET/$PREFIX/" --recursive --quiet 2>/dev/null || true

# Upload tokenizer
echo "Uploading tokenizer..."
aws --endpoint-url "$ENDPOINT" s3 cp \
   "$STAGING_DIR/tokenizer/" "s3://$BUCKET/$PREFIX/tokenizer/" --recursive --quiet

# Upload datasets
echo "Uploading datasets..."
aws --endpoint-url "$ENDPOINT" s3 cp \
   "$STAGING_DIR/datasets/" "s3://$BUCKET/$PREFIX/datasets/" --recursive --quiet

# Upload hub cache if present
if [ -d "$STAGING_DIR/hub" ]; then
   echo "Uploading hub cache..."
   aws --endpoint-url "$ENDPOINT" s3 cp \
       "$STAGING_DIR/hub/" "s3://$BUCKET/$PREFIX/hub/" --recursive --quiet
fi

TOTAL=$(aws --endpoint-url "$ENDPOINT" s3 ls "s3://$BUCKET/$PREFIX/" --recursive | wc -l)
echo "Done. $TOTAL objects in s3://$BUCKET/$PREFIX/"