batch-inference-google-search

test-dataset-submitted-input.md

Contents of input JSONL:

> bat test_dataset.jsonl
─────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
     │ File: test_dataset.jsonl
─────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
   1 │ {"custom_id":"request-1","method":"POST","url":"/v1/chat/completions","body":{"model":"gemini-3-flash-preview","messages":[{"role":"user","content":"Where is NYU located
     │ ?"}],"max_tokens":2048}}
   2 │ {"custom_id":"request-2","method":"POST","url":"/v1/chat/completions","body":{"model":"gemini-3-flash-preview","messages":[{"role":"user","content":"What resources are a
     │ vailable for genAI research at nyu?"}],"tools":[{"type":"function","function":{"name":"google_search","description":"Retrieves search results from internet to expand con
     │ text.","parameters":{"type":"object","properties":{"timeRangeFilter":{"type":"string","nullable": true,"description":"use no time range filter"}},"required":["timeRangeF
     │ ilter"],"additionalProperties":false}}}],"max_tokens":8192}}
   3 │ 
─────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────

Expanded for readability:

> bat test_dataset.jsonl | jq
{
  "custom_id": "request-1",
  "method": "POST",
  "url": "/v1/chat/completions",
  "body": {
    "model": "gemini-3-flash-preview",
    "messages": [
      {
        "role": "user",
        "content": "Where is NYU located?"
      }
    ],
    "max_tokens": 2048
  }
}
{
  "custom_id": "request-2",
  "method": "POST",
  "url": "/v1/chat/completions",
  "body": {
    "model": "gemini-3-flash-preview",
    "messages": [
      {
        "role": "user",
        "content": "What resources are available for genAI research at nyu?"
      }
    ],
    "tools": [
      {
        "type": "function",
        "function": {
          "name": "google_search",
          "description": "Retrieves search results from internet to expand context.",
          "parameters": {
            "type": "object",
            "properties": {
              "timeRangeFilter": {
                "type": "string",
                "nullable": true,
                "description": "use no time range filter"
              }
            },
            "required": [
              "timeRangeFilter"
            ],
            "additionalProperties": false
          }
        }
      }
    ],
    "max_tokens": 8192
  }
}

Contents of transformed JSONL (by Portkey):

> bat test_dataset_transformed.jsonl 
─────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
     │ File: test_dataset_transformed.jsonl
─────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
   1 │ {"request":{"contents":[{"role":"user","parts":[{"text":"Where is NYU located?"}]}],"generationConfig":{"maxOutputTokens":2048}},"requestId":"request-1"}
   2 │ {"request":{"contents":[{"role":"user","parts":[{"text":"What resources are available for genAI research at nyu?"}]}],"generationConfig":{"maxOutputTokens":8192},"tools"
     │ :[{"googleSearch":{}}]},"requestId":"request-2"}
─────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────

Expanded for readability:

> bat test_dataset_transformed.jsonl | jq
{
  "request": {
    "contents": [
      {
        "role": "user",
        "parts": [
          {
            "text": "Where is NYU located?"
          }
        ]
      }
    ],
    "generationConfig": {
      "maxOutputTokens": 2048
    }
  },
  "requestId": "request-1"
}
{
  "request": {
    "contents": [
      {
        "role": "user",
        "parts": [
          {
            "text": "What resources are available for genAI research at nyu?"
          }
        ]
      }
    ],
    "generationConfig": {
      "maxOutputTokens": 8192
    },
    "tools": [
      {
        "googleSearch": {}
      }
    ]
  },
  "requestId": "request-2"
}

Error trace when the transformed input is submitted for batch inference:

Failed to import data. Please check 'Prepare input' section of the batch predictions documentation. For Gemini models, see https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/batch-prediction-gemini#prepare_your_inputs Query error: Cannot store struct 'request.tools.googleSearch' with no fields at [1:1]