GPT4 Vision sample sequence and requests

GPT4Vision and Computer Vision

# Validation of GPT4-vision and AI Vision indexer

# Azure Open AI configuration

$AzureOpenAIEndpoint = ""
$VisionDeploymentName = ""

# Storage Configuration

$StorageAccountEndpoint = ""
$StorageContainer = "videos"
$FileName = ""

# Azure Vision Configuration

$AzureVisionEndpoint = ""
$AzureVisionIndexName = ""
$IngestionJob = ""
$AzureVisionApiKey = ""

# Get Token and get started

$TokenRequest = Get-AzAccessToken -ResourceUrl "https://cognitiveservices.azure.com"
$MyToken = $TokenRequest.token

# Set Indexer Body 

$NewIndexBody = @"
{
    "features": [
        {
            "name": "vision",
            "modelVersion": "2023-05-31",
            "domain": "surveillance"
        }
    ]
}
"@

$ExampleIndexBody = @"
{
  "metadataSchema": {
    "fields": [
      {
        "name": "cameraId",
        "searchable": false,
        "filterable": true,
        "type": "string"
      },
      {
        "name": "timestamp",
        "searchable": false,
        "filterable": true,
        "type": "datetime"
      },
      {
        "name": "description",
        "searchable": true,
        "filterable": true,
        "type": "string"
      }
    ]
  },
  "features": [
        {
            "name": "vision",
            "modelVersion": "2023-05-31",
            "domain": "surveillance"
        }
    ]
}
"@  

# PUT AI Vision indexer request

$AzureVisionRequest = @{
    Uri = "https://$($AzureVisionEndpoint)/computervision/retrieval/indexes/$($AzureVisionIndexName)?api-version=2023-05-01-preview"
    Headers = @{
        Authorization = "Bearer $($MyToken)"
        'Content-Type' = 'application/json'
        }
    Body = $NewIndexBody
    Method = 'PUT'
    }

$Response = Invoke-WebRequest @AzureVisionRequest
[Newtonsoft.Json.Linq.JObject]::Parse($Response.Content).ToString()

# GET the AI Vision Index

$GetAzureVisionIndexRequest = @{
    Uri = "https://$($AzureVisionEndpoint)/computervision/retrieval/indexes/$($AzureVisionIndexName)?api-version=2023-05-01-preview"
    Headers = @{
        Authorization = "Bearer $($MyToken)"
        }
    Method = 'GET'
    }

$Response = Invoke-WebRequest @GetAzureVisionIndexRequest
[Newtonsoft.Json.Linq.JObject]::Parse($Response.Content).ToString()

# Add videos to the indexer

$Videos = @"
{
  "videos": [
    {
      "mode": "add",
      "documentId": "$($IngestionJob)",
      "documentUrl": "https://$($StorageAccountEndpoint)/$($StorageContainer)/$($FileName)?sp=r&st=2024-02-19T15:16:37Z&se=2024-02-19T23:16:37Z&skoid=00b7deec-5086-4df3-926f-bdaceaac3576&sktid=beee8d74-3396-4d97-9ce3-73b5ee9d36ea&skt=2024-02-19T15:16:37Z&ske=2024-02-19T23:16:37Z&sks=b&skv=2022-11-02&spr=https&sv=2022-11-02&sr=b&sig=THHU8RrMl%2FGXQS1Q7CY4mVh%2BmVifkwqHGn4RDY%2BoPJA%3D",
      "metadata": {}
    }
  ],
  "generateInsightIntervals": false,
  "moderation": false,
  "filterDefectedFrames": false,
  "includeSpeechTranscript": true
}
"@

# PUT AI Vision ingestion job

$AzureVisionIngestionRequest = @{
    Uri = "https://$($AzureVisionEndpoint)/computervision/retrieval/indexes/$($AzureVisionIndexName)/ingestions/$($IngestionJob)?api-version=2023-05-01-preview"
    Headers = @{
        Authorization = "Bearer $($MyToken)"
        'Content-Type' = 'application/json'
        }
    Body = $Videos
    Method = 'PUT'
    }
$Response = Invoke-WebRequest @AzureVisionIngestionRequest
[Newtonsoft.Json.Linq.JObject]::Parse($Response.Content).ToString()

# GET AI ingestion job

$GetAzureVisionIngestionRequest = @{
    Uri = "https://$($AzureVisionEndpoint)/computervision/retrieval/indexes/$($AzureVisionIndexName)/ingestions/$($IngestionJob)?api-version=2023-05-01-preview"
    Headers = @{
        Authorization = "Bearer $($MyToken)"
        }
    Method = 'GET'
    }
$GetResponse = Invoke-WebRequest @GetAzureVisionIngestionRequest
[Newtonsoft.Json.Linq.JObject]::Parse($GetResponse.Content).ToString()


# AI Ingestion Request

$VisionBody = @"
{
    "enhancements": {
            "video": {
              "enabled": true
            }
    },
    "dataSources": [
    {
        "type": "AzureComputerVisionVideoIndex",
        "parameters": {
            "endpoint": "https://$($AzureVisionEndpoint)",
            "computerVisionApiKey": "$($AzureVisionApiKey)",
            "indexName": "$($AzureVisionIndexName)",
            "videoUrls": ["https://spainswedencentraltyb5o.blob.core.windows.net/videos/IMG_0863.MOV?sp=r&st=2024-02-19T16:05:19Z&se=2024-02-20T00:05:19Z&spr=https&sv=2022-11-02&sr=b&sig=o%2Bs0cpoLq47S%2FVG5LE%2Fi3PM9Kr%2FtAPbV8wxzq9dLqcE%3D"],
            "roleInformation": "You are an AI assistant that helps people find information."
            }
        }
    ],
    "messages": [ 
        {
            "role": "system",
            "content": [
                {
                    "type": "text",
                    "text": "You are an AI assistant that helps people find information."
                }
            ]
        },
        {
            "role": "user",
            "content": [
                    {
                        "type": "text",
                        "text": "Provide a summary of what is happening in this movie, and explain it in an engaging way for children"
                    },
                    {
                        "type": "acv_document_id",
                        "acv_document_id": "$($IngestionJob)"
                    }
            ]
        }
    ],
    "temperature": 0.7,
    "top_p": 0.95,
    "max_tokens": 150,
    "stream": false
}
"@

# POST Use Azure OpenAI with Vision

$AzureOpenAIRequest = @{
    Uri = "https://$($AzureOpenAIEndpoint)/openai/deployments/$($VisionDeploymentName)/extensions/chat/completions?api-version=2023-12-01-preview"
    Headers = @{
        Authorization = "Bearer $($MyToken)"
        'Content-Type' = 'application/json'
        }
    Body = $VisionBody
    Method = 'POST'
    }
$AOAIResponse = Invoke-WebRequest @AzureOpenAIRequest
[Newtonsoft.Json.Linq.JObject]::Parse($AOAIResponse.Content).ToString()