observe-azure-provisioning.ps1

$subscription = 'dd0d4271-9b26-4c37-a025-1284a43a4385';
$observations = @{};
foreach ($resourceGroupName in @('taskcluster-staging-workers-us-central')) {
  $observations[$resourceGroupName] = @{};
  while (($vms = @(Get-AzVm -ResourceGroupName $resourceGroupName -Status)) -or ($vms.Length -gt 0)) {
    Write-Output -InputObject ('{0} virtual machines observed in {1}' -f $vms.Length, $resourceGroupName);
    foreach ($vm in $vms) {
      $name = $vm.Name;
      if (-not $observations[$resourceGroupName].ContainsKey($name)) {
        $observations[$resourceGroupName][$name] = @{};
      }
      try {
        $vm = (Get-AzVm -ResourceGroupName $resourceGroupName -Name $name -Status -ErrorAction 'SilentlyContinue');
        Write-Output -InputObject ('- {0}/{1} ({2})' -f $resourceGroupName, $name, $vm.Statuses[$vm.Statuses.Count - 1].Code);
      } catch {
        Write-Output -InputObject ('- {0}/{1} (ProvisioningState/deleted)' -f $resourceGroupName, $name);
        #$observations[$resourceGroupName].Remove($name);
      } finally {
        $resourceId = ('/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Compute/virtualMachines/{2}' -f $subscription, $resourceGroupName, $name);
        $operations = @(Get-AzLog -ResourceId $resourceId 3> $null | ? { $_.Category.Value -ne 'Policy' }); #$_.Level -eq 'Error'
        if (($operations) -and ($operations.Length)) {
          foreach ($operation in $operations) {
            if (($operation.Properties) -and ($operation.Properties.Content.statusMessage)) {
              $errorStatus = ($operation.Properties.Content.statusMessage | ConvertFrom-Json);
              $observations[$resourceGroupName][$name][('{0:u}' -f $operation.EventTimestamp)] = ('[operation] {0} {1} {2} {3}' -f $operation.OperationName.LocalizedValue, $operation.Status.LocalizedValue, $errorStatus.error.message, $errorStatus.error.details[0].message);
            } else {
              $observations[$resourceGroupName][$name][('{0:u}' -f $operation.EventTimestamp)] = ('[operation] {0} {1}' -f $operation.OperationName.LocalizedValue, $operation.Status.LocalizedValue);
            }
          }
        }
        $latestObservationTimestamp = @($observations[$resourceGroupName][$name].Keys | Sort-Object -Descending)[0];
        if (($latestObservationTimestamp) -and ($observations[$resourceGroupName][$name][$latestObservationTimestamp] -ne ('[observation] {0}' -f $vm.Statuses[$vm.Statuses.Count - 1].Code))) {
          $observations[$resourceGroupName][$name][('{0:u}' -f (Get-Date))] = ('[observation] {0}' -f $vm.Statuses[$vm.Statuses.Count - 1].Code);
        }
        foreach ($timestamp in @($observations[$resourceGroupName][$name].Keys | Sort-Object -Descending)) {
          Write-Output -InputObject ('  - {0}: {1}' -f $timestamp, $observations[$resourceGroupName][$name][$timestamp]);
        }
      }
    }
  } else {
    Write-Output -InputObject ('no vms found');
  }
}