go program to reproduce azure devops azurerm service connection verification failure after secret rotation

main.go

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/Azure/azure-sdk-for-go/profiles/latest/resources/mgmt/subscriptions"
	rmauth "github.com/Azure/go-autorest/autorest/azure/auth"
	"github.com/fatih/color"
	"github.com/google/uuid"
	"github.com/hashicorp/go-azure-sdk/sdk/auth"
	"github.com/hashicorp/go-azure-sdk/sdk/environments"
	"github.com/manicminer/hamilton/msgraph"
	"github.com/microsoft/azure-devops-go-api/azuredevops/v7"
	"github.com/microsoft/azure-devops-go-api/azuredevops/v7/serviceendpoint"
)

var (
	targetAppObjId    = "[TARGET APP OBJECT ID]"
	targetAppClientId = "[TARGET APP CLIENT ID]"
)

// Expected environment variables:
// AZURE_TENANT_ID
// AZURE_CLIENT_ID
// AZURE_CLIENT_SECRET
// AZDO_PERSONAL_ACCESS_TOKEN
// AZDO_ORG_SERVICE_URL
func main() {
	// Fetches azure credentials from environment variables
	tenantId, clientId, clientSecret, err := FetchCredentialsFromEnv()
	if err != nil {
		log.Fatal(err)
	}

	// Creates msgraph client to add new password to the target application
	ctx := context.Background()
	appclient, err := newAppClient(ctx, tenantId, clientId, clientSecret)
	if err != nil {
		log.Fatal(err)
	}

	// Adds new password to the target application
	pwd, _, err := appclient.AddPassword(ctx, targetAppObjId, msgraph.PasswordCredential{
		DisplayName: newPtr(genRandomName()),
		EndDateTime: newPtr(time.Now().AddDate(0, 0, 1)),
	})
	if err != nil {
		log.Fatal(err)
	}

	// Creates new subscription client with new password
	subscriptionsClient := subscriptions.NewClient()
	subscriptionsClient.Authorizer, err = rmauth.NewClientCredentialsConfig(
		targetAppClientId,
		*pwd.SecretText,
		tenantId,
	).Authorizer()
	if err != nil {
		log.Fatal(err)
	}

	// Creates new AZDO connection to test service endpoint
	connection := azuredevops.NewPatConnection(os.Getenv("AZDO_ORG_SERVICE_URL"), os.Getenv("AZDO_PERSONAL_ACCESS_TOKEN"))
	serviceEndpointClient, err := serviceendpoint.NewClient(ctx, connection)
	if err != nil {
		log.Fatal(err)
	}

	// Fetches endpoint details
	endpoint, err := serviceEndpointClient.GetServiceEndpointDetails(ctx, serviceendpoint.GetServiceEndpointDetailsArgs{
		Project:    newPtr("Example Project"),
		EndpointId: newPtr(uuid.MustParse("a0c1e6be-4177-4141-b0a2-f32981eeebb5")),
	})
	if err != nil {
		log.Fatalf("error getting endpoint details: %v", err)
	}

	// Executes endpoint verification multiple times
	// also attempts to list subscriptons to verify credentials
	execServiceEndpointReqArgs := genExecuteRequestArgs(*pwd.SecretText, tenantId, *endpoint)
	fmt.Printf("testing endpoint with secret: %s\n", *pwd.SecretText)
	for i := 0; i < 100; i++ {
		subs, err := listSubscriptions(ctx, subscriptionsClient)
		if err != nil {
			fmt.Printf("%s %v\n", color.YellowString("error listing subscriptions:"), err)
		} else {
			fmt.Printf("%s %s\n", color.GreenString("subscriptions:"), strings.Trim(subs, ", "))
		}
		existingReqResult, _ := serviceEndpointClient.ExecuteServiceEndpointRequest(ctx, execServiceEndpointReqArgs)
		if *existingReqResult.StatusCode == "ok" {
			fmt.Printf("%s %s\n", color.GreenString("endpoint request result:"), *existingReqResult.StatusCode)
		} else {
			fmt.Printf("%s %s: %s\n", color.YellowString("endpoint request result:"), *existingReqResult.StatusCode, *existingReqResult.ErrorMessage)
		}
		time.Sleep(2 * time.Second)
	}
}

func newPtr[T any](v T) *T {
	return &v
}

func genExecuteRequestArgs(clientSecret, tenantId string, endpoint serviceendpoint.ServiceEndpoint) serviceendpoint.ExecuteServiceEndpointRequestArgs {
	return serviceendpoint.ExecuteServiceEndpointRequestArgs{
		Project:    newPtr("Example Project"),
		EndpointId: newPtr(endpoint.Id.String()),
		ServiceEndpointRequest: &serviceendpoint.ServiceEndpointRequest{
			ServiceEndpointDetails: &serviceendpoint.ServiceEndpointDetails{
				Data: endpoint.Data,
				Authorization: &serviceendpoint.EndpointAuthorization{
					Parameters: &map[string]string{
						"authenticationType":  "spnKey",
						"serviceprincipalid":  "bbf8d313-1e6e-4ca4-b915-d8da8e0947da",
						"serviceprincipalkey": clientSecret,
						"tenantid":            tenantId,
					},
					Scheme: newPtr("ServicePrincipal"),
				},
				Url:  endpoint.Url,
				Type: endpoint.Type,
			},
			DataSourceDetails: &serviceendpoint.DataSourceDetails{
				DataSourceName: newPtr("TestConnection"),
			},
		},
	}
}

func genRandomName() string {
	return uuid.New().String()
}

// listSubscriptions lists all subscriptions in the given tenant
func listSubscriptions(ctx context.Context, client subscriptions.Client) (string, error) {
	var subscriptions []subscriptions.Subscription
	subscriptionsPage, err := client.List(ctx)
	if err != nil {
		return "", err
	}
	for _, subscription := range subscriptionsPage.Values() {
		subscriptions = append(subscriptions, subscription)
		err = subscriptionsPage.NextWithContext(ctx)
		if err != nil {
			return "", err
		}
	}
	// join subscriptions disply names into a single string
	var subs string
	for _, subscription := range subscriptions {
		subs = subs + *subscription.DisplayName + ", "
	}

	return subs, nil
}

func newAppClient(ctx context.Context, tenantId, clientId, clientSecret string) (*msgraph.ApplicationsClient, error) {
	env := environments.AzurePublic()

	credentials := auth.Credentials{
		Environment:  *env,
		TenantID:     tenantId,
		ClientID:     clientId,
		ClientSecret: clientSecret,

		EnableAuthenticatingUsingClientSecret: true,
	}

	authorizer, err := auth.NewAuthorizerFromCredentials(ctx, credentials, env.MicrosoftGraph)
	if err != nil {
		return nil, err
	}

	appclient := msgraph.NewApplicationsClient()
	appclient.BaseClient.Authorizer = authorizer
	return appclient, nil
}

// FetchCredentialsFromEnv fetches azure credentials from environment variables
func FetchCredentialsFromEnv() (string, string, string, error) {
	tenantId := os.Getenv("AZURE_TENANT_ID")
	clientId := os.Getenv("AZURE_CLIENT_ID")
	clientSecret := os.Getenv("AZURE_CLIENT_SECRET")
	if tenantId == "" || clientId == "" || clientSecret == "" {
		return "", "", "", fmt.Errorf("missing required environment variables")
	}

	return tenantId, clientId, clientSecret, nil
}

Example output: