Example Google Cloud Platform (GCP) serverless apps communicating via private network

gcp-gae-cr-private.sh

#!/usr/bin/env bash

#####################################################################
# REFERENCES
# - https://cloud.google.com/run/docs/securing/private-networking#from-other-services
# - https://cloud.google.com/run/docs/securing/private-networking#from-vpc
# - https://cloud.google.com/appengine/docs/flexible/disable-external-ip
# - https://cloud.google.com/dns/docs/records#adding_or_removing_a_record
# - https://cloud.google.com/vpc/docs/configure-private-google-access
# - https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-go-service
# - https://cloud.google.com/appengine/docs/flexible/nodejs/create-app
#####################################################################

export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain

export GCP_REGION="us-central1" # CHANGEME (OPT)
export GCP_ZONE="us-central1-a" # CHANGEME (OPT)
export NETWORK_NAME="default"

# enable apis
gcloud services enable compute.googleapis.com \
    storage.googleapis.com \
    servicenetworking.googleapis.com \
    vpcaccess.googleapis.com \
    run.googleapis.com \
    appengine.googleapis.com \
    cloudbuild.googleapis.com \
    artifactregistry.googleapis.com \
    dns.googleapis.com

# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE


#############################################################
# NETWORKING
#############################################################
export NETWORK_NAME="app-demo"
export RESERVED_RANGE_NAME="google-managed-services"
export SUBNET_NAME="serverless-apps"
export SUBNET_RANGE="10.150.0.0/28"
export CONNECTOR_NAME="serverless-connector"
export CONNECTOR_RANGE="10.200.0.0/28"          # optional
export DNS_ZONE="private-zone"
export DNS_ZONE2="run-zone"
export DNS_TTL="300"

# create network (custom-mode)
gcloud compute networks create $NETWORK_NAME \
    --subnet-mode=custom

# create subnet
gcloud compute networks subnets create $SUBNET_NAME \
    --region=$GCP_REGION \
    --network=$NETWORK_NAME \
    --range=$SUBNET_RANGE \
    --enable-private-ip-google-access

# allocate private range
gcloud compute addresses create $RESERVED_RANGE_NAME \
    --global \
    --purpose=VPC_PEERING \
    --addresses=10.100.0.0 \
    --prefix-length=16 \
    --network=projects/$PROJECT_ID/global/networks/$NETWORK_NAME

# create peering for managed services
gcloud services vpc-peerings connect \
    --service=servicenetworking.googleapis.com \
    --ranges=$RESERVED_RANGE_NAME \
    --network=$NETWORK_NAME

# create serverless vpc connector
gcloud compute networks vpc-access connectors create $CONNECTOR_NAME \
    --region $GCP_REGION \
    --subnet $SUBNET_NAME \
    --subnet-project $PROJECT_ID

# enable firewall access from connector
gcloud compute firewall-rules create fw-allow-connector \
    --action=ALLOW \
    --rules=TCP \
    --source-ranges=35.199.224.0/19 \
    --target-tags=vpc-connector-$GCP_REGION-$CONNECTOR_NAME \
    --direction=INGRESS \
    --network=$NETWORK_NAME \
    --priority=1000 \
    --project=$PROJECT_ID

# create private zones
gcloud dns managed-zones create $DNS_ZONE \
    --description="internal zone" \
    --dns-name="googleapis.com" \
    --networks=$NETWORK_NAME \
    --labels="purpose=demo" \
    --visibility=private
gcloud dns managed-zones create $DNS_ZONE2 \
    --description="internal zone" \
    --dns-name="run.app" \
    --networks=$NETWORK_NAME \
    --labels="purpose=demo" \
    --visibility=private

# add to google private IP ranges to zones
gcloud dns record-sets transaction start --zone=$DNS_ZONE
gcloud dns record-sets transaction add 199.36.153.8 199.36.153.9 199.36.153.10 199.36.153.11 \
    --name="private.googleapis.com" --ttl=$DNS_TTL --type="A" --zone=$DNS_ZONE
gcloud dns record-sets transaction add "private.googleapis.com." \
    --zone=$DNS_ZONE --name="*.googleapis.com" --type="CNAME" --ttl=$DNS_TTL 
gcloud dns record-sets transaction execute --zone=$DNS_ZONE

gcloud dns record-sets transaction start --zone=$DNS_ZONE2
gcloud dns record-sets transaction add 199.36.153.8 199.36.153.9 199.36.153.10 199.36.153.11 \
    --name="run.app" --ttl=$DNS_TTL --type="A" --zone=$DNS_ZONE2
gcloud dns record-sets transaction add "run.app." \
    --zone=$DNS_ZONE2 --name="*.run.app" --type="CNAME" --ttl=$DNS_TTL 
gcloud dns record-sets transaction execute --zone=$DNS_ZONE2


#############################################################
# BACKEND APP
# - Go app on Cloud Run (datetime)
# - CREDIT: Rick Boss ( https://github.com/rickbau5/cr-datetime/blob/main/main.go )
#############################################################
export BACKEND_DIR="datetime"
export RUN_SERVICE="cr-datetime"

mkdir -p $BACKEND_DIR

# create cloud ignore file
cat > .gcloudignore << EOF
.git
*.sh
EOF

# create mod file
cat > $BACKEND_DIR/go.mod << EOF
module github.com/doitintl/example

go 1.20
EOF

# create sample app
cat > $BACKEND_DIR/main.go << EOF
package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"

	"time"
)

const (
	ISO8601 = "2006-01-02T15:04:05Z"
)

func main() {
	port := os.Getenv("PORT")

	if port == "" {
		panic("PORT required")
	}

	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		err := json.NewEncoder(w).Encode(map[string]any{
			"date": time.Now().UTC().Format(ISO8601),
		})

		if err != nil {
			fmt.Println("Error encoding JSON: ", err)
		}
	})

	fmt.Printf("Listening on :%s\n", port)

	if err := http.ListenAndServe(":"+port, nil); err != nil {
		panic(err)
	}
}
EOF

# create cloud run service
gcloud run deploy $RUN_SERVICE \
    --source $BACKEND_DIR \
    --region=$GCP_REGION \
    --ingress=internal-and-cloud-load-balancing \
    --allow-unauthenticated

# set URL to service
export BACKEND_URL=$(gcloud run services describe $RUN_SERVICE --region $GCP_REGION --format="value(status.url)")


#############################################################
# FRONTEND APP
# - NodeJS app on AppEngine Flexible (hello)
#############################################################
export FRONTEND_DIR="hello"

mkdir -p $FRONTEND_DIR

# initialize app engine for project
gcloud app create --project $PROJECT_ID \
    --region $GCP_REGION

# create package.json dependency file
cat > $FRONTEND_DIR/package.json << EOF
{
  "name": "hello",
  "version": "1.0.0",
  "description": "Simple hello app testing App Engine",
  "main": "app.js",
  "scripts": {
    "start": "node app.js"
  },
  "engines": {
    "node": ">=20.0.0"
  },
  "dependencies": {
    "express": "^4.19.2",
    "node-fetch": "^3.3.2"
  }
}
EOF

# create sample app
cat > $FRONTEND_DIR/app.js << EOF
'use strict';

const express = require('express');
const fetch = (...args) =>
	import('node-fetch').then(({default: fetch}) => fetch(...args));

const app = express();

app.get('/', async (req, res) => {
	const url = process.env.BACKEND_URL;

	const options = {
		method: 'GET',
	};

	// promise syntax
	fetch(url, options)
		.then(res => res.json())
		.then(json => console.log(json))
		.catch(err => console.error('error:' + err));
	try {
		let response = await fetch(url, options);
		response = await response.json();
		res.status(200).json(response);
	} catch (err) {
		console.log(err);
		res.status(500).json({msg: \`Internal Server Error.\`});
	}
});

// start the server
const PORT = parseInt(process.env.PORT) || 8080;
app.listen(PORT, () => {
    console.log(\`App listening on port \${PORT}\`);
    console.log('Press Ctrl+C to quit.');
});

module.exports = app;
EOF

# create app.yaml file
cat > $FRONTEND_DIR/app.yaml << EOF
runtime: nodejs
env: flex
runtime_config:
  operating_system: "ubuntu22"
  runtime_version: "20"
manual_scaling:
  instances: 1
resources:
  cpu: 1
  memory_gb: 0.5
  disk_size_gb: 10
vpc_access_connector:
  name: projects/$PROJECT_ID/locations/$GCP_REGION/connectors/$CONNECTOR_NAME
env_variables:
  BACKEND_URL: $BACKEND_URL
EOF

# deploy frontend app
gcloud app deploy $FRONTEND_DIR/app.yaml

# test
gcloud app browse   # should return JSON {"date": "YYYY-MM-DDTHH:MM:ssZ"}

Serverless apps interacting over Google private network

This example illustrates how to leverage Serverless VPC Connector and Private Google Access so a Google App Engine (GAE) service will interact with a Cloud Run service, despite it's xxxxxxx.a.run.app URL, only via Google's private network.

Architecture

Result

Requests to the GAE app running a NodeJS Express app that simply sends a request to the backend and then returns its results.

The setup

After deploying the Cloud Run service (backend app called cr-datetime that simply returns the current UTC date and time in ISO-8601 format), it was not accessible via the xxxxxxxx.a.run.app URL as illustrated.

Serverless VPC Connector

After adding a subnet with the --enable-private-ip-google-access flag enabled, a VPC Connector was created referencing this subnet and project (instead of allowing one to be auto-generated). The subnet and network firewall rules allowed traffic from Google's IP ranges.

Cloud DNS

Private DNS zones were created for googleapis.com and app.run with wildcard CNAME aliases pointing to them respectively. Given the VPC Connector compute runtime was created in the project, the VMs resolved the internal DNS first for the Google private access ranges.

GAE app request to Cloud Run URL routed privately

The GAE app was deployed with an environment variable BACKEND_URL that was generated from the URL of the deployed Cloud Run app. This URL itself was not accessible publicly given the ingress configuration restricted to only internal and load balancing traffic.