Terraform AWS discovery using dtan4/terraforming

collapse_state_files.js

#!/usr/bin/env node

const fs = require('fs');
const path = require('path');
const deepmerge = require('deepmerge');

if (process.argv.length <= 2) {
  console.log("Usage: " + __filename + " path/to/directory");
  process.exit(-1);
}

const dir = process.argv[2];
const tfstate_file = path.join(dir, 'terraform.tfstate');

fs.readdir(dir, function (err, items) {
  if (err) { console.error(err.message); process.exit(1); }

  const files_to_collapse = items.filter((item) => item !== 'terraform.tfstate' && item.endsWith('.tfstate'));
  console.log('Collapsing the following state files into the terraform.tfstate file:');
  console.log('  ' + files_to_collapse.join('\n  ') + '\n');

  const tfstate = JSON.parse(fs.readFileSync(tfstate_file));

  let module_state;

  files_to_collapse.forEach((f) => {
    module_state = JSON.parse(fs.readFileSync(path.normalize(path.join(dir, f))));
    // NOTE: merges assume that all files have a single module defined at path: root (likely true unless you have a large or complex landscape)
    tfstate.modules[0].outputs = deepmerge(tfstate.modules[0].outputs, module_state.modules[0].outputs);
    tfstate.modules[0].resources = deepmerge(tfstate.modules[0].resources, module_state.modules[0].resources);
  });

  console.log(JSON.stringify(tfstate, null, 2));

  fs.writeFileSync(tfstate_file, JSON.stringify(tfstate, null, 2), 'utf8');
});

get_terraform_diff.sh

#!/bin/bash

env_ok=1
if [[ ! -n ${AWS_ACCESS_KEY_ID} ]]; then
  echo "No AWS_ACCESS_KEY_ID variable found in environment"
  env_ok=0
fi
if [[ ! -n ${AWS_SECRET_ACCESS_KEY} ]]; then
  echo "No AWS_SECRET_ACCESS_KEY variable found in environment"
  env_ok=0
fi
if [[ ! -n ${AWS_DEFAULT_REGION} ]]; then
  echo "No AWS_DEFAULT_REGION variable found in environment"
  env_ok=0
fi
if [ $env_ok -eq 0 ]; then
  exit 1
fi

RUN_DIR="$(pwd)"
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd)"

if [[ ! -f "${SCRIPT_DIR}/terraform/terraform.tfstate" ]]; then
  echo "No terraform.tfstate file found at ${SCRIPT_DIR}/terraform/terraform.tfstate"
  exit 1
fi

cd "${SCRIPT_DIR}/terraform" && \
  terraform plan -refresh=true -no-color -state="./terraform.tfstate" -out="./terraform.tfplan" > "./terraform-plan-diff.txt" && \
  cd "${RUN_DIR}"

run_terraforming_container.sh

#!/bin/bash

cmd=$1

env_ok=1
if [[ ! -n ${AWS_ACCESS_KEY_ID} ]]; then
  echo "No AWS_ACCESS_KEY_ID variable found in environment"
  env_ok=0
fi
if [[ ! -n ${AWS_SECRET_ACCESS_KEY} ]]; then
  echo "No AWS_SECRET_ACCESS_KEY variable found in environment"
  env_ok=0
fi
if [[ ! -n ${AWS_DEFAULT_REGION} ]]; then
  echo "No AWS_DEFAULT_REGION variable found in environment"
  env_ok=0
fi
if [ $env_ok -eq 0 ]; then
  exit 1
fi

docker pull quay.io/dtan4/terraforming:latest && \
docker run --rm --name terraforming \
  -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
  -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
  -e AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} \
  quay.io/dtan4/terraforming:latest \
  terraforming ${cmd} > ./aws-${cmd}.tf
docker run --rm --name terraforming \
  -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
  -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
  -e AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} \
  quay.io/dtan4/terraforming:latest \
  terraforming ${cmd} --tfstate > ./aws-${cmd}-tfstate.json

Setup Terraform

First, download Terraform and install per these instructions.

Once you have Terraform ready, you'll want to create and initialize the folder to store the Terraform files. The following assumes you are in the folder where these scripts live:

mkdir terraform
cd terraform
terraform init

This last command will download the AWS Provider plugin and create a .terraform folder within the terraform one you just created. (This is much like git does with its .git folder.)

Ensure you have docker installed and working

You can install Docker Desktop by following the instructions here..

To test your installation, you can use:

docker run hello-world

Retrieve current state in AWS as Terraform code

All files assume that Terraform code and state files will live in a terraform folder that lives in the same directory as these scripts. In addition, for resource discovery, you'll need the following environment variables defined (per standard AWS credentials setup):

• AWS_ACCESS_KEY_ID
• AWS_SECRET_ACCESS_KEY
• AWS_DEFAULT_REGION

The following sequence of steps can be used to pull current state:

1. Run ./run_terraforming_container.sh $servicename for each AWS service you care about
2. Create an "empty" terraform.tfstate file (not really empty -- see Terraform docs) in the terraform folder within this repo (alongside the generated files)
3. Collapse service-level .tfstate files into terraform.tfstate by running ./collapse_tfstate_files.js ./terraform
4. Generate the diff of the tf files and the existing state in AWS via ./get_terraform_diff.sh

DISCLAIMER: This was created for my own purposes and may or may not work for you. Feel free to use it, and I hope it can help save you some time, but I make no claims about suitability for your intended work and no guarantees that this will do anything specific. Use at your own risk.