Convert GeoPackage to MBTiles with specified zoom levels

gpkg2mbtiles.sh

#!/bin/bash

# gpkg2mbtiles.sh - Convert GeoPackage to MBTiles with specified zoom levels
#
# Usage: ./gpkg2mbtiles.sh -Z <min_zoom> -z <max_zoom> <input_file>
# Example: ./gpkg2mbtiles.sh -Z 14 -z 17 wa-parcels
# 
# Created by: Josh Campbell - ThingEngineer
# Date: 2025-05-01

# ANSI color codes
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
PURPLE='\033[0;35m'
NC='\033[0m' # No Color

# Function to format seconds to minutes:seconds
format_time() {
    local seconds=$1
    local minutes=$((seconds / 60))
    local remaining_seconds=$((seconds % 60))
    printf "%d:%02d" $minutes $remaining_seconds
}

# Function to print separator
print_separator() {
    echo -e "\n${BLUE}=====================================================================================================${NC}\n"
}

# Function to print command and then execute it
run_command() {
    local cmd="$1"
    echo -e "${GREEN}Running command:${NC} ${cmd}"
    eval $cmd
    
    if [ $? -ne 0 ]; then
        echo -e "\n${RED}Error executing command:${NC} ${cmd}"
        exit 1
    fi
}

# Check if required arguments are provided
if [ "$#" -lt 3 ]; then
    echo -e "${RED}Usage: $0 -Z <min_zoom> -z <max_zoom> <input_file>${NC}"
    echo -e "${YELLOW}Example: $0 -Z 14 -z 17 wa-parcels${NC}"
    exit 1
fi

# Parse arguments
min_zoom=""
max_zoom=""

while getopts "Z:z:" opt; do
    case $opt in
        Z)
            min_zoom=$OPTARG
            ;;
        z)
            max_zoom=$OPTARG
            ;;
        \?)
            echo -e "${RED}Invalid option: -$OPTARG${NC}" >&2
            exit 1
            ;;
        :)
            echo -e "${RED}Option -$OPTARG requires an argument.${NC}" >&2
            exit 1
            ;;
    esac
done

# Get the input file (last argument)
shift $((OPTIND-1))
if [ "$#" -ne 1 ]; then
    echo -e "${RED}Please provide an input file name (without extension).${NC}"
    exit 1
fi

input_file="$1"

# Check if input file exists
if [ ! -f "${input_file}.gpkg" ]; then
    echo -e "${RED}Input file ${input_file}.gpkg not found.${NC}"
    exit 1
fi

# Validate zoom levels
if [[ ! $min_zoom =~ ^[0-9]+$ ]] || [[ ! $max_zoom =~ ^[0-9]+$ ]]; then
    echo -e "${RED}Zoom levels must be positive integers.${NC}"
    exit 1
fi

if [ "$min_zoom" -gt "$max_zoom" ]; then
    echo -e "${RED}Minimum zoom level (-Z) cannot be greater than maximum zoom level (-z).${NC}"
    exit 1
fi

# Define output files
reprojected_file="${input_file}-reprojected.gpkg"
valid_file="${input_file}-valid.gpkg"
fgb_file="${input_file}.fgb"
mbtiles_file="${input_file}-${min_zoom}-${max_zoom}.mbtiles"

# Start timer
start_time=$(date +%s)
last_time=$start_time

echo -e "${YELLOW}Starting GeoPackage to MBTiles conversion workflow for ${input_file}.gpkg${NC}"
echo -e "${YELLOW}Min zoom: ${min_zoom}, Max zoom: ${max_zoom}${NC}"
print_separator

# Step 1: Check the input GPKG file
echo -e "${PURPLE}Step 1: Checking input GeoPackage file...${NC}"
run_command "ogrinfo -al -so ${input_file}.gpkg"

step_time=$(date +%s)
elapsed_seconds=$((step_time - last_time))
formatted_time=$(format_time $elapsed_seconds)
echo -e "${YELLOW}Time elapsed: ${formatted_time}${NC}"
last_time=$step_time
print_separator

# Step 2: Reproject the GPKG file if needed
echo -e "${PURPLE}Step 2: Reprojecting GeoPackage to EPSG:4326...${NC}"
run_command "ogr2ogr -f \"GPKG\" -t_srs EPSG:4326 ${reprojected_file} ${input_file}.gpkg"

step_time=$(date +%s)
elapsed_seconds=$((step_time - last_time))
formatted_time=$(format_time $elapsed_seconds)
echo -e "${YELLOW}Time elapsed: ${formatted_time}${NC}"
last_time=$step_time
print_separator

# Step 3: Check the reprojected GPKG file
echo -e "${PURPLE}Step 3: Checking reprojected GeoPackage file...${NC}"
run_command "ogrinfo -al -so ${reprojected_file}"

step_time=$(date +%s)
elapsed_seconds=$((step_time - last_time))
formatted_time=$(format_time $elapsed_seconds)
echo -e "${YELLOW}Time elapsed: ${formatted_time}${NC}"
last_time=$step_time
print_separator

# Step 4: Validate geometries
echo -e "${PURPLE}Step 4: Validating geometries...${NC}"
run_command "ogr2ogr -f \"GPKG\" ${valid_file} ${reprojected_file} -makevalid"

step_time=$(date +%s)
elapsed_seconds=$((step_time - last_time))
formatted_time=$(format_time $elapsed_seconds)
echo -e "${YELLOW}Time elapsed: ${formatted_time}${NC}"
last_time=$step_time
print_separator

# Step 5: Check the validated GPKG file
echo -e "${PURPLE}Step 5: Checking validated GeoPackage file...${NC}"
run_command "ogrinfo -al -so ${valid_file}"

step_time=$(date +%s)
elapsed_seconds=$((step_time - last_time))
formatted_time=$(format_time $elapsed_seconds)
echo -e "${YELLOW}Time elapsed: ${formatted_time}${NC}"
last_time=$step_time
print_separator

# Step 6: Convert the GPKG to FlatGeoBuf
echo -e "${PURPLE}Step 6: Converting to FlatGeobuf format...${NC}"
run_command "ogr2ogr -f \"FlatGeobuf\" ${fgb_file} ${valid_file}"

step_time=$(date +%s)
elapsed_seconds=$((step_time - last_time))
formatted_time=$(format_time $elapsed_seconds)
echo -e "${YELLOW}Time elapsed: ${formatted_time}${NC}"
last_time=$step_time
print_separator

# Step 7: Check the FGB file
echo -e "${PURPLE}Step 7: Checking FlatGeobuf file...${NC}"
run_command "ogrinfo -al -so ${fgb_file}"

step_time=$(date +%s)
elapsed_seconds=$((step_time - last_time))
formatted_time=$(format_time $elapsed_seconds)
echo -e "${YELLOW}Time elapsed: ${formatted_time}${NC}"
last_time=$step_time
print_separator

# Step 8: Convert to MBTiles using Tippecanoe
echo -e "${PURPLE}Step 8: Converting to MBTiles using Tippecanoe...${NC}"
tippecanoe_cmd="tippecanoe -o ${mbtiles_file} -f -Z${min_zoom} -z${max_zoom} --read-parallel --hilbert --simplify-only-low-zooms --maximum-zoom=${max_zoom} --minimum-zoom=${min_zoom} ${fgb_file}"
run_command "$tippecanoe_cmd"

step_time=$(date +%s)
elapsed_seconds=$((step_time - last_time))
formatted_time=$(format_time $elapsed_seconds)
echo -e "${YELLOW}Time elapsed: ${formatted_time}${NC}"
last_time=$step_time
print_separator

# Step 9: Check the MBTiles file
echo -e "${PURPLE}Step 9: Checking MBTiles file...${NC}"
run_command "ogrinfo -al -so -q ${mbtiles_file}"

step_time=$(date +%s)
elapsed_seconds=$((step_time - last_time))
formatted_time=$(format_time $elapsed_seconds)
echo -e "${YELLOW}Time elapsed: ${formatted_time}${NC}"
print_separator

# Calculate total time
end_time=$(date +%s)
total_time=$((end_time - start_time))
formatted_total_time=$(format_time $total_time)

echo -e "${GREEN}Conversion completed successfully!${NC}"
echo -e "${GREEN}Total time: ${formatted_total_time}${NC}"
echo -e "${GREEN}Output file: ${mbtiles_file}${NC}"

# Prompt to copy file via SCP
read -p "Copy the MBTiles file to the tile server? (y/n): " copy_response
if [[ "$copy_response" =~ ^[Yy]$ ]]; then
    scp_cmd="scp ${mbtiles_file} [email protected]://home/docker/tileserver-gl-docker-compose/data"
    echo -e "${GREEN}Running command:${NC} ${scp_cmd}"
    eval $scp_cmd
    
    if [ $? -eq 0 ]; then
        echo -e "${GREEN}File copied successfully!${NC}"
    else
        echo -e "${RED}Error copying file.${NC}"
    fi
fi

echo -e "${GREEN}Script completed.${NC}"

System Prerequisites and Dependencies for gpkg2mbtiles.sh

The following system prerequisites and dependencies are required to ensure the successful execution of the gpkg2mbtiles.sh script, which converts GeoPackage (.gpkg) files to MBTiles (.mbtiles) format.

---

1. System Requirements

• Operating System: The script is designed for Linux-based environments. It should work on macOS with minor adjustments.
• Shell: The script is written in Bash and requires a Bash-compatible shell.

---

2. Software Dependencies

The following software tools must be installed and available in the system's PATH:

2.1. GDAL (Geospatial Data Abstraction Library)

• Commands Used: ogrinfo, ogr2ogr
• Purpose:
  • ogrinfo: To inspect GeoPackage, FlatGeobuf, and MBTiles files for metadata and errors.
  • ogr2ogr: To reproject GeoPackage files to the EPSG:4326 coordinate system, validate geometries, and convert between formats.
• Minimum Version: 3.1.0 (required for FlatGeobuf support)
• Installation:
  • On Ubuntu/Debian: sudo apt-get install gdal-bin
  • On macOS: brew install gdal

2.2. Tippecanoe

• Command Used: tippecanoe
• Purpose: Converts FlatGeobuf files to MBTiles format, generating vector tiles.
• Minimum Version: 1.34.3 (for the options used in the script, such as --hilbert and --read-parallel)
• Installation:
  • From source: Tippecanoe GitHub Repository

2.3. OpenSSH Client

• Command Used: scp
• Purpose: Copies the resulting MBTiles file to a remote server using SSH.
• Installation:
  • On Ubuntu/Debian: sudo apt-get install openssh-client
  • On macOS: Pre-installed

---

3. Additional Requirements

3.1. Network Access

• If the script's final step is enabled (scp), ensure:
  • The destination server is accessible (e.g., [email protected]).
  • The user has appropriate SSH access to the remote server.

3.2. File Permissions

• The script must have executable permissions.

  chmod +x gpkg2mbtiles.sh

3.3. Input File

• The input file must:
  • Be a valid GeoPackage (.gpkg) file.
  • Be located in the same directory as the script or provide an appropriate path.

---

4. Optional Tools

These tools are not required but can enhance the workflow:

4.1. time Command

• Purpose: Measures the overall execution time of the script.
• Installation:
  • On Ubuntu/Debian: Pre-installed
  • On macOS: Pre-installed

---

5. Environment Configuration

5.1. Add Tools to PATH

Ensure all required tools (ogrinfo, ogr2ogr, tippecanoe, scp) are in the system's PATH. You can check this by running:

which ogrinfo ogr2ogr tippecanoe scp

5.2. Test Environment

Run the following commands to verify dependencies:

ogrinfo --version
ogr2ogr --version
tippecanoe --version
scp -V

---

6. Troubleshooting

6.1. Missing Dependencies

If a command is not found, ensure the corresponding software is installed and available in the system's PATH.

6.2. Permission Issues

Ensure the input file and output directory have the appropriate read/write permissions.

6.3. Tippecanoe Memory Usage

Generating MBTiles for large datasets may require significant memory. Monitor system resources during execution.

---

7. Example Environment Setup

Below is an example of setting up the required dependencies on an Ubuntu system:

# Install GDAL
sudo apt-get update
sudo apt-get install -y gdal-bin

# Install Tippecanoe
git clone https://github.com/mapbox/tippecanoe.git
cd tippecanoe
make -j
sudo make install

# Verify installation
ogrinfo --version
ogr2ogr --version
tippecanoe --version
scp -V

---

By ensuring these prerequisites and dependencies are met, the gpkg2mbtiles.sh script will execute seamlessly.