run_airbyte_images.sh

#!/usr/bin/env bash

set -euo pipefail
set -x

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${ROOT_DIR}"

SOURCE_CONFIG_URL="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors/source-file/integration_tests/config.json"
SOURCE_IMAGE_NAME="airbyte/source-file"
DESTINATION_IMAGE_NAME="airbyte/destination-local-json"

docker pull "${SOURCE_IMAGE_NAME}"
docker pull "${DESTINATION_IMAGE_NAME}"

# Prepare source
docker run -v "${PWD}:${PWD}" "${SOURCE_IMAGE_NAME}" spec | jq . > "${PWD}/source-spec.json"
# curl -s -o source-config.json "${SOURCE_CONFIG_URL}"
echo '{
  "dataset_name": "integrationTestFile",
  "format": "csv",
  "reader_options": "{\"sep\": \",\", \"nrows\": 20}",
  "url": "https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv",
  "provider": {
    "storage": "HTTPS",
    "reader_impl": "gcsfs"
  }
}
' > "${PWD}/source-config.json"

docker run -v "${PWD}:${PWD}" "${SOURCE_IMAGE_NAME}" check --config "${PWD}/source-config.json"

# Prepare catalog
docker run -v "${PWD}:${PWD}" "${SOURCE_IMAGE_NAME}" discover --config "${PWD}/source-config.json" > "${PWD}/catalog-raw.json"
cat "${PWD}/catalog-raw.json" | jq -s '
{
    streams: (
        (.[1].catalog.streams| map({
            stream: .,sync_mode:"full_refresh", destination_sync_mode:"overwrite" 
        }))     
    )
}' > "${PWD}/catalog.json"

# Prepare destination
docker run -v "${PWD}:${PWD}" "${DESTINATION_IMAGE_NAME}" spec | jq  -R "fromjson?" | jq . > "${PWD}/destination-spec.json"
echo '{"destination_path": "/local/files/"}' > "${PWD}/destination-config.json" 
docker run -v "${PWD}:${PWD}" "${DESTINATION_IMAGE_NAME}" check --config "${PWD}/destination-config.json" 

# Transfer data
docker run -v "${PWD}:${PWD}" "${SOURCE_IMAGE_NAME}" read --config "${PWD}/source-config.json" --catalog "${PWD}/catalog.json" > data.json
(sleep 1; cat data.json) | docker run -v "${PWD}:${PWD}" -v "${PWD}/local/:/local/" "${DESTINATION_IMAGE_NAME}" write --config "${PWD}/destination-config.json" --catalog "${PWD}/catalog.json"