trino-dev-cheat-sheet.md

Source: https://gist.github.com/findepi/04c96f0f60dcc95329f569bb0c44a0cd .

clean build

This should build the project. In case of problems, retry without the -T flag. This is not the fastest way to build the project though. See below.

./mvnw -T1C clean install -DskipTests 

use mvnd for development

When developing, mvnd is recommended, for faster feedback loops. You can install it with

# the v2 is not stable yet, hence @1 version pinning
brew install mvndaemon/homebrew-mvnd/mvnd@1

If you don't want to use mvnd, just replace it with ./mvnw -T1C.

quick clean build

mvnd clean install -nsu -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Dair.check.skip-all=true -pl '!:trino-docs,!:trino-proxy,!:trino-verifier,!:trino-benchto-benchmarks'

for even quicker incremental build, omit the clean

run static code analysis

# checkstyle only
mvnd checkstyle:check@checkstyle

# all verifications, including Error Prone compiler
# clean is needed to force recompilation with Error Prone compiler. 
# Re-run the second command with `-r` param to resume incremental build e.g. after fixing an error.
mvnd clean &&
mvnd test-compile install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Dair.check.skip-all=true -Dskip.npm -Dskip.yarn -nsu -P errorprone-compiler -pl '!:trino-server,!:trino-docs'

code cleanups

mvnd license:format sortpom:sort
# I wish imports and formatting could be fixed from a CLI too

quick build antlr grammar alone

mvnd antlr4:antlr4 -pl :trino-grammar -nsu

# or all the grammars in the project
mvnd antlr4:antlr4 -nsu -pl "$(find . -name \*.g4 | sed -e 's@/src/.*@@' -e 's@.*/@:@' -e 's@$@,@' | sort -u)"

run Trino in a container, quickly

docker rm -f trino; docker run --rm -it --name trino -p 8080:8080 trinodb/trino

kill and remove all docker containers

docker ps -a | tail -n+2 | grep . && docker ps -aq | xargs docker rm -f ; docker volume prune -f

docs

docs/build

extract exception stacktrace from query JSON

# Just top-level exception
jq -r ' .failureInfo | [ .type + ": " + .message, (.stack[] | "    at " + .) ][] ' < query.json

# Full stacktrace
python -c '
import sys
import json
def print_exc(exc, enclosing_stacktrace=(), caption="", prefix=""):
    print("%s%s%s: %s" % (prefix, caption, exc["type"], exc["message"]))
    if "errorCode" in exc:
        print("%s\t\tcode: %s, name: %s" % (prefix, exc["errorCode"]["code"],
            exc["errorCode"]["name"]))
    stack_trace = exc["stack"]
    our_stack = unique_stacktrace(stack_trace, enclosing_stacktrace)
    for l in our_stack:
        print("%s\tat %s" % (prefix, l))
    if len(our_stack) < len(stack_trace):
        print("%s\t..." % (prefix, ))
    for s in exc["suppressed"]:
        print_exc(s, stack_trace, "Suppressed: ", prefix + "\t")
    if "cause" in exc:
        print_exc(exc["cause"], stack_trace, "Caused by: ", prefix)
def unique_stacktrace(stack, enclosing):
    stack = list(stack)
    enclosing = list(enclosing)
    while stack and enclosing and stack[-1] == enclosing[-1]:
        stack.pop()
        enclosing.pop()
    return stack
qj = json.load(sys.stdin)
print_exc(qj["failureInfo"])
' \
 < query.json

restore Product Test Launcher (ptl) removed by a very important person

# this is required for commands show below
git restore --source=$(git log -1 --format=tformat:%H -- bin/ptl)^ -- bin/ptl

product test env just as product tests do

bin/ptl env up --environment singlenode

postgresql

bin/ptl env up --environment singlenode-postgresql --without-trino
# docker exec -itu postgres ptl-postgresql psql -U test -d test

sqlserver

bin/ptl env up --environment singlenode-sqlserver --without-trino
# docker exec -it ptl-sqlserver /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P SQLServerPass1

mysql

bin/ptl env up --environment singlenode-mysql --without-trino
# docker exec -it ptl-mysql mysql -u test -ptest -D test

git bisect with plain test

git bisect run bash -xeuc '
  ./mvnw clean install -am -pl :trino-tests -T2C -DskipTests \
    -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Dair.check.skip-all=true || exit 125
  ./mvnw test -pl :trino-tests -Dair.check.skip-all=true -Dtest=TestLocalQueriesA#test
'

git bisect with product tests

git bisect run bash -xeuc '
  ./mvnw clean install -am -pl \!:trino-docs -DskipTests \
    -Dair.check.skip-all=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -T2C || exit 125
  bin/ptl test run --environment singlenode -- -g tpcds -x quarantine -t q98
'

run product test quick

# TODO update this
#   java -jar presto-product-tests/target/presto-product-tests-*-executable.jar -c presto-product-tests/src/main/resources/sql-tests -g tpch -t q17

bring up product test env

# restore Product Test Launcher (ptl) removed by a very important person
git restore --source=$(git log -1 --format=tformat:%H -- bin/ptl)^ -- bin/ptl

# get a list of envs
bin/ptl env list

# start an env of your choice, exposing JVM debug port for Trino
bin/ptl env up --environment singlenode-hdp3 --debug

# run Trino CLI against the env
client/trino-cli/target/trino-cli-*-executable.jar --debug --server localhost:8080

# run Hive beeline 
docker exec -itu hive ptl-hadoop-master bash -l
echo "press Ctrl+R for bash history search and type 'beeline'"

# run Spark SQL (when dealing with an environment containing Spark image)
docker exec -it ptl-spark spark-sql
# ... and starting in some specific catalog
docker exec -it ptl-spark spark-sql -- --conf spark.sql.defaultCatalog=iceberg_test

check all commits compile

git rebase master -x './mvnw clean package -pl \!:trino-docs -T2C -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true'

sort commits for GitHub

git rebase master -x 'git commit --amend -C HEAD --date="$(date -R)" && sleep 1.05'

rebase commits within PR

This makes GitHub's diff for force-push useful.

git rebase -i master --keep-base

run benchmark from CLI

./mvnw exec:exec -pl :trino-main -Dexec.classpathScope=test -Dexec.executable="java" \
    -Dexec.args="-cp %classpath io.trino.operator.output.BenchmarkPartitionedOutputOperator"

network grepping

sudo ngrep -s0 -W byline -d lo0

AI

Cloud Code on the Web

Cloud Code on the Web's secure environment does not work out of the box.

Network access

Custom + add allowed domains

• repo.maven.apache.org

Environment variables

MAVEN_ARGS=-s /root/.maven.settings.xml -T1C
JAVA_HOME=/usr/lib/jvm/jdk-25.0.2-oracle-x64

Setup script

#!/bin/bash
set -eu

# Install system dependencies
export DEBIAN_FRONTEND=noninteractive
apt update
apt install -y curl wget

# Install JDK 25
wget https://download.oracle.com/java/25/latest/jdk-25_linux-x64_bin.deb
dpkg -i jdk-25_linux-x64_bin.deb
export JAVA_HOME=/usr/lib/jvm/jdk-25.0.2-oracle-x64

# Install some system cert into JDK to allow forward proxy (MITM)
keytool -importcert -trustcacerts \
  -keystore $JAVA_HOME/lib/security/cacerts \
  -storepass changeit -noprompt \
  -alias swp-ca-production \
  -file /usr/local/share/ca-certificates/swp-ca-production.crt

# Extract forward proxy configuration
extract() {
  local prop="$1"
  # Match -Dprop=value where value is a non-space token
  echo "$JAVA_TOOL_OPTIONS" | grep -oP "(?<=-D${prop}=)\S+" || echo ""
}
HTTP_HOST=$(extract http.proxyHost)
HTTP_PORT=$(extract http.proxyPort)
HTTP_USER=$(extract http.proxyUser)
HTTP_PASS=$(extract http.proxyPassword)
HTTPS_HOST=$(extract https.proxyHost)
HTTPS_PORT=$(extract https.proxyPort)
HTTPS_USER=$(extract https.proxyUser)
HTTPS_PASS=$(extract https.proxyPassword)
NON_PROXY=$(extract http.nonProxyHosts)

# Configure Maven to use proxy
cat > "$HOME/.maven.settings.xml" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.2.0 https://maven.apache.org/xsd/settings-1.2.0.xsd">
  <proxies>
    <proxy>
      <id>http-proxy</id>
      <active>true</active>
      <protocol>http</protocol>
      <host>${HTTP_HOST}</host>
      <port>${HTTP_PORT}</port>
      <username>${HTTP_USER}</username>
      <password>${HTTP_PASS}</password>
      <nonProxyHosts>${NON_PROXY}</nonProxyHosts>
    </proxy>
    <proxy>
      <id>https-proxy</id>
      <active>true</active>
      <protocol>https</protocol>
      <host>${HTTPS_HOST}</host>
      <port>${HTTPS_PORT}</port>
      <username>${HTTPS_USER}</username>
      <password>${HTTPS_PASS}</password>
      <nonProxyHosts>${NON_PROXY}</nonProxyHosts>
    </proxy>
  </proxies>
</settings>
EOF

# Configure docker pull to use proxy
mkdir -p "$HOME/.docker"
cat > "$HOME/.docker/config.json" << EOF
{
  "proxies": {
    "default": {
      "httpProxy": "$HTTP_HOST",
      "httpsProxy": "$HTTP_HOST",
      "noProxy": "localhost,127.0.0.1"
    }
  }
}
EOF

# Docker is installed but not started by default
HTTP_PROXY="$HTTP_PROXY" HTTPS_PROXY="$HTTP_PROXY" dockerd &

echo Setup script all OK

@losipiuk that's awesome!