Installation script for Mail Piler (open source version) on Ubuntu 20.04 Focal

focal-master.sh

#!/bin/bash
#
#
# This script installs the latest open source version of Mail Piler (mailpiler.org) from the master
# branch on Bitbucket by compiling it from source. It also installs all dependencies, including a MySQL database.
#
# You should run this script as root on a vanilla Ubuntu 20.04 installation.
#
#
set -o errexit
set -o pipefail
set -o nounset
set -x

PILER_HOSTNAME="${PILER_HOSTNAME:-archive.yourdomain.com}"
MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
SERVER_ID="${SERVER_ID:-0}"
USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock"

MYSQL_HOSTNAME="localhost"
MYSQL_DATABASE="piler"
MYSQL_USERNAME="piler"

SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz"
DOWNLOAD_URL="https://download.mailpiler.com"
PILER_TARBALL="https://bitbucket.org/jsuto/piler/get/master.tar.gz"
PILER_USER="piler"
CONFIG_SITE_PHP="/etc/piler/config-site.php"
CONFIG_SITE_DIST_PHP="/etc/piler/config-site.dist.php"

export DEBIAN_FRONTEND=noninteractive

install_prerequisites() {
   apt-get update
   apt-get -y --no-install-recommends install \
      wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
      php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \
      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \
      mariadb-server-10.3 build-essential libssl-dev libtre-dev libzip-dev

   wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
   tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"
}

create_user() {
   egrep -i "^x$PILER_USER:" /etc/passwd || adduser --no-create-home --disabled-password --disabled-login --gecos "" $PILER_USER
}

create_mysql_user_and_database() {
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "CREATE USER IF NOT EXISTS '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "SET PASSWORD FOR '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME' = PASSWORD('$MYSQL_PILER_PASSWORD');"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE CHARACTER SET 'utf8mb4';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "GRANT ALL PRIVILEGES ON $MYSQL_DATABASE.* to '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME' IDENTIFIED BY '$MYSQL_PILER_PASSWORD';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "FLUSH PRIVILEGES;"

   mysql -u $MYSQL_USERNAME -p"$MYSQL_PILER_PASSWORD" $MYSQL_DATABASE < /etc/piler/db-mysql.sql
}

fix_mysql_settings() {
   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
[mysqld]

innodb_buffer_pool_size=512M
innodb_flush_log_at_trx_commit=1
innodb_log_buffer_size=64M
innodb_log_file_size=64M
innodb_read_io_threads=4
innodb_write_io_threads=4
innodb_log_files_in_group=2

innodb_file_per_table
PILER_CNF
}

start_mysql() {
   fix_mysql_settings
   service mysql restart
}

install_piler() {
   wget "${PILER_TARBALL}" -O "/tmp/piler.tar.gz"
   tar -zxvf "/tmp/piler.tar.gz" -C /tmp
   pushd /tmp/jsuto-piler-*
   ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb --enable-tcpwrappers --enable-memcached
   make clean all install
   cp ./contrib/webserver/piler-nginx.conf /etc/piler/piler-nginx.conf.dist
   cp ./etc/sphinx.conf.dist /etc/piler/sphinx.conf
   cp ./util/db-mysql.sql /etc/piler/db-mysql.sql
   popd

   crontab -u "$PILER_USER" /usr/share/piler/piler.cron

   touch /var/piler/.bash_history
   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
}

create_my_cnf() {
   local user=$1
   local password=$2
   local my_cnf=$3

   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
   chown $PILER_USER:$PILER_USER "$my_cnf"
   chmod 600 "$my_cnf"
}


fix_config_site_php() {
   cp $CONFIG_SITE_DIST_PHP $CONFIG_SITE_PHP
   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"

   {
      echo "\$config['SERVER_ID'] = $SERVER_ID;"
      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
      echo "\$config['SPHINX_VERSION'] = 331;"
   } >> "$CONFIG_SITE_PHP"

   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
   fi

   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
}


add_systemd_services() {
   pushd /etc/systemd/system

   ln -sf /usr/libexec/piler/piler.service .
   ln -sf /usr/libexec/piler/piler-smtp.service .
   ln -sf /usr/libexec/piler/pilersearch.service .

   popd

   systemctl daemon-reload

   systemctl enable piler
   systemctl enable piler-smtp
   systemctl enable pilersearch
}

create_cipher_key() {
   dd if=/dev/urandom bs=56 count=1 of=/etc/piler/piler.key
   chmod 640 /etc/piler/piler.key
   chown piler:piler /etc/piler/piler.key
}


fix_configs() {
   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
      ln -s /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf

      nginx -t
      nginx -s reload
   fi

   if [[ ! -f /etc/piler/piler.conf ]]; then
      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.yourdomain.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
      chmod 600 /etc/piler/piler.conf
      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
   fi

   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
          /etc/piler/sphinx.conf
}

install_prerequisites
create_user
install_piler
create_mysql_user_and_database
start_mysql

create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf

fix_configs
fix_config_site_php

add_systemd_services
create_cipher_key

su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER

[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler

systemctl start pilersearch
systemctl start piler
systemctl start piler-smtp

For Anyone coming here in the future, here is what you would need to do to use manticore instead of sphinx:

Remove these lines from script above:

wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"

Assuming Ubuntu 22.04: install manticore:
wget https://repo.manticoresearch.com/manticore-repo.noarch.deb
sudo dpkg -i manticore-repo.noarch.deb
sudo apt update
sudo apt install manticore manticore-extra

Change line 172 from /etc/piler/sphinx.conf to /etc/piler/manticore.conf

run this command to let piler know to use manticore: touch /etc/piler/MANTICORE

That should be all that's needed. for reference, also check out this link : https://www.mailpiler.org/manticore-search/

Hope this helps someone

Anyone further in the future I updated this script for Ubuntu 24.04, Piler install from the new GitHib location, and PHP 8.3.
https://gist.github.com/gkwurst/87d240cf518cdbea8a78403c51f14885

Thanks so much for this write up. For me it worked right out of the box. This is one of the very few scripts that have automagically worked the first time for me without any modifications/troubleshooting (except of course for the personalized custom entries).

The challenge that I am facing regards Piler configuration, specifically, the creation of LDAP authenticated Auditor. I am able to authenticate users via LDAP/AD successfully, but I am unable to make any of these LDAP users have Auditor Piler privileges.

I have followed the instructions in the Piler documentation that says to create dedicated a PilerAuditor group in LDAP, and put specific users who should belong to Piler Auditors in that LDAP Group. I have properly defined this group's LDAP DN in the config and queried it and it returns positive results. But after these steps those users fails to get assigned Auditor privileges when they log in to Piler.

I have also additionally tried to manually map these LDAP Auditor/Admin groups in Piler GUI but it doesn't work. Also trying to add these users in Piler and defining the LDAP groups fails.

I don't know whether I am doing this correctly, or whether this function works in the opensource version of Piler.