[DEPRECATED] [Now called mussel] Building an x86-64 cross compiler targeting musl libc

generate-toolchain.sh

#!/usr/bin/dash -e

# Copyright (c) 2020, Firas Khalil Khana
# Distributed under the terms of the ISC License

#
# This script is now officialy known as mussel and is maintained by
# firasuke over at: https://github.com/firasuke/mussel
#

#
# Credits to Aurelian, qword and managram for making this possible
#

set -e
umask 022
unalias -a

#
# Colors
#
BLUEC='\033[1;34m'
REDC='\033[1;31m'
GREENC='\033[1;32m'
NORMALC='\033[0m'

#
# Package Versions
#
binutils_ver=2.34
gcc_ver=10.1.0
gmp_ver=6.2.0
isl_ver=0.22.1
mpc_ver=1.1.0
mpfr_ver=4.0.2
musl_ver=1.2.0

#
# Package URLs (The usage of ftpmirror for GNU packages is preferred.)
#
binutils_url=https://ftpmirror.gnu.org/binutils/binutils-$binutils_ver.tar.lz
gcc_url=https://ftpmirror.gnu.org/gcc/gcc-$gcc_ver/gcc-$gcc_ver.tar.xz
gmp_url=https://ftpmirror.gnu.org/gmp/gmp-$gmp_ver.tar.lz
isl_url=http://isl.gforge.inria.fr/isl-$isl_ver.tar.xz
mpc_url=https://ftpmirror.gnu.org/mpc/mpc-$mpc_ver.tar.gz
mpfr_url=https://www.mpfr.org/mpfr-current/mpfr-$mpfr_ver.tar.xz
musl_url=https://www.musl-libc.org/releases/musl-$musl_ver.tar.gz

#
# Package Checksums (sha512sum)
#
binutils_sum=f4aadea1afa85d9ceb7be377afab9270a42ab0fd1fae86a7c69510b80de1aaac76f15cfb8730f9d233466a89fd020ab7e6e705e754c6b40f5fe2d16a5214562e
gcc_sum=0cb2a74c793face751f42bc580960b00e2bfea785872a0a2155f1f1dbfaa248f9591b67f4322db0f096f8844aca9243bc02732bda106c3b6e43b02bb67eb3096
gmp_sum=9975e8766e62a1d48c0b6d7bbdd2fccb5b22243819102ca6c8d91f0edd2d3a1cef21c526d647c2159bb29dd2a7dcbd0d621391b2e4b48662cf63a8e6749561cd
isl_sum=8dc7b0c14e5bfdca8f2161be51d3c9afcd18bc217bb19b7de01dbba0c6f3fdc2b725fb999f8562c77bf2918d3005c9247f7a58474a6da7697390067944d4d4aa
mpc_sum=72d657958b07c7812dc9c7cbae093118ce0e454c68a585bfb0e2fa559f1bf7c5f49b93906f580ab3f1073e5b595d23c6494d4d76b765d16dde857a18dd239628
mpfr_sum=d583555d08863bf36c89b289ae26bae353d9a31f08ee3894520992d2c26e5683c4c9c193d7ad139632f71c0a476d85ea76182702a98bf08dde7b6f65a54f8b88
musl_sum=58bd88189a6002356728cea1c6f6605a893fe54f7687595879add4eab283c8692c3b031eb9457ad00d1edd082cfe62fcc0eb5eb1d3bf4f1d749c0efa2a95fec1

#
# Development Directories
#
CURDIR="$PWD"
SRCDIR="$CURDIR/sources"
BLDDIR="$CURDIR/builds"
PCHDIR="$CURDIR/patches"

[ ! -d $SRCDIR ] && echo "${BLUEC}=>${NORMALC} Creating the sources directory...\n" && mkdir $SRCDIR
[ ! -d $BLDDIR ] && echo "${BLUEC}=>${NORMALC} Creating the builds directory...\n" && mkdir $BLDDIR
[ ! -d $PCHDIR ] && echo "${BLUEC}=>${NORMALC} Creating the patches directory...\n" && mkdir $PCHDIR

#
# Preparation Function - gtpackage()
#
gtpackage() {
  cd $SRCDIR

  if [ ! -d $1 ]; then
    mkdir $1
  else
    echo "${REDC}=>${NORMALC} $1 source directory already exists, skipping..."
  fi

  cd $1

  HOLDER="$(basename $2)"

  if [ ! -f "$HOLDER" ]; then
    echo "${GREENC}=>${NORMALC} Fetching "$HOLDER"..."
    wget "$2"
  else
    echo "${REDC}=>${NORMALC} "$HOLDER" already exists, skipping..."
  fi

  echo "${GREENC}=>${NORMALC} Verifying "$HOLDER"..."
  echo "$3 $HOLDER" | sha512sum -c || {
    echo "${REDC}=>${NORMALC} "$HOLDER" is corrupted, redownloading..." &&
    rm "$HOLDER" &&
    wget "$2";
  }

  rm -fr $1-$4
  echo "${GREENC}=>${NORMALC} Unpacking $HOLDER..."
  tar xf $HOLDER -C .

  echo
}

gtpackage binutils "$binutils_url" $binutils_sum $binutils_ver
gtpackage gcc "$gcc_url" $gcc_sum $gcc_ver
gtpackage gmp "$gmp_url" $gmp_sum $gmp_ver
gtpackage isl "$isl_url" $isl_sum $isl_ver
gtpackage mpc "$mpc_url" $mpc_sum $mpc_ver
gtpackage mpfr "$mpfr_url" $mpfr_sum $mpfr_ver
gtpackage musl "$musl_url" $musl_sum $musl_ver

#
# Patching
#

#
# The musl patch allows us to pass `-ffast-math` in CFLAGS when building musl
# since musl requires libgcc and libgcc requires musl, so the build script needs
# patching so that you can pass -ffast-math to CFLAGS. (Aurelian)
#
# Apparently musl only relies on libgcc for the following symbols `__muslsc3`,
# `__muldc3`, `__muldxc3`, and and `__powidf2` its configure script can be
# patched (simply by passing `--ffast-math` to prevent it from relying on
# libgcc). (Aurelian & firasuke)
#
cd $PCHDIR
[ ! -d musl ] && mkdir musl
cd musl

if [ ! -f 0002-enable-fast-math.patch ]; then
  echo "${GREENC}=>${NORMALC} Fetching musl 0002-enable-fast-math.patch from qword..."
  wget https://raw.githubusercontent.com/glaucuslinux/glaucus/master/cerata/musl/patches/qword/0002-enable-fast-math.patch
else
  echo "${REDC}=>${NORMALC} 0002-enable-fast-math.patch already exists, skipping..."
fi

echo "${BLUEC}=>${NORMALC} Applying musl 0002-enable-fast-math.patch from qword..."
cd $SRCDIR/musl/musl-$musl_ver
patch -p0 -i $PCHDIR/musl/0002-enable-fast-math.patch

#
# The gcc patch is for a bug that forces CET when cross compiling in both lto-plugin
# and libiberty.
#
cd $PCHDIR
[ ! -d gcc ] && mkdir gcc
cd gcc

if [ ! -f Enable-CET-in-cross-compiler-if-possible.patch ]; then
  echo "${GREENC}=>${NORMALC} Fetching gcc Enable-CET-in-cross-compiler-if-possible.patch from upstream..."
  wget https://raw.githubusercontent.com/glaucuslinux/glaucus/master/cerata/gcc/patches/upstream/Enable-CET-in-cross-compiler-if-possible.patch
else
  echo "${REDC}=>${NORMALC} Enable-CET-in-cross-compiler-if-possible.patch already exists, skipping..."
fi

echo "${BLUEC}=>${NORMALC} Applying gcc Enable-CET-in-cross-compiler-if-possible.patch from upstream..."
cd $SRCDIR/gcc/gcc-$gcc_ver
patch -p1 -i $PCHDIR/gcc/Enable-CET-in-cross-compiler-if-possible.patch

echo

#
# Don't change anything from here on, unless you know what you're doing.
#
if [ -d "$CURDIR/builds" ]; then
  echo "${GREENC}=>${NORMALC} Cleaning builds directory..."
  rm -fr "$CURDIR/builds"
  mkdir "$CURDIR/builds"
fi

if [ -d "$CURDIR/toolchain" ]; then
  echo "${GREENC}=>${NORMALC} Cleaning toolchain directory..."
  rm -fr "$CURDIR/toolchain"
  mkdir "$CURDIR/toolchain"
fi

if [ -d "$CURDIR/sysroot" ]; then
  echo "${GREENC}=>${NORMALC} Cleaning sysroot directory..."
  rm -fr "$CURDIR/sysroot"
  mkdir "$CURDIR/sysroot"
fi

#
# Build Directories
#
# Please don't change $GTSYSROOT to `$CURDIR/toolchain/$XARCH` like CLFS and
# other implementations because it'll break here (even if binutils insists
# on installing stuff to that directory).
#
GTPREFIX="$CURDIR/toolchain"
GTSYSROOT="$CURDIR/sysroot"

#
# Available Architectures
#
XARCH=x86_64-linux-musl

#
# FLAGS
#
CFLAGS=-O2
CXXFLAGS=-O2

#
# PATH (Use host tools, then switch to ours when they're available)
#
PATH=$GTPREFIX/bin:/usr/bin:/bin

echo

#
# Step 1: musl-headers
#
echo "=> Preparing musl-headers..."
cd $BLDDIR
mkdir musl-headers
cd musl-headers

#
# Note the use of `--host` instead of `--target` (musl-cross-make, Aurelian)
#
# Also, notice the use of `--disable-shared` to prevent the installation of
# additional files such as crt0.o... (Aurelian).
#
# Additionally, we specify`--prefix=/usr` because this is where we expect musl
# to be in the final system. (musl wiki)
#
# `CC` must be equal to the host's C compiler because ours isn't ready yet.
#
echo "=> Configuring musl-headers..."
ARCH=x86_64 \
CC=gcc \
CROSS_COMPILE= \
CFLAGS='-O2 -Wno-error=implicit-function-declaration' \
$SRCDIR/musl/musl-$musl_ver/configure \
  --host=$XARCH \
  --prefix=/usr \
  --disable-shared

#
# We only want the headers to configure gcc... Also with musl installs, you
# almost always should use a DESTDIR (that also should 99% be equal to gcc's
# and binutils `--with-sysroot` value...
#
echo "=> Installing musl-headers..."
make \
  DESTDIR=$GTSYSROOT \
  install-headers

echo

#
# Step 2: cross-binutils
#
echo "=> Preparing cross-binutils..."
cd $BLDDIR
mkdir cross-binutils
cd cross-binutils

#
# Unlike musl, `--prefix` for GNU stuff means where we expect them to be
# installed, so specifying it will save you the need to add a `DESTDIR` when
# installing.
# 
# One question though, doesn't `--prefix` gets baked into binaries?
#
# The `--target` specifies that we're cross compiling, and binutils tools will
# be prefixed by the value provided to it. There's no need to specify `--build`
# and `--host` as `config.guess`/`config.sub` are now smart enough to figure
# them in almost all GNU packages.
#
# The use of `--disable-werror` is almost a neccessity now, without it the build
# may fail, or throw implicit-fallthrough warnings, among others (Aurelian).
#
# Notice how we specify a `--with-sysroot` here to tell binutils to consider
# the passed value as the root directory of our target system in which it'll
# search for target headers and libraries.
#
echo "=> Configuring cross-binutils..."
CFLAGS=-O2 \
$SRCDIR/binutils/binutils-$binutils_ver/configure \
  --prefix=$GTPREFIX \
  --target=$XARCH \
  --with-sysroot=$GTSYSROOT \
  --disable-werror

echo "=> Building cross-binutils..."
make \
  all-binutils \
  all-gas \
  all-ld

echo "=> Installing cross-binutils..."
make \
  install-binutils \
  install-gas \
  install-ld

echo

#
# Step 3: GCC Prerequisites
# 
# We track them manually instead of using `contrib/download_prerequisites` in
# gcc's sources.
#
echo "=> Preparing GCC prerequisites..."
cp -ar $SRCDIR/gmp/gmp-$gmp_ver $SRCDIR/gcc/gcc-$gcc_ver/gmp
cp -ar $SRCDIR/mpfr/mpfr-$mpfr_ver $SRCDIR/gcc/gcc-$gcc_ver/mpfr
cp -ar $SRCDIR/mpc/mpc-$mpc_ver $SRCDIR/gcc/gcc-$gcc_ver/mpc
cp -ar $SRCDIR/isl/isl-$isl_ver $SRCDIR/gcc/gcc-$gcc_ver/isl

echo

#
# Step 4: system-gcc (compiler)
#
echo "=> Preparing system-gcc..."
cd $BLDDIR
mkdir system-gcc
cd system-gcc

#
# Again, everything said in cross-binutils applies here.
#
# We need c++ language support to be able to build GCC, since GCC has big parts
# of its source code written in C++.
#
echo "=> Configuring system-gcc..."
CFLAGS=-O2 \
CXXFLAGS=-O2 \
$SRCDIR/gcc/gcc-$gcc_ver/configure \
  --prefix=$GTPREFIX \
  --target=$XARCH \
  --with-sysroot=$GTSYSROOT \
  --enable-languages=c,c++ \
  --disable-multilib \
  --enable-initfini-array

echo "=> Building system-gcc compiler..."
mkdir -p $GTSYSROOT/usr/include
make \
  all-gcc

echo "=> Installing system-gcc compiler..."
make \
  install-gcc

echo

#
# Step 5: musl
#
echo "=> Preparing musl..."
cd $BLDDIR
mkdir musl
cd musl

#
# Notice how we build it with `CC` set to our cross compiler. It's important
# that the PATH to this cross compiler is set correctly, or it won't be picked
# up.
#
# Also notice how `CROSS_COMPILE` isn't empty here, and it should end with `-`.
#
# We also disable `--disable-static` since we want a shared version.
#
echo "=> Configuring musl..."
ARCH=x86_64 \
CC=$GTPREFIX/bin/$XARCH-gcc \
CROSS_COMPILE=$XARCH- \
CFLAGS='-O2 -Wno-error=implicit-function-declaration -ffast-math' \
$SRCDIR/musl/musl-$musl_ver/configure \
  --host=$XARCH \
  --prefix=/usr \
  --disable-static

echo "=> Building musl..."
make

echo "=> Installing musl..."
make \
  DESTDIR=$GTSYSROOT \
  install-libs \
  install-tools

#
# Almost all implementations of musl based toolchains would want to change the
# symlink between LDSO and the libc.so because it'll be wrong almost always...
#
rm -f $GTSYSROOT/lib/ld-musl-x86_64.so.1
cp $GTSYSROOT/usr/lib/libc.so $GTSYSROOT/lib/ld-musl-x86_64.so.1

echo

#
# Step 6: system-gcc (libgcc)
#
echo "=> Preparing system-gcc libgcc..."
cd $BLDDIR/system-gcc

echo "=> Building system-gcc libgcc..."
make \
  all-target-libgcc

echo "=> Installing system-gcc libgcc..."
make \
  install-target-libgcc

echo

#
# Step 7: system-gcc (libstdc++-v3)
#
echo "=> Building system-gcc libstdc++-v3..."
make \
  all-target-libstdc++-v3

echo "=> Installing system-gcc libstdc++-v3..."
make \
  install-target-libstdc++-v3

echo

##
## [Optional] Step 8: Cross/Native (or Native) gcc
##
#echo "=> Preparing Cross/Native (or Native) gcc..."
#cd $BLDDIR
#mkdir gcc
#cd gcc

##
## Notice how `--host=--target` because this is a cross/native build of GCC that
## is build on the current machine (regardless of what it is) to target `$XARCH`.
## If the current machine (`--build`) happens to be equal to `$XARCH`, then this
## becomes a native build of GCC.
##
## Note how `--prefix=/usr` because this should only be built if you were on
## the final system itself and wanted to install/update its GCC.
##
## Note the sysroot `--with-sysroot=/` to keep track of the absolute path to all
## the tools (can also be set?) (Aurelian)
##
## Note the flag `--with-build-sysroot` that was used for legacy purposes with
## older versions of GCC, but it can be removed (Aurelian)
##
#echo "=> Configuring Cross/Native (or Native) gcc..."
#CFLAGS=-O2 \
#CXXFLAGS=-O2 \
#$SRCDIR/gcc/gcc-$gcc_ver/configure \
  #--host=$XARCH \
  #--prefix=/usr \
  #--target=$XARCH \
  #--with-sysroot=/ \
  #--with-build-sysroot=$GTSYSROOT \
  #--enable-languages=c,c++ \
  #--enable-initfini-array \
  #--disable-multilib \
  #--disable-nls

#echo "=> Building Cross/Native (or Native) gcc..."
#make \
  #all-gcc \
  #all-target-libgcc \
  #all-target-libstdc++-v3

#echo "=> Installing Cross/Native (or Native) gcc..."
#make \
  #install-gcc \
  #install-target-libgcc \
  #install-target-libstdc++-v3

#echo

##
## [Optional] Step 9: Cross/Native (or Native) binutils
##
#echo "=> Preparing Cross/Native (or Native) binutils..."
#cd $BLDDIR
#mkdir binutils
#cd binutils

##
## Same notes from [Optional] Step 8: Cross/Native (or Native) gcc
##
#echo "=> Configuring Cross/Native (or Native) binutils..."
#CFLAGS=-O2 \
#$SRCDIR/binutils/binutils-$binutils_ver/configure \
  #--host=$XARCH \
  #--prefix=/usr \
  #--target=$XARCH \
  #--with-sysroot=/ \
  #--disable-nls \
  #--disable-werror

#echo "=> Building binutils..."
#make \
  #all-binutils \
  #all-gas \
  #all-ld

#echo "=> Building binutils..."
#make \
  #install-binutils \
  #install-gas \
  #install-ld

#echo

echo "=> Done! Have fun!"

This script is now known as mussel and is being officially maintained by firasuke.