Created
November 9, 2018 05:11
-
-
Save hcho3/bbdfa074525555c3059c04899217fe08 to your computer and use it in GitHub Desktop.
How to compile libHDFS and XGBoost on Mac OS X
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# Compile HDFS native library from source | |
cd | |
wget http://mirrors.advancedhosters.com/apache/hadoop/common/hadoop-3.1.1/hadoop-3.1.1-src.tar.gz | |
tar xvf hadoop-3.1.1-src.tar.gz | |
cd hadoop-3.1.1-src/hadoop-hdfs-project/hadoop-hdfs-native-client | |
CC=gcc-8 CXX=g++-8 mvn compile -Pnative # Use Homebrew GCC | |
# Create HDFS home under the home directory | |
cd | |
mkdir hdfs_home | |
cd hdfs_home | |
mkdir -p lib/native | |
# Copy libhdfs.a and libhdfs.dylib | |
cp -rv ~/hadoop-3.1.1-src/hadoop-hdfs-project/hadoop-hdfs-native-client/target/native/target/usr/local/lib/ ./lib/native/ | |
# Copy hdfs.h | |
cp -rv ~/hadoop-3.1.1-src/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/ ./include/ | |
# Now build XGBoost with libhdfs | |
cd /path/to/xgboost | |
mkdir build | |
cd build | |
HADOOP_HDFS_HOME=${HOME}/hdfs_home CC=gcc-8 CXX=g++-8 cmake .. -DUSE_HDFS=ON | |
make |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment