Skip to content

Instantly share code, notes, and snippets.

@smengcl

smengcl/grpc-java-v1.33.0-m1.diff Secret

Last active January 25, 2025 04:56
Show Gist options
  • Save smengcl/6a269f44d909ef9245a9ad8c7c6d41e0 to your computer and use it in GitHub Desktop.
Save smengcl/6a269f44d909ef9245a9ad8c7c6d41e0 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
grpc-java-v1.33.0-m1.diff
diff --git a/compiler/build.gradle b/compiler/build.gradle
index 60d3a436f..409df7a03 100644
--- a/compiler/build.gradle
+++ b/compiler/build.gradle
@@ -31,6 +31,7 @@ def addLibraryIfNotLinked = { libName, argList ->
}
def String arch = rootProject.hasProperty('targetArch') ? rootProject.targetArch : osdetector.arch
+// arch = arch.replace('osx_arm-v8', 'osx_aarch64')
def boolean vcDisable = rootProject.hasProperty('vcDisable') ? rootProject.vcDisable : false
def boolean usingVisualCpp // Whether VisualCpp is actually available and selected
@@ -56,8 +57,10 @@ model {
linker.executable = 'aarch64-linux-gnu-g++'
}
target("s390_64")
+ target("osx_aarch64")
}
clang(Clang) {
+ target("osx_arm-v8")
}
}
@@ -67,6 +70,8 @@ model {
ppcle_64 { architecture "ppcle_64" }
aarch_64 { architecture "aarch_64" }
s390_64 { architecture "s390_64" }
+ osx_aarch64 { architecture "osx_aarch64" }
+ 'osx_arm-v8' { architecture "osx_aarch64" }
}
components {
@@ -76,11 +81,14 @@ model {
'x86_64',
'ppcle_64',
'aarch_64',
- 's390_64'
+ 's390_64',
+ 'osx_aarch64'
]) {
// If arch is not within the defined platforms, we do not specify the
// targetPlatform so that Gradle will choose what is appropriate.
targetPlatform arch
+ } else if (arch in [ 'osx_arm-v8' ]) {
+ targetPlatform 'osx_aarch64'
}
baseName "$protocPluginBaseName"
}
diff --git a/compiler/check-artifact.sh b/compiler/check-artifact.sh
index 13ae89c74..341709d17 100755
--- a/compiler/check-artifact.sh
+++ b/compiler/check-artifact.sh
@@ -81,6 +81,8 @@ checkArch ()
assertEq "$format" "i386" $LINENO
elif [[ "$ARCH" == x86_64 ]]; then
assertEq "$format" "x86_64" $LINENO
+ elif [[ "$ARCH" == arm64 ]]; then
+ assertEq "$format" "arm64" $LINENO
else
fail "Unsupported arch: $ARCH"
fi
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index bb8b2fc26..ffed3a254 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
Raw
ozone-m1.md

M1 Mac Native Ozone build (Experimental)

Protobuf

brew install maven

# I'm using Zulu JDK 8 downloaded by IntelliJ IDEA,
# or download here manually: https://www.azul.com/downloads/?version=java-8-lts&os=macos&architecture=arm-64-bit&package=jdk
# Haven't tested with JDK 11, should work as well: ${HOME}/Library/Java/JavaVirtualMachines/azul-11.0.13/Contents/Home
export JAVA_HOME="${HOME}/Library/Java/JavaVirtualMachines/azul-1.8.0_312/Contents/Home"

PROTOBUF_VERSION="3.12.0"
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz
tar xzf protobuf-all-${PROTOBUF_VERSION}.tar.gz
cd protobuf-${PROTOBUF_VERSION}
./configure --disable-shared
make -j
sudo make install
mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=${PROTOBUF_VERSION} -Dclassifier=osx-aarch_64 -Dpackaging=exe -Dfile=src/protoc

cd ..
PROTOBUF_VERSION="3.7.1"
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz
tar xzf protobuf-all-${PROTOBUF_VERSION}.tar.gz
cd protobuf-${PROTOBUF_VERSION}
./configure --disable-shared
make -j
mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=${PROTOBUF_VERSION} -Dclassifier=osx-aarch_64 -Dpackaging=exe -Dfile=src/protoc

cd ..
PROTOBUF_VERSION="2.5.0"
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
tar xzf protobuf-2.5.0.tar.gz
cd protobuf-${PROTOBUF_VERSION}

vim src/google/protobuf/stubs/platform_macros.h
# Append the following after line 59:

#elif defined(__arm64__)
#define GOOGLE_PROTOBUF_ARCH_ARM 1
#define GOOGLE_PROTOBUF_ARCH_64_BIT 1

make check
./configure --disable-shared
make -j
mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 -Dclassifier=osx-aarch_64 -Dpackaging=exe -Dfile=src/protoc

Build native gRPC

git clone https://github.com/grpc/grpc-java.git
cd grpc-java
git checkout -b v1.33.0 v1.33.0

# Apply attached patch
# Inspired by https://github.com/grpc/grpc-java/issues/2202#issuecomment-250764314
git apply grpc-java-v1.33.0-m1.diff

# Building gRPC requires JDK 8, as mentioned in COMPILING.md
export JAVA_HOME="${HOME}/Library/Java/JavaVirtualMachines/azul-1.8.0_332/Contents/Home"

# https://github.com/grpc/grpc-java/blob/master/COMPILING.md#notes-for-unsupported-operating-systems
# Note: We need the Code Generation Plugin when building Ozone
./gradlew publishToMavenLocal -PskipAndroid=true -PskipCodegen=false -PtargetArch=arm64 -Pprotoc=/usr/local/bin/protoc -PvcDisable=true
# Verify artifact existence in maven local
$ find ~/.m2 -iname "protoc-gen-grpc*"
/Users/user/.m2/repository/io/grpc/protoc-gen-grpc-java
/Users/user/.m2/repository/io/grpc/protoc-gen-grpc-java/1.33.0/protoc-gen-grpc-java-1.33.0.pom.lastUpdated
/Users/user/.m2/repository/io/grpc/protoc-gen-grpc-java/1.33.0/protoc-gen-grpc-java-1.33.0.pom
/Users/user/.m2/repository/io/grpc/protoc-gen-grpc-java/1.33.0/protoc-gen-grpc-java-1.33.0.pom.sha1
/Users/user/.m2/repository/io/grpc/protoc-gen-grpc-java/1.33.0/protoc-gen-grpc-java-1.33.0-osx-arm64.exe
/Users/user/.m2/repository/io/grpc/protoc-gen-grpc-java/1.33.0/protoc-gen-grpc-java-1.33.0-osx-aarch_64.exe.lastUpdated

$ find . -iname "protoc-gen-grpc-java*"                    
./compiler/build/artifacts/java_plugin/protoc-gen-grpc-java.exe
./compiler/build/libs/protoc-gen-grpc-java-1.33.0.jar
./compiler/build/libs/protoc-gen-grpc-java-1.33.0-sources.jar
./compiler/build/libs/protoc-gen-grpc-java-1.33.0-javadoc.jar
./compiler/build/exe/java_plugin/protoc-gen-grpc-java

$ file ./compiler/build/artifacts/java_plugin/protoc-gen-grpc-java.exe
./compiler/build/artifacts/java_plugin/protoc-gen-grpc-java.exe: Mach-O 64-bit executable arm64

$ mvn install:install-file -DgroupId=io.grpc -DartifactId=protoc-gen-grpc-java -Dversion=1.33.0 -Dclassifier=osx-aarch_64 -Dpackaging=exe -Dfile=./compiler/build/artifacts/java_plugin/protoc-gen-grpc-java.exe

RocksDBJNI (Only necessary for RocksDB < 6.29.0)

Note: Manually compiling macOS arm64 binary is no longer necessary since 6.29, see RocksDB release notes:

New Features
Added RocksJava support for MacOS universal binary (ARM+x86)

# https://github.com/facebook/rocksdb/blob/master/java/RELEASE.md
# Ref: https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/rocksdb.rb

# Raise Docker resource memory limit to at least 5.50 GB if using Docker to compile.

brew install cmake
git clone https://github.com/facebook/rocksdb.git
cd rocksdb
git checkout -b v6.25.3 v6.25.3

# Manually remove other arch profiles from rocksdbjavastaticreleasedocker from Makefile
# or simply apply the attached diff:
git apply rdb-m1.diff

make jclean clean rocksdbjavastaticreleasedocker

file java/target/librocksdbjni-osx.jnilib
java/target/librocksdbjni-osx.jnilib: Mach-O 64-bit dynamically linked shared library arm64

mvn install:install-file -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=6.25.3 -Dpackaging=jar -Dfile=java/target/rocksdbjni-6.25.3.jar

# Or combine the arm64 jnilib with the official x86 jnilib:
# Insert aarch64 lib into the official jar
# First get the official jar from https://search.maven.org/search?q=org.rocksdb.rocksdbjni
unzip -l rocksdbjni-6.25.3.jar | grep librocksdbjni
 11434960  10-26-2021 21:38   librocksdbjni-linux32-musl.so
 10262035  10-26-2021 21:07   librocksdbjni-linux32.so
 11473832  10-26-2021 21:58   librocksdbjni-linux64-musl.so
 11344904  10-26-2021 21:22   librocksdbjni-linux64.so
  7975560  10-26-2021 20:55   librocksdbjni-osx.jnilib
 12459752  10-26-2021 23:13   librocksdbjni-linux-ppc64le.so
 12984368  10-26-2021 23:13   librocksdbjni-linux-ppc64le-musl.so
 10232960  10-26-2021 23:14   librocksdbjni-linux-aarch64.so
 10797328  10-26-2021 23:15   librocksdbjni-linux-aarch64-musl.so
  7002624  10-26-2021 23:12   librocksdbjni-win64.dll

# Extract the osx jnilib from the official jar
unzip rocksdbjni-6.25.3.jar librocksdbjni-osx.jnilib

mv librocksdbjni-osx.jnilib librocksdbjni-osx-x86.jnilib

# Make universal binary
lipo -create librocksdbjni-osx-x86.jnilib /Users/you/repo/rocksdb/java/target/librocksdbjni-osx.jnilib -output librocksdbjni-osx.jnilib

# Put the new jnilib back
zip rocksdbjni-6.25.3.jar librocksdbjni-osx.jnilib

mvn install:install-file -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=6.25.3 -Dpackaging=jar -Dfile=rocksdbjni-6.25.3.jar

Build Ozone!

# In ./pom.xml :

# 1. Check that proto-backwards-compatibility version >= 1.0.7 (which has the arm64 mac protolock binary)
# https://github.com/salesforce/proto-backwards-compat-maven-plugin

# 2. Check rocksdbjni version >= 6.25.3
# IMPORTANT: Be sure to "Load maven changes" if changed the pom.xml in IntelliJ.
# Otherwise running UTs inside IntelliJ may still pick up old RDB JNI version.

mvn clean install -Dmaven.javadoc.skip -DskipTests -DskipShade

[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time:  01:38 min
[INFO] Finished at: 2021-10-21T01:11:31-07:00
[INFO] ------------------------------------------------------------------------

(Optional) For IntelliJ QoL, IntelliSense for huge protoc generated classes:

Help - Edit Custom Properties ...:

idea.max.intellisense.filesize=10000
Raw
rdb-m1.diff
diff --git a/Makefile b/Makefile
index 274864378..4032235d7 100644
--- a/Makefile
+++ b/Makefile
@@ -2200,9 +2200,9 @@ rocksdbjavastaticrelease: rocksdbjavastatic
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1
-rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
+rocksdbjavastaticreleasedocker: rocksdbjavastatic
cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
- cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
+ cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.jnilib
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1
@smengcl
Copy link
Author

smengcl commented Jan 25, 2025

Some extra steps to work with recent master:

# Works as of 67fdb88b95050feea654a00649c54d014331b691

# New mvn install doesn't append the .exe extension somehow
mv ${HOME}/.m2/repository/com/google/protobuf/protoc/3.19.6/protoc-3.19.6-osx-aarch_64{,.exe}
mv ${HOME}/.m2/repository/com/google/protobuf/protoc/2.5.0/protoc-2.5.0-osx-aarch_64{,.exe}

# New gRPC uses a different suffix
cp ${HOME}/.m2/repository/io/grpc/protoc-gen-grpc-java/1.58.0/protoc-gen-grpc-java-1.58.0-osx-aarch_64.exe ${HOME}/.m2/repository/io/grpc/protoc-gen-grpc-java/1.58.0/protoc-gen-grpc-java-1.58.0-osx-arm64.exe

# Or optionally use hard link / symlink to save space

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment