Skip to content

Instantly share code, notes, and snippets.

@praton1729
Created April 13, 2025 19:25
Show Gist options
  • Save praton1729/cdfb3bb92f5f7887a881f17fb7a20f02 to your computer and use it in GitHub Desktop.
Save praton1729/cdfb3bb92f5f7887a881f17fb7a20f02 to your computer and use it in GitHub Desktop.
Create a highly compressed binary from aarch64 baremetal elfs
#!/usr/bin/env python3
"""
Binary Section Analyzer and Compressor with Final Binary Generator for AARCH64
This script analyzes an ELF file, extracts its sections, compresses them with
various algorithms, and creates a final compressed binary with metadata for decompression.
Specifically configured for AARCH64 architecture.
"""
import os
import sys
import subprocess
import zlib
import lzma
import bz2
import tempfile
import shutil
import struct
import argparse
import json
import pickle
from collections import defaultdict, Counter
from pathlib import Path
import math
# Magic number for our custom compressed binary format
MAGIC = b'SECCOMP'
FORMAT_VERSION = 1
def run_command(cmd):
"""Run a shell command and return the output."""
try:
result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, shell=True)
return result.stdout
except subprocess.CalledProcessError as e:
print(f"Error executing command: {cmd}")
print(f"Error: {e.stderr}")
sys.exit(1)
def extract_sections(elf_file, prefix="aarch64-linux-gnu-"):
"""Extract all sections from an ELF file and return a dictionary of their info."""
sections = {}
# Get section information using AARCH64 tools
objdump_cmd = f"{prefix}objdump -h {elf_file}"
objdump_output = run_command(objdump_cmd)
# Parse the output
current_section = None
for line in objdump_output.splitlines():
line = line.strip()
if not line:
continue
if line.startswith('Idx'): # Header line
continue
parts = line.split()
if len(parts) >= 7 and parts[0].isdigit():
section_name = parts[1]
if section_name.startswith('.'):
size_hex = parts[2]
size = int(size_hex, 16)
vma = parts[3]
vma_int = int(vma, 16)
lma = parts[4]
file_off = parts[5]
flags = parts[6]
sections[section_name] = {
'size': size,
'vma': vma,
'vma_int': vma_int,
'lma': lma,
'file_off': file_off,
'flags': flags,
'alloc': 'ALLOC' in flags,
'data': None # Will store actual binary data later
}
# Extract sections that have ALLOC flag set
tmpdir = tempfile.mkdtemp()
try:
for section_name, info in sections.items():
if info['size'] > 0: # Only extract non-empty sections
output_file = os.path.join(tmpdir, f"{section_name.replace('/', '_')}.bin")
objcopy_cmd = f"{prefix}objcopy -O binary --only-section={section_name} {elf_file} {output_file}"
run_command(objcopy_cmd)
# Read the extracted binary data
if os.path.exists(output_file) and os.path.getsize(output_file) > 0:
with open(output_file, 'rb') as f:
sections[section_name]['data'] = f.read()
else:
# For sections like .bss that don't have actual file content
sections[section_name]['data'] = b'\x00' * info['size'] if info['alloc'] else None
finally:
shutil.rmtree(tmpdir)
# Get entry point using AARCH64 readelf
readelf_cmd = f"{prefix}readelf -h {elf_file}"
readelf_output = run_command(readelf_cmd)
entry_point = None
for line in readelf_output.splitlines():
if "Entry point address" in line:
entry_point = int(line.split(':')[1].strip(), 16)
break
return sections, entry_point
def analyze_section_entropy(data):
"""Calculate the entropy of a binary section."""
if not data or len(data) == 0:
return 0.0
# Count byte frequencies
freqs = Counter(data)
total = len(data)
# Calculate entropy
entropy = 0
for count in freqs.values():
probability = count / total
entropy -= probability * math.log2(probability)
return entropy
def find_repeating_patterns(data, min_length=4, max_length=32, step=4):
"""Find repeating byte patterns in the data."""
if not data or len(data) < min_length:
return {}
patterns = defaultdict(int)
# Check for patterns of different lengths
for pattern_len in range(min_length, min(max_length, len(data)), step):
for i in range(len(data) - pattern_len + 1):
pattern = data[i:i+pattern_len]
patterns[pattern] += 1
# Filter out patterns that don't repeat
repeating = {pattern: count for pattern, count in patterns.items() if count > 1}
# Sort by (count * length) to find the most valuable patterns
sorted_patterns = sorted(
repeating.items(),
key=lambda x: len(x[0]) * x[1],
reverse=True
)
return dict(sorted_patterns[:20]) # Return top 20 most valuable patterns
def simple_delta_encode(data):
"""Apply simple delta encoding to a byte array."""
if not data or len(data) <= 1:
return data
delta_encoded = bytearray(len(data))
delta_encoded[0] = data[0]
for i in range(1, len(data)):
delta_encoded[i] = (data[i] - data[i-1]) & 0xFF
return bytes(delta_encoded)
def simple_delta_decode(data):
"""Decode simple delta encoding."""
if not data or len(data) <= 1:
return data
decoded = bytearray(len(data))
decoded[0] = data[0]
for i in range(1, len(data)):
decoded[i] = (decoded[i-1] + data[i]) & 0xFF
return bytes(decoded)
def word_delta_encode(data):
"""Apply word-aligned delta encoding - optimized for AARCH64's 32-bit/64-bit instructions."""
if len(data) < 4:
return data
# For AARCH64, we'll use 4-byte alignment
# (could also use 8 bytes for some instruction sets)
word_size = 4
# Ensure data length is multiple of word_size by padding if needed
if len(data) % word_size != 0:
padded_data = bytearray(data)
padded_data.extend(b'\x00' * (word_size - (len(data) % word_size)))
data = bytes(padded_data)
result = bytearray()
# First word is stored as-is
result.extend(data[0:word_size])
# Process remaining words
for i in range(word_size, len(data), word_size):
for j in range(word_size):
result.append(data[i+j] ^ data[i-word_size+j])
return bytes(result)
def word_delta_decode(data):
"""Decode word-aligned delta encoding for AARCH64's word size."""
if len(data) < 4:
return data
word_size = 4 # Using 4-byte words for AARCH64
result = bytearray(len(data))
# First word is copied as-is
result[0:word_size] = data[0:word_size]
# Process remaining words
for i in range(word_size, len(data), word_size):
for j in range(word_size):
result[i+j] = data[i+j] ^ result[i-word_size+j]
return bytes(result)
def aarch64_instruction_encode(data):
"""Special encoding for AARCH64 instructions, looking for common patterns."""
if len(data) < 4:
return data
# AARCH64 uses fixed 4-byte instructions
# We can exploit patterns like:
# - Common opcodes
# - Register patterns
# - Small immediate values
# For now, a simplified approach:
result = bytearray()
# Check if it's likely code (high entropy but with patterns)
entropy = analyze_section_entropy(data)
if entropy < 6.0 or entropy > 7.5:
return data # Probably not instructions, use default
# Extract 4-byte chunks
chunks = []
for i in range(0, len(data), 4):
if i + 4 <= len(data):
chunks.append(data[i:i+4])
if not chunks:
return data
# Look for similar chunks (differing only in registers or immediates)
similarities = []
for i in range(len(chunks)):
for j in range(i+1, len(chunks)):
# Count differing bytes
diff_count = sum(b1 != b2 for b1, b2 in zip(chunks[i], chunks[j]))
if diff_count <= 2: # Similar instructions
similarities.append((i, j, diff_count))
# If we found many similarities, use delta encoding
if len(similarities) > len(chunks) // 4:
return word_delta_encode(data)
# Otherwise fall back to regular data
return data
def aarch64_instruction_decode(data):
"""Decode the AARCH64 instruction encoding."""
# For now, we'll just call word_delta_decode since our encoder is simplified
return word_delta_decode(data)
def dictionary_encode(data):
"""Apply simple dictionary encoding to the data."""
if len(data) < 16:
return data # Too small to be worth it
# Find repeating patterns
patterns = find_repeating_patterns(data, min_length=8, max_length=64)
if not patterns:
return data # No good patterns found
# Sort patterns by value (bytes saved)
sorted_patterns = sorted(
patterns.items(),
key=lambda x: (len(x[0]) - 2) * x[1], # Value minus overhead
reverse=True
)
# Use top patterns (limit dictionary size)
dictionary = []
for pattern, _ in sorted_patterns[:16]: # Limit to 16 entries
if len(pattern) >= 8: # Only use if pattern is long enough
dictionary.append(pattern)
if not dictionary:
return data
# Encode the data
result = bytearray()
# First byte is dictionary size
result.append(len(dictionary))
# Add dictionary entries (length + pattern)
for pattern in dictionary:
result.append(len(pattern))
result.extend(pattern)
# Process the data
i = 0
while i < len(data):
# Check if current position matches any pattern
found = False
for idx, pattern in enumerate(dictionary):
if i + len(pattern) <= len(data) and data[i:i+len(pattern)] == pattern:
# Found a match - use index (0x80 and above are dictionary references)
result.append(0x80 | idx)
i += len(pattern)
found = True
break
if not found:
# No match, copy the byte literally
result.append(data[i])
i += 1
# Return original if our encoding is larger
return bytes(result) if len(result) < len(data) else data
def dictionary_decode(data):
"""Decode dictionary-encoded data."""
if not data or len(data) < 2:
return data
# First byte is dictionary size
dict_size = data[0]
if dict_size == 0 or dict_size > 16:
return data # Invalid dictionary size or not dictionary encoded
# Read dictionary
dictionary = []
pos = 1
for _ in range(dict_size):
if pos >= len(data):
return data # Invalid format
pattern_len = data[pos]
pos += 1
if pos + pattern_len > len(data):
return data # Invalid format
pattern = data[pos:pos+pattern_len]
dictionary.append(pattern)
pos += pattern_len
# Decode the data
result = bytearray()
while pos < len(data):
byte = data[pos]
pos += 1
if byte & 0x80: # Dictionary reference
idx = byte & 0x7F
if idx < len(dictionary):
result.extend(dictionary[idx])
else:
return data # Invalid reference
else:
# Literal byte
result.append(byte)
return bytes(result)
def compress_with_algorithms(data, is_code=False):
"""Compress data with various algorithms and return results."""
if not data:
return {}
results = {}
# Standard compression algorithms with their implementation
results['none'] = {
'size': len(data),
'data': data,
'encode': lambda x: x,
'decode': lambda x: x
}
# Standard compression algorithms
try:
compressed = zlib.compress(data)
results['zlib'] = {
'size': len(compressed),
'data': compressed,
'encode': zlib.compress,
'decode': zlib.decompress
}
except:
pass
try:
compressed = lzma.compress(data)
results['lzma'] = {
'size': len(compressed),
'data': compressed,
'encode': lzma.compress,
'decode': lzma.decompress
}
except:
pass
try:
compressed = bz2.compress(data)
results['bz2'] = {
'size': len(compressed),
'data': compressed,
'encode': bz2.compress,
'decode': bz2.decompress
}
except:
pass
# Simple delta encoding
if len(data) > 1:
try:
delta_encoded = simple_delta_encode(data)
compressed = zlib.compress(delta_encoded)
results['delta'] = {
'size': len(compressed),
'data': compressed,
'encode': lambda x: zlib.compress(simple_delta_encode(x)),
'decode': lambda x: simple_delta_decode(zlib.decompress(x))
}
except:
pass
# Word-aligned delta - good for AARCH64 instructions
if len(data) >= 4:
try:
word_delta = word_delta_encode(data)
compressed = zlib.compress(word_delta)
results['word_delta'] = {
'size': len(compressed),
'data': compressed,
'encode': lambda x: zlib.compress(word_delta_encode(x)),
'decode': lambda x: word_delta_decode(zlib.decompress(x))
}
except:
pass
# AARCH64-specific instruction encoding - only if code section
if is_code and len(data) >= 8:
try:
aarch_encoded = aarch64_instruction_encode(data)
compressed = zlib.compress(aarch_encoded)
results['aarch64_code'] = {
'size': len(compressed),
'data': compressed,
'encode': lambda x: zlib.compress(aarch64_instruction_encode(x)),
'decode': lambda x: aarch64_instruction_decode(zlib.decompress(x))
}
except:
pass
# Dictionary-based approach
if len(data) > 32:
try:
dict_encoded = dictionary_encode(data)
if len(dict_encoded) < len(data):
compressed = zlib.compress(dict_encoded)
results['dictionary'] = {
'size': len(compressed),
'data': compressed,
'encode': lambda x: zlib.compress(dictionary_encode(x)),
'decode': lambda x: dictionary_decode(zlib.decompress(x))
}
except:
pass
return results
def analyze_elf(elf_file, create_binary=False, output_file=None, decompressor=False, prefix="aarch64-linux-gnu-"):
"""Analyze an ELF file's sections and compression potential."""
print(f"Analyzing AARCH64 ELF file: {elf_file}")
# Extract sections using AARCH64 tools
sections, entry_point = extract_sections(elf_file, prefix)
# Analyze and compress each section
results = []
print("\nSection Analysis:")
print("-" * 80)
print(f"{'Section':<20} {'Size':<10} {'Entropy':<10} {'Best Algorithm':<16} {'Comp Size':<10} {'Ratio':<10}")
print("-" * 80)
section_results = {}
for name, info in sorted(sections.items(), key=lambda x: x[1]['size'], reverse=True):
data = info['data']
if data is None or len(data) == 0:
continue
# Skip non-ALLOC sections if we're creating a binary
if create_binary and not info['alloc']:
continue
orig_size = len(data)
entropy = analyze_section_entropy(data)
# Check if this is likely a code section
is_code = name in ('.text', '.init', '.fini', '.plt') or 'CODE' in info['flags']
compression_results = compress_with_algorithms(data, is_code)
# Find best compression
best_algo = min(compression_results.items(), key=lambda x: x[1]['size'])
algo_name = best_algo[0]
best_size = best_algo[1]['size']
best_ratio = best_size / orig_size if orig_size > 0 else 1.0
# Store result
section_results[name] = {
'name': name,
'size': orig_size,
'entropy': entropy,
'vma': info['vma_int'],
'alloc': info['alloc'],
'flags': info['flags'],
'is_code': is_code,
'algo': algo_name,
'comp_size': best_size,
'comp_data': best_algo[1]['data'],
'encode': best_algo[1]['encode'],
'decode': best_algo[1]['decode'],
'ratio': best_ratio
}
print(f"{name:<20} {orig_size:<10} {entropy:<10.2f} {algo_name:<16} {best_size:<10} {best_ratio:<10.2f}")
results.append(section_results[name])
# Print summary
print("\nCompression Summary:")
print("-" * 80)
# Group sections by type
section_types = defaultdict(list)
for r in results:
# Determine section type based on name and flags
if r['is_code']:
section_type = 'Code'
elif r['name'] in ('.data', '.rodata'):
section_type = 'Data'
elif r['name'].startswith('.debug'):
section_type = 'Debug'
else:
section_type = 'Other'
section_types[section_type].append(r)
# Print results by section type
for section_type, sections in section_types.items():
total_size = sum(s['size'] for s in sections)
best_size = sum(s['comp_size'] for s in sections)
print(f"{section_type} sections:")
print(f" Total size: {total_size} bytes")
print(f" Compressed size: {best_size} bytes")
print(f" Overall ratio: {best_size/total_size:.2f}")
# Best algorithms per type
algo_counts = Counter(s['algo'] for s in sections)
best_algos = algo_counts.most_common(2)
print(f" Best algorithms: {', '.join(f'{algo} ({count})' for algo, count in best_algos)}")
print()
# Create compressed binary if requested
if create_binary and output_file:
create_compressed_binary(output_file, section_results, entry_point, decompressor)
return section_results, entry_point
def create_compressed_binary(output_file, section_results, entry_point, include_decompressor=False):
"""Create a compressed binary file using the best compression for each section."""
# Filter to only include ALLOC sections
alloc_sections = {name: info for name, info in section_results.items() if info['alloc']}
# Create header
header = bytearray()
# Magic number
header.extend(MAGIC)
# Format version (1 byte)
header.append(FORMAT_VERSION)
# Entry point (8 bytes for AARCH64 to support full 64-bit addresses)
header.extend(struct.pack("<Q", entry_point))
# Number of sections (2 bytes)
header.extend(struct.pack("<H", len(alloc_sections)))
# Section directory
section_data = bytearray()
for name, info in sorted(alloc_sections.items(), key=lambda x: x[1]['vma']):
# Section name (null-terminated, max 16 chars)
name_bytes = name.encode('ascii')[:15] + b'\0'
header.extend(name_bytes.ljust(16, b'\0'))
# Virtual memory address (8 bytes for AARCH64)
header.extend(struct.pack("<Q", info['vma']))
# Uncompressed size (4 bytes)
header.extend(struct.pack("<I", info['size']))
# Compression algorithm (1 byte)
algo_id = {
'none': 0,
'zlib': 1,
'lzma': 2,
'bz2': 3,
'delta': 4,
'word_delta': 5,
'dictionary': 6,
'aarch64_code': 7
}.get(info['algo'], 0)
header.append(algo_id)
# Offset to compressed data (4 bytes) - will be filled in later
offset_pos = len(header)
header.extend(b'\0\0\0\0')
# Compressed size (4 bytes)
header.extend(struct.pack("<I", info['comp_size']))
# Store the compressed data
data_offset = len(section_data)
section_data.extend(info['comp_data'])
# Update the offset in the header
struct.pack_into("<I", header, offset_pos, len(header) + data_offset)
# If including a decompressor, add it here
if include_decompressor:
# Create a minimal decompressor in C
decompressor_code = generate_aarch64_decompressor_code()
decompressor_size = len(decompressor_code)
# Add decompressor info to header
header.extend(struct.pack("<I", decompressor_size))
header.extend(decompressor_code)
else:
# No decompressor
header.extend(struct.pack("<I", 0))
# Write the final binary
with open(output_file, 'wb') as f:
f.write(header)
f.write(section_data)
print(f"\nCreated compressed binary: {output_file}")
print(f" Header size: {len(header)} bytes")
print(f" Data size: {len(section_data)} bytes")
print(f" Total size: {len(header) + len(section_data)} bytes")
# Create JSON metadata file
metadata = {
'architecture': 'aarch64',
'entry_point': entry_point,
'sections': []
}
for name, info in alloc_sections.items():
metadata['sections'].append({
'name': name,
'vma': info['vma'],
'size': info['size'],
'comp_size': info['comp_size'],
'algo': info['algo'],
'is_code': info['is_code']
})
meta_file = f"{output_file}.meta.json"
with open(meta_file, 'w') as f:
json.dump(metadata, f, indent=2)
print(f"Created metadata file: {meta_file}")
def generate_aarch64_decompressor_code():
"""Generate a minimal AARCH64 decompressor."""
# This would be a simple ARM assembly decompressor
# For simplicity, we'll just return placeholder bytes
return b'AARCH64_DECOMPRESSOR_PLACEHOLDER'
def create_aarch64_decompressor(output_file, section_results, entry_point, prefix="aarch64-linux-gnu-"):
"""Create a standalone decompressor executable for AARCH64."""
# This would generate a C file with the decompression logic
# For now, we'll just create a simple C file stub
c_code = """
#include <stdio.h>
#include <stdlib.h>
int main(int argc, char **argv) {
if (argc != 3) {
printf("Usage: %s input.bin output.bin\\n", argv[0]);
return 1;
}
printf("AARCH64 Decompressor stub - would decompress %s to %s\\n", argv[1], argv[2]);
return 0;
}
"""
c_file = f"{output_file}.c"
with open(c_file, 'w') as f:
f.write(c_code)
# Compile with AARCH64 toolchain
gcc_cmd = f"{prefix}gcc -o {output_file} {c_file}"
try:
run_command(gcc_cmd)
print(f"Created AARCH64 decompressor: {output_file}")
except:
print(f"Created decompressor source: {c_file}")
print(f"Compile with: {gcc_cmd}")
def main():
parser = argparse.ArgumentParser(description='Analyze AARCH64 ELF file sections and create optimized binary')
parser.add_argument('elf_file', help='Path to the AARCH64 ELF file to analyze')
parser.add_argument('-o', '--output', help='Output file for compressed binary')
parser.add_argument('-c', '--create-binary', action='store_true', help='Create compressed binary')
parser.add_argument('-d', '--decompressor', action='store_true', help='Include decompressor in binary')
parser.add_argument('-p', '--prefix', default='aarch64-linux-gnu-', help='Toolchain prefix (default: aarch64-linux-gnu-)')
args = parser.parse_args()
if not os.path.exists(args.elf_file):
print(f"Error: File {args.elf_file} not found")
sys.exit(1)
output_file = args.output or f"{args.elf_file}.compressed.bin"
analyze_elf(args.elf_file, args.create_binary, output_file, args.decompressor, args.prefix)
if args.create_binary and args.decompressor:
create_aarch64_decompressor(f"{output_file}.decomp", None, None, args.prefix)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment