Created
April 13, 2025 19:25
-
-
Save praton1729/cdfb3bb92f5f7887a881f17fb7a20f02 to your computer and use it in GitHub Desktop.
Create a highly compressed binary from aarch64 baremetal elfs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
""" | |
Binary Section Analyzer and Compressor with Final Binary Generator for AARCH64 | |
This script analyzes an ELF file, extracts its sections, compresses them with | |
various algorithms, and creates a final compressed binary with metadata for decompression. | |
Specifically configured for AARCH64 architecture. | |
""" | |
import os | |
import sys | |
import subprocess | |
import zlib | |
import lzma | |
import bz2 | |
import tempfile | |
import shutil | |
import struct | |
import argparse | |
import json | |
import pickle | |
from collections import defaultdict, Counter | |
from pathlib import Path | |
import math | |
# Magic number for our custom compressed binary format | |
MAGIC = b'SECCOMP' | |
FORMAT_VERSION = 1 | |
def run_command(cmd): | |
"""Run a shell command and return the output.""" | |
try: | |
result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, | |
stderr=subprocess.PIPE, text=True, shell=True) | |
return result.stdout | |
except subprocess.CalledProcessError as e: | |
print(f"Error executing command: {cmd}") | |
print(f"Error: {e.stderr}") | |
sys.exit(1) | |
def extract_sections(elf_file, prefix="aarch64-linux-gnu-"): | |
"""Extract all sections from an ELF file and return a dictionary of their info.""" | |
sections = {} | |
# Get section information using AARCH64 tools | |
objdump_cmd = f"{prefix}objdump -h {elf_file}" | |
objdump_output = run_command(objdump_cmd) | |
# Parse the output | |
current_section = None | |
for line in objdump_output.splitlines(): | |
line = line.strip() | |
if not line: | |
continue | |
if line.startswith('Idx'): # Header line | |
continue | |
parts = line.split() | |
if len(parts) >= 7 and parts[0].isdigit(): | |
section_name = parts[1] | |
if section_name.startswith('.'): | |
size_hex = parts[2] | |
size = int(size_hex, 16) | |
vma = parts[3] | |
vma_int = int(vma, 16) | |
lma = parts[4] | |
file_off = parts[5] | |
flags = parts[6] | |
sections[section_name] = { | |
'size': size, | |
'vma': vma, | |
'vma_int': vma_int, | |
'lma': lma, | |
'file_off': file_off, | |
'flags': flags, | |
'alloc': 'ALLOC' in flags, | |
'data': None # Will store actual binary data later | |
} | |
# Extract sections that have ALLOC flag set | |
tmpdir = tempfile.mkdtemp() | |
try: | |
for section_name, info in sections.items(): | |
if info['size'] > 0: # Only extract non-empty sections | |
output_file = os.path.join(tmpdir, f"{section_name.replace('/', '_')}.bin") | |
objcopy_cmd = f"{prefix}objcopy -O binary --only-section={section_name} {elf_file} {output_file}" | |
run_command(objcopy_cmd) | |
# Read the extracted binary data | |
if os.path.exists(output_file) and os.path.getsize(output_file) > 0: | |
with open(output_file, 'rb') as f: | |
sections[section_name]['data'] = f.read() | |
else: | |
# For sections like .bss that don't have actual file content | |
sections[section_name]['data'] = b'\x00' * info['size'] if info['alloc'] else None | |
finally: | |
shutil.rmtree(tmpdir) | |
# Get entry point using AARCH64 readelf | |
readelf_cmd = f"{prefix}readelf -h {elf_file}" | |
readelf_output = run_command(readelf_cmd) | |
entry_point = None | |
for line in readelf_output.splitlines(): | |
if "Entry point address" in line: | |
entry_point = int(line.split(':')[1].strip(), 16) | |
break | |
return sections, entry_point | |
def analyze_section_entropy(data): | |
"""Calculate the entropy of a binary section.""" | |
if not data or len(data) == 0: | |
return 0.0 | |
# Count byte frequencies | |
freqs = Counter(data) | |
total = len(data) | |
# Calculate entropy | |
entropy = 0 | |
for count in freqs.values(): | |
probability = count / total | |
entropy -= probability * math.log2(probability) | |
return entropy | |
def find_repeating_patterns(data, min_length=4, max_length=32, step=4): | |
"""Find repeating byte patterns in the data.""" | |
if not data or len(data) < min_length: | |
return {} | |
patterns = defaultdict(int) | |
# Check for patterns of different lengths | |
for pattern_len in range(min_length, min(max_length, len(data)), step): | |
for i in range(len(data) - pattern_len + 1): | |
pattern = data[i:i+pattern_len] | |
patterns[pattern] += 1 | |
# Filter out patterns that don't repeat | |
repeating = {pattern: count for pattern, count in patterns.items() if count > 1} | |
# Sort by (count * length) to find the most valuable patterns | |
sorted_patterns = sorted( | |
repeating.items(), | |
key=lambda x: len(x[0]) * x[1], | |
reverse=True | |
) | |
return dict(sorted_patterns[:20]) # Return top 20 most valuable patterns | |
def simple_delta_encode(data): | |
"""Apply simple delta encoding to a byte array.""" | |
if not data or len(data) <= 1: | |
return data | |
delta_encoded = bytearray(len(data)) | |
delta_encoded[0] = data[0] | |
for i in range(1, len(data)): | |
delta_encoded[i] = (data[i] - data[i-1]) & 0xFF | |
return bytes(delta_encoded) | |
def simple_delta_decode(data): | |
"""Decode simple delta encoding.""" | |
if not data or len(data) <= 1: | |
return data | |
decoded = bytearray(len(data)) | |
decoded[0] = data[0] | |
for i in range(1, len(data)): | |
decoded[i] = (decoded[i-1] + data[i]) & 0xFF | |
return bytes(decoded) | |
def word_delta_encode(data): | |
"""Apply word-aligned delta encoding - optimized for AARCH64's 32-bit/64-bit instructions.""" | |
if len(data) < 4: | |
return data | |
# For AARCH64, we'll use 4-byte alignment | |
# (could also use 8 bytes for some instruction sets) | |
word_size = 4 | |
# Ensure data length is multiple of word_size by padding if needed | |
if len(data) % word_size != 0: | |
padded_data = bytearray(data) | |
padded_data.extend(b'\x00' * (word_size - (len(data) % word_size))) | |
data = bytes(padded_data) | |
result = bytearray() | |
# First word is stored as-is | |
result.extend(data[0:word_size]) | |
# Process remaining words | |
for i in range(word_size, len(data), word_size): | |
for j in range(word_size): | |
result.append(data[i+j] ^ data[i-word_size+j]) | |
return bytes(result) | |
def word_delta_decode(data): | |
"""Decode word-aligned delta encoding for AARCH64's word size.""" | |
if len(data) < 4: | |
return data | |
word_size = 4 # Using 4-byte words for AARCH64 | |
result = bytearray(len(data)) | |
# First word is copied as-is | |
result[0:word_size] = data[0:word_size] | |
# Process remaining words | |
for i in range(word_size, len(data), word_size): | |
for j in range(word_size): | |
result[i+j] = data[i+j] ^ result[i-word_size+j] | |
return bytes(result) | |
def aarch64_instruction_encode(data): | |
"""Special encoding for AARCH64 instructions, looking for common patterns.""" | |
if len(data) < 4: | |
return data | |
# AARCH64 uses fixed 4-byte instructions | |
# We can exploit patterns like: | |
# - Common opcodes | |
# - Register patterns | |
# - Small immediate values | |
# For now, a simplified approach: | |
result = bytearray() | |
# Check if it's likely code (high entropy but with patterns) | |
entropy = analyze_section_entropy(data) | |
if entropy < 6.0 or entropy > 7.5: | |
return data # Probably not instructions, use default | |
# Extract 4-byte chunks | |
chunks = [] | |
for i in range(0, len(data), 4): | |
if i + 4 <= len(data): | |
chunks.append(data[i:i+4]) | |
if not chunks: | |
return data | |
# Look for similar chunks (differing only in registers or immediates) | |
similarities = [] | |
for i in range(len(chunks)): | |
for j in range(i+1, len(chunks)): | |
# Count differing bytes | |
diff_count = sum(b1 != b2 for b1, b2 in zip(chunks[i], chunks[j])) | |
if diff_count <= 2: # Similar instructions | |
similarities.append((i, j, diff_count)) | |
# If we found many similarities, use delta encoding | |
if len(similarities) > len(chunks) // 4: | |
return word_delta_encode(data) | |
# Otherwise fall back to regular data | |
return data | |
def aarch64_instruction_decode(data): | |
"""Decode the AARCH64 instruction encoding.""" | |
# For now, we'll just call word_delta_decode since our encoder is simplified | |
return word_delta_decode(data) | |
def dictionary_encode(data): | |
"""Apply simple dictionary encoding to the data.""" | |
if len(data) < 16: | |
return data # Too small to be worth it | |
# Find repeating patterns | |
patterns = find_repeating_patterns(data, min_length=8, max_length=64) | |
if not patterns: | |
return data # No good patterns found | |
# Sort patterns by value (bytes saved) | |
sorted_patterns = sorted( | |
patterns.items(), | |
key=lambda x: (len(x[0]) - 2) * x[1], # Value minus overhead | |
reverse=True | |
) | |
# Use top patterns (limit dictionary size) | |
dictionary = [] | |
for pattern, _ in sorted_patterns[:16]: # Limit to 16 entries | |
if len(pattern) >= 8: # Only use if pattern is long enough | |
dictionary.append(pattern) | |
if not dictionary: | |
return data | |
# Encode the data | |
result = bytearray() | |
# First byte is dictionary size | |
result.append(len(dictionary)) | |
# Add dictionary entries (length + pattern) | |
for pattern in dictionary: | |
result.append(len(pattern)) | |
result.extend(pattern) | |
# Process the data | |
i = 0 | |
while i < len(data): | |
# Check if current position matches any pattern | |
found = False | |
for idx, pattern in enumerate(dictionary): | |
if i + len(pattern) <= len(data) and data[i:i+len(pattern)] == pattern: | |
# Found a match - use index (0x80 and above are dictionary references) | |
result.append(0x80 | idx) | |
i += len(pattern) | |
found = True | |
break | |
if not found: | |
# No match, copy the byte literally | |
result.append(data[i]) | |
i += 1 | |
# Return original if our encoding is larger | |
return bytes(result) if len(result) < len(data) else data | |
def dictionary_decode(data): | |
"""Decode dictionary-encoded data.""" | |
if not data or len(data) < 2: | |
return data | |
# First byte is dictionary size | |
dict_size = data[0] | |
if dict_size == 0 or dict_size > 16: | |
return data # Invalid dictionary size or not dictionary encoded | |
# Read dictionary | |
dictionary = [] | |
pos = 1 | |
for _ in range(dict_size): | |
if pos >= len(data): | |
return data # Invalid format | |
pattern_len = data[pos] | |
pos += 1 | |
if pos + pattern_len > len(data): | |
return data # Invalid format | |
pattern = data[pos:pos+pattern_len] | |
dictionary.append(pattern) | |
pos += pattern_len | |
# Decode the data | |
result = bytearray() | |
while pos < len(data): | |
byte = data[pos] | |
pos += 1 | |
if byte & 0x80: # Dictionary reference | |
idx = byte & 0x7F | |
if idx < len(dictionary): | |
result.extend(dictionary[idx]) | |
else: | |
return data # Invalid reference | |
else: | |
# Literal byte | |
result.append(byte) | |
return bytes(result) | |
def compress_with_algorithms(data, is_code=False): | |
"""Compress data with various algorithms and return results.""" | |
if not data: | |
return {} | |
results = {} | |
# Standard compression algorithms with their implementation | |
results['none'] = { | |
'size': len(data), | |
'data': data, | |
'encode': lambda x: x, | |
'decode': lambda x: x | |
} | |
# Standard compression algorithms | |
try: | |
compressed = zlib.compress(data) | |
results['zlib'] = { | |
'size': len(compressed), | |
'data': compressed, | |
'encode': zlib.compress, | |
'decode': zlib.decompress | |
} | |
except: | |
pass | |
try: | |
compressed = lzma.compress(data) | |
results['lzma'] = { | |
'size': len(compressed), | |
'data': compressed, | |
'encode': lzma.compress, | |
'decode': lzma.decompress | |
} | |
except: | |
pass | |
try: | |
compressed = bz2.compress(data) | |
results['bz2'] = { | |
'size': len(compressed), | |
'data': compressed, | |
'encode': bz2.compress, | |
'decode': bz2.decompress | |
} | |
except: | |
pass | |
# Simple delta encoding | |
if len(data) > 1: | |
try: | |
delta_encoded = simple_delta_encode(data) | |
compressed = zlib.compress(delta_encoded) | |
results['delta'] = { | |
'size': len(compressed), | |
'data': compressed, | |
'encode': lambda x: zlib.compress(simple_delta_encode(x)), | |
'decode': lambda x: simple_delta_decode(zlib.decompress(x)) | |
} | |
except: | |
pass | |
# Word-aligned delta - good for AARCH64 instructions | |
if len(data) >= 4: | |
try: | |
word_delta = word_delta_encode(data) | |
compressed = zlib.compress(word_delta) | |
results['word_delta'] = { | |
'size': len(compressed), | |
'data': compressed, | |
'encode': lambda x: zlib.compress(word_delta_encode(x)), | |
'decode': lambda x: word_delta_decode(zlib.decompress(x)) | |
} | |
except: | |
pass | |
# AARCH64-specific instruction encoding - only if code section | |
if is_code and len(data) >= 8: | |
try: | |
aarch_encoded = aarch64_instruction_encode(data) | |
compressed = zlib.compress(aarch_encoded) | |
results['aarch64_code'] = { | |
'size': len(compressed), | |
'data': compressed, | |
'encode': lambda x: zlib.compress(aarch64_instruction_encode(x)), | |
'decode': lambda x: aarch64_instruction_decode(zlib.decompress(x)) | |
} | |
except: | |
pass | |
# Dictionary-based approach | |
if len(data) > 32: | |
try: | |
dict_encoded = dictionary_encode(data) | |
if len(dict_encoded) < len(data): | |
compressed = zlib.compress(dict_encoded) | |
results['dictionary'] = { | |
'size': len(compressed), | |
'data': compressed, | |
'encode': lambda x: zlib.compress(dictionary_encode(x)), | |
'decode': lambda x: dictionary_decode(zlib.decompress(x)) | |
} | |
except: | |
pass | |
return results | |
def analyze_elf(elf_file, create_binary=False, output_file=None, decompressor=False, prefix="aarch64-linux-gnu-"): | |
"""Analyze an ELF file's sections and compression potential.""" | |
print(f"Analyzing AARCH64 ELF file: {elf_file}") | |
# Extract sections using AARCH64 tools | |
sections, entry_point = extract_sections(elf_file, prefix) | |
# Analyze and compress each section | |
results = [] | |
print("\nSection Analysis:") | |
print("-" * 80) | |
print(f"{'Section':<20} {'Size':<10} {'Entropy':<10} {'Best Algorithm':<16} {'Comp Size':<10} {'Ratio':<10}") | |
print("-" * 80) | |
section_results = {} | |
for name, info in sorted(sections.items(), key=lambda x: x[1]['size'], reverse=True): | |
data = info['data'] | |
if data is None or len(data) == 0: | |
continue | |
# Skip non-ALLOC sections if we're creating a binary | |
if create_binary and not info['alloc']: | |
continue | |
orig_size = len(data) | |
entropy = analyze_section_entropy(data) | |
# Check if this is likely a code section | |
is_code = name in ('.text', '.init', '.fini', '.plt') or 'CODE' in info['flags'] | |
compression_results = compress_with_algorithms(data, is_code) | |
# Find best compression | |
best_algo = min(compression_results.items(), key=lambda x: x[1]['size']) | |
algo_name = best_algo[0] | |
best_size = best_algo[1]['size'] | |
best_ratio = best_size / orig_size if orig_size > 0 else 1.0 | |
# Store result | |
section_results[name] = { | |
'name': name, | |
'size': orig_size, | |
'entropy': entropy, | |
'vma': info['vma_int'], | |
'alloc': info['alloc'], | |
'flags': info['flags'], | |
'is_code': is_code, | |
'algo': algo_name, | |
'comp_size': best_size, | |
'comp_data': best_algo[1]['data'], | |
'encode': best_algo[1]['encode'], | |
'decode': best_algo[1]['decode'], | |
'ratio': best_ratio | |
} | |
print(f"{name:<20} {orig_size:<10} {entropy:<10.2f} {algo_name:<16} {best_size:<10} {best_ratio:<10.2f}") | |
results.append(section_results[name]) | |
# Print summary | |
print("\nCompression Summary:") | |
print("-" * 80) | |
# Group sections by type | |
section_types = defaultdict(list) | |
for r in results: | |
# Determine section type based on name and flags | |
if r['is_code']: | |
section_type = 'Code' | |
elif r['name'] in ('.data', '.rodata'): | |
section_type = 'Data' | |
elif r['name'].startswith('.debug'): | |
section_type = 'Debug' | |
else: | |
section_type = 'Other' | |
section_types[section_type].append(r) | |
# Print results by section type | |
for section_type, sections in section_types.items(): | |
total_size = sum(s['size'] for s in sections) | |
best_size = sum(s['comp_size'] for s in sections) | |
print(f"{section_type} sections:") | |
print(f" Total size: {total_size} bytes") | |
print(f" Compressed size: {best_size} bytes") | |
print(f" Overall ratio: {best_size/total_size:.2f}") | |
# Best algorithms per type | |
algo_counts = Counter(s['algo'] for s in sections) | |
best_algos = algo_counts.most_common(2) | |
print(f" Best algorithms: {', '.join(f'{algo} ({count})' for algo, count in best_algos)}") | |
print() | |
# Create compressed binary if requested | |
if create_binary and output_file: | |
create_compressed_binary(output_file, section_results, entry_point, decompressor) | |
return section_results, entry_point | |
def create_compressed_binary(output_file, section_results, entry_point, include_decompressor=False): | |
"""Create a compressed binary file using the best compression for each section.""" | |
# Filter to only include ALLOC sections | |
alloc_sections = {name: info for name, info in section_results.items() if info['alloc']} | |
# Create header | |
header = bytearray() | |
# Magic number | |
header.extend(MAGIC) | |
# Format version (1 byte) | |
header.append(FORMAT_VERSION) | |
# Entry point (8 bytes for AARCH64 to support full 64-bit addresses) | |
header.extend(struct.pack("<Q", entry_point)) | |
# Number of sections (2 bytes) | |
header.extend(struct.pack("<H", len(alloc_sections))) | |
# Section directory | |
section_data = bytearray() | |
for name, info in sorted(alloc_sections.items(), key=lambda x: x[1]['vma']): | |
# Section name (null-terminated, max 16 chars) | |
name_bytes = name.encode('ascii')[:15] + b'\0' | |
header.extend(name_bytes.ljust(16, b'\0')) | |
# Virtual memory address (8 bytes for AARCH64) | |
header.extend(struct.pack("<Q", info['vma'])) | |
# Uncompressed size (4 bytes) | |
header.extend(struct.pack("<I", info['size'])) | |
# Compression algorithm (1 byte) | |
algo_id = { | |
'none': 0, | |
'zlib': 1, | |
'lzma': 2, | |
'bz2': 3, | |
'delta': 4, | |
'word_delta': 5, | |
'dictionary': 6, | |
'aarch64_code': 7 | |
}.get(info['algo'], 0) | |
header.append(algo_id) | |
# Offset to compressed data (4 bytes) - will be filled in later | |
offset_pos = len(header) | |
header.extend(b'\0\0\0\0') | |
# Compressed size (4 bytes) | |
header.extend(struct.pack("<I", info['comp_size'])) | |
# Store the compressed data | |
data_offset = len(section_data) | |
section_data.extend(info['comp_data']) | |
# Update the offset in the header | |
struct.pack_into("<I", header, offset_pos, len(header) + data_offset) | |
# If including a decompressor, add it here | |
if include_decompressor: | |
# Create a minimal decompressor in C | |
decompressor_code = generate_aarch64_decompressor_code() | |
decompressor_size = len(decompressor_code) | |
# Add decompressor info to header | |
header.extend(struct.pack("<I", decompressor_size)) | |
header.extend(decompressor_code) | |
else: | |
# No decompressor | |
header.extend(struct.pack("<I", 0)) | |
# Write the final binary | |
with open(output_file, 'wb') as f: | |
f.write(header) | |
f.write(section_data) | |
print(f"\nCreated compressed binary: {output_file}") | |
print(f" Header size: {len(header)} bytes") | |
print(f" Data size: {len(section_data)} bytes") | |
print(f" Total size: {len(header) + len(section_data)} bytes") | |
# Create JSON metadata file | |
metadata = { | |
'architecture': 'aarch64', | |
'entry_point': entry_point, | |
'sections': [] | |
} | |
for name, info in alloc_sections.items(): | |
metadata['sections'].append({ | |
'name': name, | |
'vma': info['vma'], | |
'size': info['size'], | |
'comp_size': info['comp_size'], | |
'algo': info['algo'], | |
'is_code': info['is_code'] | |
}) | |
meta_file = f"{output_file}.meta.json" | |
with open(meta_file, 'w') as f: | |
json.dump(metadata, f, indent=2) | |
print(f"Created metadata file: {meta_file}") | |
def generate_aarch64_decompressor_code(): | |
"""Generate a minimal AARCH64 decompressor.""" | |
# This would be a simple ARM assembly decompressor | |
# For simplicity, we'll just return placeholder bytes | |
return b'AARCH64_DECOMPRESSOR_PLACEHOLDER' | |
def create_aarch64_decompressor(output_file, section_results, entry_point, prefix="aarch64-linux-gnu-"): | |
"""Create a standalone decompressor executable for AARCH64.""" | |
# This would generate a C file with the decompression logic | |
# For now, we'll just create a simple C file stub | |
c_code = """ | |
#include <stdio.h> | |
#include <stdlib.h> | |
int main(int argc, char **argv) { | |
if (argc != 3) { | |
printf("Usage: %s input.bin output.bin\\n", argv[0]); | |
return 1; | |
} | |
printf("AARCH64 Decompressor stub - would decompress %s to %s\\n", argv[1], argv[2]); | |
return 0; | |
} | |
""" | |
c_file = f"{output_file}.c" | |
with open(c_file, 'w') as f: | |
f.write(c_code) | |
# Compile with AARCH64 toolchain | |
gcc_cmd = f"{prefix}gcc -o {output_file} {c_file}" | |
try: | |
run_command(gcc_cmd) | |
print(f"Created AARCH64 decompressor: {output_file}") | |
except: | |
print(f"Created decompressor source: {c_file}") | |
print(f"Compile with: {gcc_cmd}") | |
def main(): | |
parser = argparse.ArgumentParser(description='Analyze AARCH64 ELF file sections and create optimized binary') | |
parser.add_argument('elf_file', help='Path to the AARCH64 ELF file to analyze') | |
parser.add_argument('-o', '--output', help='Output file for compressed binary') | |
parser.add_argument('-c', '--create-binary', action='store_true', help='Create compressed binary') | |
parser.add_argument('-d', '--decompressor', action='store_true', help='Include decompressor in binary') | |
parser.add_argument('-p', '--prefix', default='aarch64-linux-gnu-', help='Toolchain prefix (default: aarch64-linux-gnu-)') | |
args = parser.parse_args() | |
if not os.path.exists(args.elf_file): | |
print(f"Error: File {args.elf_file} not found") | |
sys.exit(1) | |
output_file = args.output or f"{args.elf_file}.compressed.bin" | |
analyze_elf(args.elf_file, args.create_binary, output_file, args.decompressor, args.prefix) | |
if args.create_binary and args.decompressor: | |
create_aarch64_decompressor(f"{output_file}.decomp", None, None, args.prefix) | |
if __name__ == "__main__": | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment