simplifying hintable nop analysis in IDA

fthn.py

# original hint-proj proj was private, but seems like general idea is public now.
# i'm aware that some idiots who had no idea about original project gonna use it in own purposes.
#
# here is a simple plugin to prevent these "attempts".
# code is super basic, as algos. you can improve it. hf.

import idaapi
import idautils
import ida_funcs
import ida_bytes
import idc
import ida_ida

try:
    from iced_x86 import Decoder, Instruction, Mnemonic
except ImportError:
    # cmd -> pip install iced_x86.
    print("[-] iced_x86 module is not available in IDAPython environment")
    raise

def recover_instruction(bitness, ea):
    
    # max instruction size is 15 bytes on the x86_64.
    data = ida_bytes.get_bytes(ea, 15)
    if not data:
        return False

    # iced seems to be a good choice for now.
    # you can replace it with whatever you want to.
    decoder = Decoder(bitness, data)
    insn = Instruction()
    
    try:
        decoder.decode_out(insn)
    except Exception as e:
        print(f"[-] failed to decode instruction at 0x{ea:X}, reason = {e}")
        return False
    
    # let's be sure that it's an actual hintable.
    if insn.mnemonic != Mnemonic.RESERVEDNOP:
        return False

    # patching..
    length = insn.len # small patch for prefixes.
    for i in range(length):
        ida_bytes.patch_byte(ea + i, 0x90)
        
    idc.create_insn(ea)
    print(f"[+] recognized {insn:m} at 0x{ea:X}, end = 0x{ea + length:X}")
    
    return True
    
def reload_segment(targets):
    
    # a little trick to avoid function analysis overflow.
    last_reload_end = 0;
        
    # now, let's reload a code segment.
    for ea in targets:
        
        # if function is already exist at this address, then let's try to re-analyze it.
        fn = ida_funcs.get_func(ea)
        if fn is not None and ea > last_reload_end:
            
            ida_funcs.reanalyze_function(fn)
            print(f"[+] function {ida_funcs.get_func_name(ea)} was re-analyzed")
            
            # update last analyzed function bounds.
            last_reload_end = idc.get_func_attr(ea, idc.FUNCATTR_END)
            return
            
        # otherwise, let's try to create a new one.
        if ida_funcs.add_func(ea, idc.BADADDR):
            print(f"[+] function {ida_funcs.get_func_name(ea)} was created, size = 0x{ida_funcs.calc_func_size(ida_funcs.get_func(ea)):X}")
        else:
            print(f"[-] can't create function at 0x{ea:X}")

def main():
    
    if ida_ida.inf_is_32bit_exactly():
        bitness = 32
    else:
        bitness = 64
    
    for seg_ea in idautils.Segments():
        
        seg = idaapi.getseg(seg_ea)
        
        # we don't want to touch rdata and stuff...
        if seg is None or not (seg.perm & idaapi.SEGPERM_EXEC):
            continue
        
        seg_name = ida_segment.get_segm_name(seg);
        print(f"[!] processing segment {seg_name} (0x{seg.start_ea:X} -> 0x{seg.end_ea:X})...")
        
        targets = [];
        ea = seg.start_ea
        while ea < seg.end_ea:
            
            # need to follow two rules:
            # 1. bytes must be undefined, but in same time, processing section must have exec flag.
            # 2. since ida treats hint-nops as aligns sometimes, let's avoid false-positive stuff.
            flags = idaapi.get_full_flags(ea)
            if not ida_bytes.is_code(flags) and not ida_bytes.is_align(flags):
                if (recover_instruction(bitness, ea)):
                    targets.append(ea)

            # pretty simple and yet effective loop:
            # let's probe undefined bytes 1-by-1 or via disasm.length step.
            step = ida_bytes.get_item_size(ea)
            if step == 0:
                step = 1
                
            ea += step
            
        # now, we can reload (re-analyze) segment.
        #
        # we're doing two simple things here:
        # 1. found some broken instruction inside of func? let's reanalyze whole func one time.
        # 2. found some broken instruction inside of bin? let's try to create function on it.
        reload_segment(targets)
        
        if targets:
            print(f"[+] {len(targets)} instructions was recovered in segment {seg_name}!")
        else:
            print(f"[-] failed to find any hint nops in segment {seg_name}!")
    
if __name__:
    main()