From 7b4f968cc660d52dbebe7e82d0c247a6adad58b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Bac=C5=82awski?= Date: Thu, 13 Jun 2013 22:19:33 +0200 Subject: [PATCH] Heavy refatorization. --- tools/dumphunks.py | 311 +----------------------- tools/objtools/hunk.py | 525 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 488 insertions(+), 348 deletions(-) diff --git a/tools/dumphunks.py b/tools/dumphunks.py index 92015ac..3b2ff34 100755 --- a/tools/dumphunks.py +++ b/tools/dumphunks.py @@ -1,315 +1,14 @@ #!/usr/bin/env python2.7 -B -from collections import defaultdict -import struct import sys -import textwrap -from objtools import util -from objtools.hunk import (GetHunkName, GetHunkExtName, GetHunkFlags, Hunk, - Header) +from objtools.hunk import HunkParser -class HunkParser(object): - def __init__(self, hunkfile): - self.data = hunkfile.read() - self.index = 0 - - def ReadLong(self): - data = struct.unpack_from('>I', self.data, self.index)[0] - self.index += 4 - return data - - def ReadWord(self): - data = struct.unpack_from('>H', self.data, self.index)[0] - self.index += 2 - return data - - def ReadInt(self): - data = struct.unpack_from('>i', self.data, self.index)[0] - self.index += 4 - return data - - def ReadBytes(self, n=None): - if n is None: - n = self.ReadLong() * 4 - data = self.data[self.index:self.index + n] - self.index += n - return data - - def ReadString(self, n=None): - return self.ReadBytes(n).strip('\0') - - def ReadSymbol(self, length): - symbol = self.ReadString(length) - value = self.ReadInt() - return (symbol, value) - - def ReadSymbols(self): - symbols = [] - - while True: - length = self.ReadLong() * 4 - - if not length: - break - - symbols.append(self.ReadSymbol(length)) - - return symbols - - def ReadSymbolRefs(self, length): - symbol = self.ReadString(length) - count = self.ReadLong() - refs = [self.ReadLong() for i in range(count)] - return (symbol, refs) - - def ReadHunkExt(self): - hunks = defaultdict(list) - - while True: - longs = self.ReadLong() - - if not longs: - break - - length = (longs & 0xffffff) * 4 - extName = GetHunkExtName(longs >> 24) - - if extName in ['EXT_DEF', 'EXT_ABS', 'EXT_REL']: - data = self.ReadSymbol(length) - elif extName in ['EXT_REF32', 'EXT_REF16', 'EXT_REF8', - 'EXT_DEXT32', 'EXT_DEXT16', 'EXT_DEXT8']: - data = self.ReadSymbolRefs(length) - else: - raise NotImplementedError('%s not handled.' % extName) - - hunks[extName].append(data) - - return hunks - - def ReadRelocs(self): - relocs = {} - - while True: - longs = self.ReadLong() - - if not longs: - break - - hunkRef = self.ReadLong() - offsets = [self.ReadLong() for i in range(longs)] - relocs[hunkRef] = offsets - - return relocs - - def ReadShortRelocs(self): - start = self.index - relocs = {} - - while True: - words = self.ReadWord() - - if not words: - break - - hunkRef = self.ReadWord() - offsets = [self.ReadWord() for i in range(words)] - relocs[hunkRef] = offsets - - if (self.index - start) & 3: - self.index += 2 - - return relocs - - def ReadHeader(self): - residents = [] - - while True: - longs = self.ReadLong() - - if not longs: - break - - residents.append(self.ReadString(longs * 4)) - - hunks = self.ReadLong() - first = self.ReadLong() - last = self.ReadLong() - specifiers = [self.ReadLong() for i in range(last - first + 1)] - - return Header(residents, hunks, first, last, specifiers) - - def ReadOverlay(self): - length = self.ReadLong() * 4 - self.index += length + 4 - return '' - - def ReadIndex(self): - length = self.ReadLong() * 4 - last = self.index + length - - strsize = self.ReadWord() - strdata = self.ReadBytes(strsize) - names = {} - - s = 0 - - while True: - e = strdata.find('\0', s, strsize) - - if e == -1: - break - - if e > s: - names[s] = strdata[s:e] - - s = e + 1 - - units = [] - - while self.index < last: - unit_name = names[self.ReadWord()] - first_hunk = self.ReadWord() * 4 - - hunks_count = self.ReadWord() - hunks = [] - - for i in range(hunks_count): - h_name = names[self.ReadWord()] - h_size = self.ReadWord() * 4 - h_type = self.ReadWord() - - refs_count = self.ReadWord() - refs = [] - - for i in range(refs_count): - n = self.ReadWord() - try: - refs.append(names[n]) - except KeyError: - refs.append(names[n + 1]) - - symbols_count = self.ReadWord() - symbols = [] - - for i in range(symbols_count): - s_name = names[self.ReadWord()] - s_value = self.ReadWord() - s_type = self.ReadWord() - symbols.append((s_name, s_value, s_type)) - - hunks.append((h_name, h_size, h_type, refs, symbols)) - - units.append((unit_name, first_hunk, hunks)) - - return units - - def Parse(self): - executable = None - hunks = [] - - while self.index < len(self.data): - hunkId = self.ReadLong() - - try: - flags = GetHunkFlags(hunkId) - name = GetHunkName(hunkId) - except ValueError: - print 'Parse error at position %d.' % self.index - raise - - if executable is None: - executable = (name == 'HUNK_HEADER') - - kind = 'binary' - - if name in ['HUNK_END', 'HUNK_BREAK']: - data = None - kind = 'separator' - elif name == 'HUNK_EXT': - data = self.ReadHunkExt() - elif name == 'HUNK_SYMBOL': - data = self.ReadSymbols() - kind = 'symbols' - elif name == 'HUNK_HEADER': - data = self.ReadHeader() - kind = 'header' - elif name == 'HUNK_INDEX': - data = self.ReadIndex() - kind = 'index' - elif name in ['HUNK_NAME', 'HUNK_UNIT']: - data = self.ReadString() - kind = 'string' - elif name in ['HUNK_CODE', 'HUNK_PPC_CODE', 'HUNK_DATA', 'HUNK_DEBUG']: - data = self.ReadBytes() - elif name == 'HUNK_OVERLAY': - data = self.ReadOverlay() - elif name in ['HUNK_BSS', 'HUNK_LIB']: - data = self.ReadLong() * 4 - kind = 'container' - elif name in ['HUNK_RELOC32', 'HUNK_RELOC16', 'HUNK_RELOC8']: - data = self.ReadRelocs() - kind = 'relocs' - elif name in ['HUNK_DREL32', 'HUNK_DREL16', 'HUNK_DREL8']: - if executable: - data = self.ReadShortRelocs() - else: - data = self.ReadRelocs() - kind = 'relocs' - else: - raise NotImplementedError('%s not handled.' % name) - - hunks.append(Hunk(name, kind, data, flags)) - - return hunks - if __name__ == '__main__': for path in sys.argv[1:]: print 'Parsing "%s".' % path - with open(path) as hunkfile: - parser = HunkParser(hunkfile) - for hunk in parser.Parse(): - print hunk.name + print '' - if hunk.kind == 'separator': - print '' - elif hunk.kind == 'header': - print hunk.data - elif hunk.name == 'HUNK_EXT': - for name, symbols in hunk.data.items(): - print ' ', name - sl = max(len(s) for s, _ in symbols) - for symbol, value in symbols: - print ' ', symbol.ljust(sl, ' '), '=', value - elif hunk.kind == 'relocs': - for k, nums in hunk.data.items(): - prefix = ' %d: ' % k - print textwrap.fill('[' + ', '.join(str(n) for n in nums) + ']', - width=68, - initial_indent=prefix, - subsequent_indent=' ' * (len(prefix) + 1)) - elif hunk.kind == 'binary': - util.hexdump(hunk.data) - elif hunk.kind == 'symbols': - namelen = max(len(name) for name, _ in hunk.data) + 1 - for name, offset in hunk.data: - print ' {0}: {1}'.format(name.ljust(namelen, ' '), offset) - elif hunk.kind == 'string': - print ' ', hunk.data - elif hunk.kind == 'index': - for u in hunk.data: - print ' ', 'UNIT', repr(u[0]), u[1] - for h in u[2]: - print ' ', GetHunkName(h[2]), repr(h[0]), h[1] - if h[3]: - print ' ', 'REFS' - for s in sorted(h[3]): - print ' ', s - if h[4]: - print ' ', 'DEFS' - l = max(len(s[0]) for s in h[4]) - for s in sorted(h[4], key=lambda x: x[1]): - print ' ', s[0].ljust(l), '=', s[1] - print '' - else: - print ' ', repr(hunk.data) + for hunk in HunkParser(path): + hunk.dump() + print '' diff --git a/tools/objtools/hunk.py b/tools/objtools/hunk.py index fc670ed..a18853c 100644 --- a/tools/objtools/hunk.py +++ b/tools/objtools/hunk.py @@ -1,6 +1,11 @@ -from collections import namedtuple import os import struct +import textwrap + +from collections import defaultdict, namedtuple +from contextlib import contextmanager + +import util HunkMap = { @@ -63,75 +68,511 @@ HunkFlagsMap = { } -def GetHunkName(number): - number &= 0x1fffffff - - for name, value in HunkMap.items(): - if value == number: - return name - - raise ValueError('Unknown Hunk: %d' % number) +Symbol = namedtuple('Symbol', 'name size refs') -def GetHunkExtName(number): - for name, value in HunkExtMap.items(): - if value == number: - return name +class Hunk(object): + def __init__(self, type_): + self.type = type_ - raise ValueError('Unknown HunkExt: %d' % number) + @staticmethod + def getType(number): + number &= 0x1fffffff + + for name, value in HunkMap.items(): + if value == number: + return name + + raise ValueError('Unknown Hunk: %d' % number) + + @staticmethod + def getFlags(number): + return [name for name, value in HunkFlagsMap.items() if value & number] -def GetHunkFlags(number): - flags = [] +class HunkSep(Hunk): + @classmethod + def parse(cls, hf): + type_, _ = hf.readHunk('HUNK_END', 'HUNK_BREAK') + return cls(type_) - for name, value in HunkFlagsMap.items(): - if value & number: - flags.append(name) - - return flags + def dump(self): + print self.type -class Hunk(namedtuple('Hunk', 'name kind data flags')): - def __repr__(self): +class HunkStr(Hunk): + def __init__(self, type_, name=''): + Hunk.__init__(self, type_) + self.name = name + + @classmethod + def parse(cls, hf): + type_, _ = hf.readHunk('HUNK_NAME', 'HUNK_UNIT') + return cls(type_, hf.readString()) + + def dump(self): + print self.type + print ' ' + repr(self.name) + + +class HunkBinary(Hunk): + def __init__(self, type_, flags=None, data=''): + Hunk.__init__(self, type_) + self.flags = flags or [] + self.data = data + + @classmethod + def parse(cls, hf): + type_, flags = hf.readHunk('HUNK_DATA', 'HUNK_CODE') + return cls(type_, flags, hf.readBytes()) + + def dump(self): + print '{0} {1}'.format(self.type, ', '.join(self.flags)) if self.data: - if self.flags: - flags = ', flags=%s' % '|'.join(self.flags) - else: - flags = '' - return '%s(%r%s)' % (self.name, self.data, flags) + util.hexdump(self.data) else: - return self.name + print ' [empty]' -Header = namedtuple('Header', 'residents hunks first last specifiers') +class HunkDebug(Hunk): + # HUNK_DEBUG: + # magic-number=0x10b symtabsize strtabsize symtabdata + # [length=symtabsize] strtabdata [length=strtabsize] [pad bytes] + + def __init__(self, data=''): + Hunk.__init__(self, 'HUNK_DEBUG') + self.data = data + + @classmethod + def parse(cls, hf): + hf.readHunk('HUNK_DEBUG') + return cls(hf.readBytes()) + + def dump(self): + print self.type + util.hexdump(self.data) -class HunkFile(object): +class HunkOverlay(Hunk): def __init__(self): - self._file = None + Hunk.__init__(self, 'HUNK_OVERLAY') - def open(self, path, mode='r'): - self._file = open(path, mode) + @classmethod + def parse(cls, hf): + hf.readHunk('HUNK_OVERLAY') + hf.skip(hf.readLong() * 4 + 4) + return cls() - def close(self): - self._file.close() - def readLong(self): - return struct.unpack_from('>I', self._file.read(4))[0] +class HunkBss(Hunk): + def __init__(self, flags=None, size=0): + Hunk.__init__(self, 'HUNK_BSS') + self.flags = flags or [] + self.size = size + + @classmethod + def parse(cls, hf): + _, flags = hf.readHunk('HUNK_BSS') + return cls(flags, hf.readLong() * 4) + + def dump(self): + print self.type + print ' {0} bytes'.format(self.size) + + +class HunkLib(Hunk): + def __init__(self, size=0): + Hunk.__init__(self, 'HUNK_LIB') + self.size = size + + @classmethod + def parse(cls, hf): + _, flags = hf.readHunk('HUNK_LIB') + return cls(hf.readLong() * 4) + + def dump(self): + print self.type + + +class HunkReloc(Hunk): + def __init__(self, type_, relocs=None): + Hunk.__init__(self, type_) + self.relocs = relocs or {} + + @classmethod + def parse(cls, hf): + type_, _ = hf.readHunk('HUNK_RELOC32', 'HUNK_RELOC16', 'HUNK_RELOC8', + 'HUNK_DREL32', 'HUNK_DREL16', 'HUNK_DREL8') + + if hf.type is 'executable' and type_ in ['HUNK_DREL32', 'HUNK_DREL16', + 'HUNK_DREL8']: + relocs = hf.readShortRelocs() + else: + relocs = hf.readRelocs() + + return cls(type_, relocs) + + def dump(self): + print self.type + for k, nums in self.relocs.items(): + prefix = ' %d: ' % k + print textwrap.fill('[' + ', '.join(str(n) for n in nums) + ']', + width=68, initial_indent=prefix, + subsequent_indent=' ' * (len(prefix) + 1)) + + +class HunkSymbol(Hunk): + def __init__(self, symbols=None): + Hunk.__init__(self, 'HUNK_SYMBOL') + self.symbols = symbols or [] + + @classmethod + def parse(cls, hf): + hf.readHunk('HUNK_SYMBOL') + return cls(hf.readSymbols()) + + def dump(self): + print self.type + + l = max(len(s.name) for s in self.symbols) + 1 + + for s in sorted(self.symbols, key=lambda s: s.name): + print ' {0}: {1}'.format(s.name.ljust(l, ' '), s.refs) + + +class HunkHeader(Hunk): + def __init__(self, residents=None, hunks=0, first=0, last=0, + specifiers=None): + Hunk.__init__(self, 'HUNK_HEADER') + self.residents = residents or [] + self.hunks = hunks + self.first = first + self.last = last + self.specifiers = specifiers or [] + + @classmethod + def parse(cls, hf): + hf.readHunk('HUNK_HEADER') + + residents = [] + + while True: + longs = hf.readLong() + + if not longs: + break + + residents.append(hf.readString(longs * 4)) + + hunks, first, last = hf.readLong(), hf.readLong(), hf.readLong() + specifiers = [hf.readLong() for i in range(last - first + 1)] + + return cls(residents, hunks, first, last, specifiers) + + def dump(self): + print self.type + print ' hunks={0}, first={1}, last={2}'.format(self.hunks, self.first, + self.last) + print ' residents : ' + repr(self.residents) + print ' specifiers : ' + repr(self.specifiers) + + +class HunkExt(Hunk): + def __init__(self, hunks=None): + Hunk.__init__(self, 'HUNK_EXT') + self.hunks = hunks or defaultdict(list) + + @staticmethod + def getType(number): + for name, value in HunkExtMap.items(): + if value == number: + return name + + raise ValueError('Unknown HunkExt: %d' % number) + + @classmethod + def parse(cls, hf): + hf.readHunk('HUNK_EXT') + + hunks = defaultdict(list) + + while True: + longs = hf.readLong() + + if not longs: + break + + length = (longs & 0xffffff) * 4 + extName = HunkExt.getType(longs >> 24) + + if extName in ['EXT_DEF', 'EXT_ABS', 'EXT_REL']: + symbol = hf.readSymbol(length) + elif extName in ['EXT_REF32', 'EXT_REF16', 'EXT_REF8', + 'EXT_DEXT32', 'EXT_DEXT16', 'EXT_DEXT8']: + symbol = hf.readString(length) + count = hf.readLong() + refs = [hf.readLong() for i in range(count)] + symbol = Symbol(symbol, None, refs) + elif extName in ['EXT_COMMON']: + name = hf.readString(length) + size = hf.readLong() + refs_num = hf.readLong() + refs = [hf.readLong() for i in range(refs_num)] + symbol = Symbol(name, size, refs) + else: + raise NotImplementedError('%s not handled.' % extName) + + hunks[extName].append(symbol) + + return cls(hunks) + + def dump(self): + print self.type + + for name, symbols in self.hunks.items(): + print ' ', name + sl = max(len(s.name) for s in symbols) + for symbol, size, value in symbols: + if value: + print ' ', symbol.ljust(sl, ' '), '=', value + else: + print ' ', symbol.ljust(sl, ' '), ':', size + + +class HunkIndex(Hunk): + def __init__(self, units=None): + Hunk.__init__(self, 'HUNK_INDEX') + self.units = units or [] + + @classmethod + def parse(cls, hf): + hf.readHunk('HUNK_INDEX') + + length = hf.readLong() * 4 + last = hf.tell() + length + + strsize = hf.readWord() + strdata = hf.readBytes(strsize) + names = {} + + s = 0 + + while True: + e = strdata.find('\0', s, strsize) + + if e == -1: + names[s + 1] = strdata[s:] + break + + if e > s: + names[s + 1] = strdata[s:e] + + s = e + 1 + + units = [] + + while hf.tell() < last: + unit_name = names[hf.readWord()] + first_hunk = hf.readWord() * 4 + + hunks_count = hf.readWord() + hunks = [] + + for i in range(hunks_count): + h_name = names[hf.readWord()] + h_size = hf.readWord() * 4 + h_type = hf.readWord() + + refs_count = hf.readWord() + refs = [] + + for i in range(refs_count): + n = hf.readWord() + try: + refs.append(names[n]) + except KeyError: + refs.append(names[n + 1]) + + symbols_count = hf.readWord() + symbols = [] + + for i in range(symbols_count): + s_name = names[hf.readWord()] + s_value = hf.readWord() + s_type = hf.readWord() + symbols.append((s_name, s_value, s_type)) + + hunks.append((h_name, h_size, h_type, refs, symbols)) + + units.append((unit_name, first_hunk, hunks)) + + return cls(units) + + def dump(self): + print self.type + + for u in self.units: + print ' ', 'UNIT', repr(u[0]), u[1] + for h in u[2]: + print ' ', Hunk.getType(h[2]), repr(h[0]), h[1] + if h[3]: + print ' ', 'REFS' + for s in sorted(h[3]): + print ' ', s + if h[4]: + print ' ', 'DEFS' + l = max(len(s[0]) for s in h[4]) + for s in sorted(h[4], key=lambda x: x[1]): + print ' ', s[0].ljust(l), '=', s[1] + print '' + + +class HunkFile(file): + def __init__(self, *args, **kwargs): + file.__init__(self, *args, **kwargs) + + self.size = os.stat(self.name).st_size + self.type = 'object' + + @contextmanager + def rollback(self): + pos = self.tell() + yield self + self.seek(pos, os.SEEK_SET) def readWord(self): - return struct.unpack_from('>H', self._file.read(2))[0] + return struct.unpack_from('>H', self.read(2))[0] + + def readLong(self): + return struct.unpack_from('>I', self.read(4))[0] def readInt(self): - return struct.unpack_from('>i', self._file.read(4))[0] + return struct.unpack_from('>i', self.read(4))[0] def readBytes(self, n=None): if n is None: n = self.readLong() * 4 - return self._file.read(n) + return self.read(n).strip('\0') def readString(self, n=None): return self.readBytes(n).strip('\0') + def readSymbol(self, length): + symbol = self.readString(length) + value = self.readInt() + return Symbol(symbol, None, value) + + def readSymbols(self): + symbols = [] + + while True: + length = self.readLong() * 4 + + if not length: + break + + symbols.append(self.readSymbol(length)) + + return symbols + + def readHunk(self, *types): + hunkId = self.readLong() + hunkType = Hunk.getType(hunkId) + hunkFlags = Hunk.getFlags(hunkId) + + if types: + if hunkType not in types: + raise ValueError('Unexpected hunk type: %s', hunkType) + + return hunkType, hunkFlags + + def readRelocs(self): + relocs = {} + + while True: + longs = self.readLong() + + if not longs: + break + + hunkRef = self.readLong() + offsets = [self.readLong() for i in range(longs)] + relocs[hunkRef] = offsets + + return relocs + + def readShortRelocs(self): + start = self.tell() + relocs = {} + + while True: + words = self.readWord() + + if not words: + break + + hunkRef = self.readWord() + offsets = [self.readWord() for i in range(words)] + relocs[hunkRef] = offsets + + if (self.tell() - start) & 3: + self.skip(2) + + return relocs + def skip(self, n): - self._file.seek(n, os.SEEK_CUR) + self.seek(n, os.SEEK_CUR) + + def eof(self): + return self.tell() == self.size + + +HunkClassMap = { + 'HUNK_END': HunkSep, + 'HUNK_BREAK': HunkSep, + 'HUNK_EXT': HunkExt, + 'HUNK_SYMBOL': HunkSymbol, + 'HUNK_HEADER': HunkHeader, + 'HUNK_INDEX': HunkIndex, + 'HUNK_NAME': HunkStr, + 'HUNK_UNIT': HunkStr, + 'HUNK_CODE': HunkBinary, + 'HUNK_DATA': HunkBinary, + 'HUNK_OVERLAY': HunkOverlay, + 'HUNK_BSS': HunkBss, + 'HUNK_LIB': HunkLib, + 'HUNK_RELOC32': HunkReloc, + 'HUNK_RELOC16': HunkReloc, + 'HUNK_RELOC8': HunkReloc, + 'HUNK_DREL32': HunkReloc, + 'HUNK_DREL16': HunkReloc, + 'HUNK_DREL8': HunkReloc, + 'HUNK_DEBUG': HunkDebug +} + + +def HunkParser(path): + with HunkFile(path) as hf: + hunks = [] + + while not hf.eof(): + with hf.rollback(): + hunkId = hf.readLong() + + type_ = Hunk.getType(hunkId) + + if type_ is 'HUNK_HEADER': + hf.type = 'executable' + + hunk = HunkClassMap.get(type_, None) + + if not hunk: + raise NotImplementedError('%s not handled.' % type_) + + try: + hunks.append(hunk.parse(hf)) + except ValueError: + print 'Parse error at position 0x%x.' % hf.tell() + util.hexdump(hf.read()) + + return hunks