Heavy refatorization.

This commit is contained in:
Krystian Bacławski
2013-06-13 22:19:33 +02:00
parent fcb9d51555
commit 7b4f968cc6
2 changed files with 488 additions and 348 deletions
+5 -306
View File
@@ -1,315 +1,14 @@
#!/usr/bin/env python2.7 -B
from collections import defaultdict
import struct
import sys
import textwrap
from objtools import util
from objtools.hunk import (GetHunkName, GetHunkExtName, GetHunkFlags, Hunk,
Header)
from objtools.hunk import HunkParser
class HunkParser(object):
def __init__(self, hunkfile):
self.data = hunkfile.read()
self.index = 0
def ReadLong(self):
data = struct.unpack_from('>I', self.data, self.index)[0]
self.index += 4
return data
def ReadWord(self):
data = struct.unpack_from('>H', self.data, self.index)[0]
self.index += 2
return data
def ReadInt(self):
data = struct.unpack_from('>i', self.data, self.index)[0]
self.index += 4
return data
def ReadBytes(self, n=None):
if n is None:
n = self.ReadLong() * 4
data = self.data[self.index:self.index + n]
self.index += n
return data
def ReadString(self, n=None):
return self.ReadBytes(n).strip('\0')
def ReadSymbol(self, length):
symbol = self.ReadString(length)
value = self.ReadInt()
return (symbol, value)
def ReadSymbols(self):
symbols = []
while True:
length = self.ReadLong() * 4
if not length:
break
symbols.append(self.ReadSymbol(length))
return symbols
def ReadSymbolRefs(self, length):
symbol = self.ReadString(length)
count = self.ReadLong()
refs = [self.ReadLong() for i in range(count)]
return (symbol, refs)
def ReadHunkExt(self):
hunks = defaultdict(list)
while True:
longs = self.ReadLong()
if not longs:
break
length = (longs & 0xffffff) * 4
extName = GetHunkExtName(longs >> 24)
if extName in ['EXT_DEF', 'EXT_ABS', 'EXT_REL']:
data = self.ReadSymbol(length)
elif extName in ['EXT_REF32', 'EXT_REF16', 'EXT_REF8',
'EXT_DEXT32', 'EXT_DEXT16', 'EXT_DEXT8']:
data = self.ReadSymbolRefs(length)
else:
raise NotImplementedError('%s not handled.' % extName)
hunks[extName].append(data)
return hunks
def ReadRelocs(self):
relocs = {}
while True:
longs = self.ReadLong()
if not longs:
break
hunkRef = self.ReadLong()
offsets = [self.ReadLong() for i in range(longs)]
relocs[hunkRef] = offsets
return relocs
def ReadShortRelocs(self):
start = self.index
relocs = {}
while True:
words = self.ReadWord()
if not words:
break
hunkRef = self.ReadWord()
offsets = [self.ReadWord() for i in range(words)]
relocs[hunkRef] = offsets
if (self.index - start) & 3:
self.index += 2
return relocs
def ReadHeader(self):
residents = []
while True:
longs = self.ReadLong()
if not longs:
break
residents.append(self.ReadString(longs * 4))
hunks = self.ReadLong()
first = self.ReadLong()
last = self.ReadLong()
specifiers = [self.ReadLong() for i in range(last - first + 1)]
return Header(residents, hunks, first, last, specifiers)
def ReadOverlay(self):
length = self.ReadLong() * 4
self.index += length + 4
return ''
def ReadIndex(self):
length = self.ReadLong() * 4
last = self.index + length
strsize = self.ReadWord()
strdata = self.ReadBytes(strsize)
names = {}
s = 0
while True:
e = strdata.find('\0', s, strsize)
if e == -1:
break
if e > s:
names[s] = strdata[s:e]
s = e + 1
units = []
while self.index < last:
unit_name = names[self.ReadWord()]
first_hunk = self.ReadWord() * 4
hunks_count = self.ReadWord()
hunks = []
for i in range(hunks_count):
h_name = names[self.ReadWord()]
h_size = self.ReadWord() * 4
h_type = self.ReadWord()
refs_count = self.ReadWord()
refs = []
for i in range(refs_count):
n = self.ReadWord()
try:
refs.append(names[n])
except KeyError:
refs.append(names[n + 1])
symbols_count = self.ReadWord()
symbols = []
for i in range(symbols_count):
s_name = names[self.ReadWord()]
s_value = self.ReadWord()
s_type = self.ReadWord()
symbols.append((s_name, s_value, s_type))
hunks.append((h_name, h_size, h_type, refs, symbols))
units.append((unit_name, first_hunk, hunks))
return units
def Parse(self):
executable = None
hunks = []
while self.index < len(self.data):
hunkId = self.ReadLong()
try:
flags = GetHunkFlags(hunkId)
name = GetHunkName(hunkId)
except ValueError:
print 'Parse error at position %d.' % self.index
raise
if executable is None:
executable = (name == 'HUNK_HEADER')
kind = 'binary'
if name in ['HUNK_END', 'HUNK_BREAK']:
data = None
kind = 'separator'
elif name == 'HUNK_EXT':
data = self.ReadHunkExt()
elif name == 'HUNK_SYMBOL':
data = self.ReadSymbols()
kind = 'symbols'
elif name == 'HUNK_HEADER':
data = self.ReadHeader()
kind = 'header'
elif name == 'HUNK_INDEX':
data = self.ReadIndex()
kind = 'index'
elif name in ['HUNK_NAME', 'HUNK_UNIT']:
data = self.ReadString()
kind = 'string'
elif name in ['HUNK_CODE', 'HUNK_PPC_CODE', 'HUNK_DATA', 'HUNK_DEBUG']:
data = self.ReadBytes()
elif name == 'HUNK_OVERLAY':
data = self.ReadOverlay()
elif name in ['HUNK_BSS', 'HUNK_LIB']:
data = self.ReadLong() * 4
kind = 'container'
elif name in ['HUNK_RELOC32', 'HUNK_RELOC16', 'HUNK_RELOC8']:
data = self.ReadRelocs()
kind = 'relocs'
elif name in ['HUNK_DREL32', 'HUNK_DREL16', 'HUNK_DREL8']:
if executable:
data = self.ReadShortRelocs()
else:
data = self.ReadRelocs()
kind = 'relocs'
else:
raise NotImplementedError('%s not handled.' % name)
hunks.append(Hunk(name, kind, data, flags))
return hunks
if __name__ == '__main__':
for path in sys.argv[1:]:
print 'Parsing "%s".' % path
with open(path) as hunkfile:
parser = HunkParser(hunkfile)
for hunk in parser.Parse():
print hunk.name
print ''
if hunk.kind == 'separator':
print ''
elif hunk.kind == 'header':
print hunk.data
elif hunk.name == 'HUNK_EXT':
for name, symbols in hunk.data.items():
print ' ', name
sl = max(len(s) for s, _ in symbols)
for symbol, value in symbols:
print ' ', symbol.ljust(sl, ' '), '=', value
elif hunk.kind == 'relocs':
for k, nums in hunk.data.items():
prefix = ' %d: ' % k
print textwrap.fill('[' + ', '.join(str(n) for n in nums) + ']',
width=68,
initial_indent=prefix,
subsequent_indent=' ' * (len(prefix) + 1))
elif hunk.kind == 'binary':
util.hexdump(hunk.data)
elif hunk.kind == 'symbols':
namelen = max(len(name) for name, _ in hunk.data) + 1
for name, offset in hunk.data:
print ' {0}: {1}'.format(name.ljust(namelen, ' '), offset)
elif hunk.kind == 'string':
print ' ', hunk.data
elif hunk.kind == 'index':
for u in hunk.data:
print ' ', 'UNIT', repr(u[0]), u[1]
for h in u[2]:
print ' ', GetHunkName(h[2]), repr(h[0]), h[1]
if h[3]:
print ' ', 'REFS'
for s in sorted(h[3]):
print ' ', s
if h[4]:
print ' ', 'DEFS'
l = max(len(s[0]) for s in h[4])
for s in sorted(h[4], key=lambda x: x[1]):
print ' ', s[0].ljust(l), '=', s[1]
print ''
else:
print ' ', repr(hunk.data)
for hunk in HunkParser(path):
hunk.dump()
print ''
+483 -42
View File
@@ -1,6 +1,11 @@
from collections import namedtuple
import os
import struct
import textwrap
from collections import defaultdict, namedtuple
from contextlib import contextmanager
import util
HunkMap = {
@@ -63,75 +68,511 @@ HunkFlagsMap = {
}
def GetHunkName(number):
number &= 0x1fffffff
for name, value in HunkMap.items():
if value == number:
return name
raise ValueError('Unknown Hunk: %d' % number)
Symbol = namedtuple('Symbol', 'name size refs')
def GetHunkExtName(number):
for name, value in HunkExtMap.items():
if value == number:
return name
class Hunk(object):
def __init__(self, type_):
self.type = type_
raise ValueError('Unknown HunkExt: %d' % number)
@staticmethod
def getType(number):
number &= 0x1fffffff
for name, value in HunkMap.items():
if value == number:
return name
raise ValueError('Unknown Hunk: %d' % number)
@staticmethod
def getFlags(number):
return [name for name, value in HunkFlagsMap.items() if value & number]
def GetHunkFlags(number):
flags = []
class HunkSep(Hunk):
@classmethod
def parse(cls, hf):
type_, _ = hf.readHunk('HUNK_END', 'HUNK_BREAK')
return cls(type_)
for name, value in HunkFlagsMap.items():
if value & number:
flags.append(name)
return flags
def dump(self):
print self.type
class Hunk(namedtuple('Hunk', 'name kind data flags')):
def __repr__(self):
class HunkStr(Hunk):
def __init__(self, type_, name=''):
Hunk.__init__(self, type_)
self.name = name
@classmethod
def parse(cls, hf):
type_, _ = hf.readHunk('HUNK_NAME', 'HUNK_UNIT')
return cls(type_, hf.readString())
def dump(self):
print self.type
print ' ' + repr(self.name)
class HunkBinary(Hunk):
def __init__(self, type_, flags=None, data=''):
Hunk.__init__(self, type_)
self.flags = flags or []
self.data = data
@classmethod
def parse(cls, hf):
type_, flags = hf.readHunk('HUNK_DATA', 'HUNK_CODE')
return cls(type_, flags, hf.readBytes())
def dump(self):
print '{0} {1}'.format(self.type, ', '.join(self.flags))
if self.data:
if self.flags:
flags = ', flags=%s' % '|'.join(self.flags)
else:
flags = ''
return '%s(%r%s)' % (self.name, self.data, flags)
util.hexdump(self.data)
else:
return self.name
print ' [empty]'
Header = namedtuple('Header', 'residents hunks first last specifiers')
class HunkDebug(Hunk):
# HUNK_DEBUG:
# magic-number=0x10b symtabsize strtabsize symtabdata
# [length=symtabsize] strtabdata [length=strtabsize] [pad bytes]
def __init__(self, data=''):
Hunk.__init__(self, 'HUNK_DEBUG')
self.data = data
@classmethod
def parse(cls, hf):
hf.readHunk('HUNK_DEBUG')
return cls(hf.readBytes())
def dump(self):
print self.type
util.hexdump(self.data)
class HunkFile(object):
class HunkOverlay(Hunk):
def __init__(self):
self._file = None
Hunk.__init__(self, 'HUNK_OVERLAY')
def open(self, path, mode='r'):
self._file = open(path, mode)
@classmethod
def parse(cls, hf):
hf.readHunk('HUNK_OVERLAY')
hf.skip(hf.readLong() * 4 + 4)
return cls()
def close(self):
self._file.close()
def readLong(self):
return struct.unpack_from('>I', self._file.read(4))[0]
class HunkBss(Hunk):
def __init__(self, flags=None, size=0):
Hunk.__init__(self, 'HUNK_BSS')
self.flags = flags or []
self.size = size
@classmethod
def parse(cls, hf):
_, flags = hf.readHunk('HUNK_BSS')
return cls(flags, hf.readLong() * 4)
def dump(self):
print self.type
print ' {0} bytes'.format(self.size)
class HunkLib(Hunk):
def __init__(self, size=0):
Hunk.__init__(self, 'HUNK_LIB')
self.size = size
@classmethod
def parse(cls, hf):
_, flags = hf.readHunk('HUNK_LIB')
return cls(hf.readLong() * 4)
def dump(self):
print self.type
class HunkReloc(Hunk):
def __init__(self, type_, relocs=None):
Hunk.__init__(self, type_)
self.relocs = relocs or {}
@classmethod
def parse(cls, hf):
type_, _ = hf.readHunk('HUNK_RELOC32', 'HUNK_RELOC16', 'HUNK_RELOC8',
'HUNK_DREL32', 'HUNK_DREL16', 'HUNK_DREL8')
if hf.type is 'executable' and type_ in ['HUNK_DREL32', 'HUNK_DREL16',
'HUNK_DREL8']:
relocs = hf.readShortRelocs()
else:
relocs = hf.readRelocs()
return cls(type_, relocs)
def dump(self):
print self.type
for k, nums in self.relocs.items():
prefix = ' %d: ' % k
print textwrap.fill('[' + ', '.join(str(n) for n in nums) + ']',
width=68, initial_indent=prefix,
subsequent_indent=' ' * (len(prefix) + 1))
class HunkSymbol(Hunk):
def __init__(self, symbols=None):
Hunk.__init__(self, 'HUNK_SYMBOL')
self.symbols = symbols or []
@classmethod
def parse(cls, hf):
hf.readHunk('HUNK_SYMBOL')
return cls(hf.readSymbols())
def dump(self):
print self.type
l = max(len(s.name) for s in self.symbols) + 1
for s in sorted(self.symbols, key=lambda s: s.name):
print ' {0}: {1}'.format(s.name.ljust(l, ' '), s.refs)
class HunkHeader(Hunk):
def __init__(self, residents=None, hunks=0, first=0, last=0,
specifiers=None):
Hunk.__init__(self, 'HUNK_HEADER')
self.residents = residents or []
self.hunks = hunks
self.first = first
self.last = last
self.specifiers = specifiers or []
@classmethod
def parse(cls, hf):
hf.readHunk('HUNK_HEADER')
residents = []
while True:
longs = hf.readLong()
if not longs:
break
residents.append(hf.readString(longs * 4))
hunks, first, last = hf.readLong(), hf.readLong(), hf.readLong()
specifiers = [hf.readLong() for i in range(last - first + 1)]
return cls(residents, hunks, first, last, specifiers)
def dump(self):
print self.type
print ' hunks={0}, first={1}, last={2}'.format(self.hunks, self.first,
self.last)
print ' residents : ' + repr(self.residents)
print ' specifiers : ' + repr(self.specifiers)
class HunkExt(Hunk):
def __init__(self, hunks=None):
Hunk.__init__(self, 'HUNK_EXT')
self.hunks = hunks or defaultdict(list)
@staticmethod
def getType(number):
for name, value in HunkExtMap.items():
if value == number:
return name
raise ValueError('Unknown HunkExt: %d' % number)
@classmethod
def parse(cls, hf):
hf.readHunk('HUNK_EXT')
hunks = defaultdict(list)
while True:
longs = hf.readLong()
if not longs:
break
length = (longs & 0xffffff) * 4
extName = HunkExt.getType(longs >> 24)
if extName in ['EXT_DEF', 'EXT_ABS', 'EXT_REL']:
symbol = hf.readSymbol(length)
elif extName in ['EXT_REF32', 'EXT_REF16', 'EXT_REF8',
'EXT_DEXT32', 'EXT_DEXT16', 'EXT_DEXT8']:
symbol = hf.readString(length)
count = hf.readLong()
refs = [hf.readLong() for i in range(count)]
symbol = Symbol(symbol, None, refs)
elif extName in ['EXT_COMMON']:
name = hf.readString(length)
size = hf.readLong()
refs_num = hf.readLong()
refs = [hf.readLong() for i in range(refs_num)]
symbol = Symbol(name, size, refs)
else:
raise NotImplementedError('%s not handled.' % extName)
hunks[extName].append(symbol)
return cls(hunks)
def dump(self):
print self.type
for name, symbols in self.hunks.items():
print ' ', name
sl = max(len(s.name) for s in symbols)
for symbol, size, value in symbols:
if value:
print ' ', symbol.ljust(sl, ' '), '=', value
else:
print ' ', symbol.ljust(sl, ' '), ':', size
class HunkIndex(Hunk):
def __init__(self, units=None):
Hunk.__init__(self, 'HUNK_INDEX')
self.units = units or []
@classmethod
def parse(cls, hf):
hf.readHunk('HUNK_INDEX')
length = hf.readLong() * 4
last = hf.tell() + length
strsize = hf.readWord()
strdata = hf.readBytes(strsize)
names = {}
s = 0
while True:
e = strdata.find('\0', s, strsize)
if e == -1:
names[s + 1] = strdata[s:]
break
if e > s:
names[s + 1] = strdata[s:e]
s = e + 1
units = []
while hf.tell() < last:
unit_name = names[hf.readWord()]
first_hunk = hf.readWord() * 4
hunks_count = hf.readWord()
hunks = []
for i in range(hunks_count):
h_name = names[hf.readWord()]
h_size = hf.readWord() * 4
h_type = hf.readWord()
refs_count = hf.readWord()
refs = []
for i in range(refs_count):
n = hf.readWord()
try:
refs.append(names[n])
except KeyError:
refs.append(names[n + 1])
symbols_count = hf.readWord()
symbols = []
for i in range(symbols_count):
s_name = names[hf.readWord()]
s_value = hf.readWord()
s_type = hf.readWord()
symbols.append((s_name, s_value, s_type))
hunks.append((h_name, h_size, h_type, refs, symbols))
units.append((unit_name, first_hunk, hunks))
return cls(units)
def dump(self):
print self.type
for u in self.units:
print ' ', 'UNIT', repr(u[0]), u[1]
for h in u[2]:
print ' ', Hunk.getType(h[2]), repr(h[0]), h[1]
if h[3]:
print ' ', 'REFS'
for s in sorted(h[3]):
print ' ', s
if h[4]:
print ' ', 'DEFS'
l = max(len(s[0]) for s in h[4])
for s in sorted(h[4], key=lambda x: x[1]):
print ' ', s[0].ljust(l), '=', s[1]
print ''
class HunkFile(file):
def __init__(self, *args, **kwargs):
file.__init__(self, *args, **kwargs)
self.size = os.stat(self.name).st_size
self.type = 'object'
@contextmanager
def rollback(self):
pos = self.tell()
yield self
self.seek(pos, os.SEEK_SET)
def readWord(self):
return struct.unpack_from('>H', self._file.read(2))[0]
return struct.unpack_from('>H', self.read(2))[0]
def readLong(self):
return struct.unpack_from('>I', self.read(4))[0]
def readInt(self):
return struct.unpack_from('>i', self._file.read(4))[0]
return struct.unpack_from('>i', self.read(4))[0]
def readBytes(self, n=None):
if n is None:
n = self.readLong() * 4
return self._file.read(n)
return self.read(n).strip('\0')
def readString(self, n=None):
return self.readBytes(n).strip('\0')
def readSymbol(self, length):
symbol = self.readString(length)
value = self.readInt()
return Symbol(symbol, None, value)
def readSymbols(self):
symbols = []
while True:
length = self.readLong() * 4
if not length:
break
symbols.append(self.readSymbol(length))
return symbols
def readHunk(self, *types):
hunkId = self.readLong()
hunkType = Hunk.getType(hunkId)
hunkFlags = Hunk.getFlags(hunkId)
if types:
if hunkType not in types:
raise ValueError('Unexpected hunk type: %s', hunkType)
return hunkType, hunkFlags
def readRelocs(self):
relocs = {}
while True:
longs = self.readLong()
if not longs:
break
hunkRef = self.readLong()
offsets = [self.readLong() for i in range(longs)]
relocs[hunkRef] = offsets
return relocs
def readShortRelocs(self):
start = self.tell()
relocs = {}
while True:
words = self.readWord()
if not words:
break
hunkRef = self.readWord()
offsets = [self.readWord() for i in range(words)]
relocs[hunkRef] = offsets
if (self.tell() - start) & 3:
self.skip(2)
return relocs
def skip(self, n):
self._file.seek(n, os.SEEK_CUR)
self.seek(n, os.SEEK_CUR)
def eof(self):
return self.tell() == self.size
HunkClassMap = {
'HUNK_END': HunkSep,
'HUNK_BREAK': HunkSep,
'HUNK_EXT': HunkExt,
'HUNK_SYMBOL': HunkSymbol,
'HUNK_HEADER': HunkHeader,
'HUNK_INDEX': HunkIndex,
'HUNK_NAME': HunkStr,
'HUNK_UNIT': HunkStr,
'HUNK_CODE': HunkBinary,
'HUNK_DATA': HunkBinary,
'HUNK_OVERLAY': HunkOverlay,
'HUNK_BSS': HunkBss,
'HUNK_LIB': HunkLib,
'HUNK_RELOC32': HunkReloc,
'HUNK_RELOC16': HunkReloc,
'HUNK_RELOC8': HunkReloc,
'HUNK_DREL32': HunkReloc,
'HUNK_DREL16': HunkReloc,
'HUNK_DREL8': HunkReloc,
'HUNK_DEBUG': HunkDebug
}
def HunkParser(path):
with HunkFile(path) as hf:
hunks = []
while not hf.eof():
with hf.rollback():
hunkId = hf.readLong()
type_ = Hunk.getType(hunkId)
if type_ is 'HUNK_HEADER':
hf.type = 'executable'
hunk = HunkClassMap.get(type_, None)
if not hunk:
raise NotImplementedError('%s not handled.' % type_)
try:
hunks.append(hunk.parse(hf))
except ValueError:
print 'Parse error at position 0x%x.' % hf.tell()
util.hexdump(hf.read())
return hunks