Skip to content

Instantly share code, notes, and snippets.

@atomizer

atomizer/extract.py

Last active January 17, 2019 19:04
Download ZIP

Revisions

  1. atomizer revised this gist Sep 25, 2013. 1 changed file with 3 additions and 2 deletions.
    5 changes: 3 additions & 2 deletions readme.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,7 +1,8 @@
    depends on [swftools](http://swftools.org)

    use python 2.7 (might work on 2.6 or 2.5 if you install `argparse`)

    ### usage (`--help`)
    #### usage (`--help`)

    ```
    usage: extract.py [-h] [-t {production,testing}] [--test] [--prod]
    @@ -21,7 +22,7 @@ optional arguments:

    `-t` defaults to `testing`

    ### output
    #### output

    creates 3 subdirectories with:

  2. atomizer created this gist Sep 25, 2013.
    227 changes: 227 additions & 0 deletions extract.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,227 @@
    #!/usr/bin/python

    import re, os, sys
    import tempfile
    import shutil
    import argparse
    from subprocess import *
    from urllib2 import urlopen, HTTPError

    # for parsing swfdump output
    TAGRE = re.compile(r'\[(?P<type>\d\d\d)\]\s+(?P<len>\d+)\s+(?P<tag>\S+) defines id (?P<id>\d+)')
    EXPORTRE = re.compile(r'exports (?P<id>\d+) as "kabam.rotmg.assets.Embedded(Assets|Data)_(?P<name>\S+?)(CXML|Embed|_)')

    # to identify xml parts
    XMLRE = re.compile(r'(?:<\?[^>]*>)?\s*<(?P<tag>\w+)>(?P<cont>.*)</\1>', re.M + re.S)
    XMLHEADER = re.compile(r'<\?[^>]*>')

    TAGS = {
    'DEFINEBINARY': 'txt',
    'DEFINEBITSLOSSLESS2': 'png',
    }

    EXTRACTFLAGS = {
    'txt': 'b',
    'png': 'p'
    }

    HOSTS = {
    'testing': 'rotmgtesting.appspot.com',
    'production': 'realmofthemadgod.appspot.com'
    }

    tempdir = ''
    outdir = ''


    def die(reason=''):
    if reason:
    print reason
    if tempdir:
    os.chdir('/')
    shutil.rmtree(tempdir)
    sys.exit(1)


    def getversion(target):
    print 'reading version.txt'
    try:
    version = urlopen('http://' + HOSTS[target] + '/version.txt').read()
    except:
    die('failed to read version.txt')
    return version


    def download_swf(target, version=''):
    print 'downloading swf'
    if not version:
    version = getversion(target)
    name = 'AssembleeGameClient' + version + '.swf'
    try:
    b = urlopen('http://' + HOSTS[target] + '/' + name).read()
    except HTTPError as e:
    die('HTTP error: code %s, reason "%s"' % (e.code, e.reason))
    except:
    die('swf download failed')

    try:
    with open('temp.swf', 'wb') as f:
    f.write(b)
    except:
    die('failed to write temp.swf')

    return version


    ids = {} # 'png': [id1, id2, ...], ...
    names = {} # id: name


    def extract(swfname):
    print 'extracting'
    lines = Popen(['swfdump', swfname], stdout=PIPE).communicate()[0].splitlines()
    lines = [x.strip() for x in lines]

    for t in TAGS:
    ids[TAGS[t]] = []

    for l in lines:
    m = TAGRE.match(l)
    if m:
    m = m.groupdict()
    # parsing "DEFINE"
    if m['tag'] in TAGS:
    ids[TAGS[m['tag']]].append(m['id'])
    continue
    m = EXPORTRE.match(l)
    if not m:
    continue
    m = m.groupdict()
    # parsing "exports"
    names[m['id']] = m['name']

    for type in ids:
    for id in ids[type]:
    if id in names:
    name = names[id]
    else:
    name = id
    # extract the thing
    args = ['swfextract',
    '-' + EXTRACTFLAGS[type], id,
    '-o', name + '.' + type, swfname]
    Popen(args).wait()
    sys.stdout.write('.')
    print


    def writefile(path, what):
    with open(path, 'wb') as f:
    f.write(what)


    def dotxt():
    print 'dealing with txt',
    rev = {}
    sorted = []
    for id in ids['txt']:
    if id not in names:
    continue
    rev[names[id]] = id
    sorted = rev.keys()
    sorted.sort()

    # saving concatenated xml files based on root tag
    files = {}

    for name in sorted:
    fname = name + '.txt'
    with open(fname, 'rb') as f:
    s = f.read()
    m = re.match(XMLRE, s)
    if m is None:
    # detect empty xml
    if not len(s) or re.match(XMLHEADER, s) is not None:
    continue
    # 3d objects
    writefile(os.path.join(outdir, 'obj', name + '.obj'), s)
    else:
    # xml, write separate file and add to concatenated
    writefile(os.path.join(outdir, 'xml', name + '.xml'), s)
    t = m.group('tag')
    if t not in files:
    files[t] = open(os.path.join(outdir, t + '.xml'), 'wb')
    files[t].write('<' + t + '>')
    # prefix each file's content with its name
    files[t].write('\n\n<!' + '-- ' + name + ' -->\n\n')
    files[t].write(m.group('cont'))

    for t in files:
    files[t].write('\n</' + t + '>')
    files[t].close()
    print


    def dopng():
    print 'dealing with png',
    for id in ids['png']:
    if id not in names:
    continue
    name = names[id]
    shutil.copy(name + '.png', os.path.join(outdir, 'sheets'))


    if __name__ == '__main__':
    p = argparse.ArgumentParser()
    p.add_argument('-t', dest='target', choices=HOSTS.keys(), default='testing')
    p.add_argument('--test', dest='target', action='store_const', const='testing', help='same as "-t testing"')
    p.add_argument('--prod', dest='target', action='store_const', const='production', help='same as "-t production"')
    p.add_argument('-v', dest='version', help='download specific version (default: latest)')
    p.add_argument('-f', '--file', type=argparse.FileType('rb'), help='use local swf')
    p.add_argument('-d', '--dir', help='output directory (default: current directory)')
    p.add_argument('--archive', action='store_true',
    help='write output to a subdir "type-version". If exists, do nothing (no effect with -f)')

    args = p.parse_args()

    outdir = os.getcwd()
    if args.dir:
    outdir = os.path.abspath(args.dir)
    if not (os.path.exists(outdir) and os.path.isdir(outdir)):
    die('invalid output directory')


    tempdir = tempfile.mkdtemp()
    os.chdir(tempdir)

    if args.file:
    with open('temp.swf', 'wb') as f:
    f.write(args.file.read())
    else:
    if not args.version:
    args.version = getversion(args.target)
    if args.archive:
    outdir = os.path.join(outdir, args.target + '-' + args.version)
    if os.path.exists(outdir):
    die('archive already exists')
    try:
    os.mkdir(outdir)
    except:
    die('failed to create archive dir')
    download_swf(args.target, args.version)

    try:
    for d in ['xml', 'obj', 'sheets']:
    p = os.path.join(outdir, d)
    if not os.path.exists(p):
    os.mkdir(p)
    except:
    die('failed to create output directories')

    extract('temp.swf')

    dotxt()
    dopng()

    os.chdir('/')
    shutil.rmtree(tempdir)
    32 changes: 32 additions & 0 deletions readme.md
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,32 @@
    depends on [swftools](http://swftools.org)
    use python 2.7 (might work on 2.6 or 2.5 if you install `argparse`)

    ### usage (`--help`)

    ```
    usage: extract.py [-h] [-t {production,testing}] [--test] [--prod]
    [-v VERSION] [-f FILE] [-d DIR] [--archive]
    optional arguments:
    -h, --help show this help message and exit
    -t {production,testing}
    --test same as "-t testing"
    --prod same as "-t production"
    -v VERSION download specific version (default: latest)
    -f FILE, --file FILE use local swf
    -d DIR, --dir DIR output directory (default: current directory)
    --archive write output to a subdir "type-version". If exists, do
    nothing (no effect with -f)
    ```

    `-t` defaults to `testing`

    ### output

    creates 3 subdirectories with:

    - `obj`: any non-xml data from DEFINEBINARY, right now only 3d models
    - `sheets`: spritesheets
    - `xml`: the xml files as-is

    also writes concatenated xml files grouped by root tag (Objects.xml etc)