cargo-json-to-tupfile.py
author Chris Manchester <cmanchester@mozilla.com>
Thu, 17 May 2018 12:09:57 -0700
changeset 796525 d3f9db3fae551be81107781bc5387801ccd92760
parent 796523 fca8607363df1b3049b62936cdcbd8c90cb87af3
permissions -rw-r--r--
Bug 1461836 - Write out complete configure dependencies from configure for consumption by make and non-make backends. MozReview-Commit-ID: 792seCZ2rs1

import os
import sys
import json
import codecs
from collections import defaultdict

UTF8Writer = codecs.getwriter('utf8')
sys.stdout = UTF8Writer(sys.stdout)

libc = True
if len(sys.argv) > 1 and sys.argv[1] == '--no-libc':
    libc = False

with open('plan.json') as f:
    data = json.load(f)

invocations = data['invocations']
processed = {}

extra_outputs = {
    'bindgen': [
        'tests.rs',
        'host-target.txt',
    ],
    'cssparser': [
        'tokenizer.rs',
    ],
    'gleam': [
        'gl_and_gles_bindings.rs',
        'gl_bindings.rs',
        'gles_bindings.rs',
    ],
    'khronos_api': [
        'webgl_exts.rs',
    ],
    'libloading': [
        'libglobal_static.a',
        'src/os/unix/global_static.o',
    ],
    'selectors': [
        'ascii_case_insensitive_html_attributes.rs',
    ],
    'style': [
        'gecko/atom_macro.rs',
        'gecko/pseudo_element_definition.rs',
        'gecko_properties.rs',
        'properties.rs',
        'gecko/bindings.rs',
        'gecko/structs.rs',
    ],
    'webrender': [
        'shaders.rs',
    ],
}

objdir_extra_outputs = {
#    'style': [
#        'dist/rust_bindings/style/atom_macro.rs',
#        'dist/rust_bindings/style/pseudo_element_definition.rs',
#        'dist/rust_bindings/style/bindings.rs',
#        'dist/rust_bindings/style/structs.rs',
#    ],
}

extra_flags = {
    'style': [
        '-l', 'static=global_static',
        '-L', 'native=%(libloading_outdir)s',
    ]
}

def display_name(invocation):
    output_str = ''
    if invocation['outputs']:
        output_str = ' -> %s' % ' '.join([os.path.basename(f) for f in invocation['outputs']])
    return '%s %s v%s %s%s' % (
        os.path.basename(invocation['program']).upper(),
        invocation['package_name'],
        invocation['package_version'],
        invocation['kind'],
        output_str,
    )

def shell_quote(s):
    x = "'%s'" % s.replace("'", "'\"'\"'") if '"' in s or ' ' in s else s
    # TODO HACKS:
    #  1) remove dep-info
    #  2) force debug build
    return x.replace('dep-info,', '').replace('opt-level=2', 'debuginfo=0').replace('\n', '\\n')

def get_libloading_outdir():
    for invocation in invocations:
        if invocation['package_name'] == 'libloading' and invocation['outputs'][0].endswith('.rlib'):
            return invocation['env']['OUT_DIR']

def process(key, invocation):
    if key in processed:
        return
    processed[key] = True
    inputs = set()
    mydeps = []
    invocation['uses'] = 0
    shortname = invocation['package_name']

    mydeps.extend(invocation['deps'])

    depth = 0
    for dep in mydeps:
        # We'd expect to just handle dependencies transitively (so use
        # invocations[dep]['outputs'] here, but because the weird host dependencies
        # sometimes get used in the final library and not intermediate
        # libraries, tup doesn't work well with them. So build up the full set
        # of intermediate dependencies with 'full-deps'
        depmod = invocations[dep]
        process(dep, depmod)
        inputs.update(depmod['full-deps'])
        depmod['uses'] += 1
        depth = max(depth, depmod['depth'])
    invocation['depth'] = depth + 1
#    print >> sys.stderr, "%s %s" % (invocation['depth'], key)

    if not invocation['program']:
        print >> sys.stderr, "Skipping: %s" % display_name(invocation)
        sys.exit(1)
        return
    command = [
        'cd %s &&' % invocation['cwd'],
        'env',
    ]
    envvars = invocation.get('env')
    for k, v in envvars.iteritems():
        command.append("%s=%s" % (k, shell_quote(v)))
    if 'sccache' not in invocation['program']:
        command.append(invocation['program'])
    command.extend([shell_quote(a) for a in invocation['args']])

    # TODO: Remove dep-info flag by option
    #
    # TODO: Don't create any files under target/
    #
    # TODO: Sort output of dependencies in cargo somehow? output is
    # non-deterministic
    #
    # TODO: Link/copy hello_world to the -ba7b98a79 version
    #
    # TODO: build.rs support?

    outputs = invocation['outputs']
    if os.path.basename(invocation['program']) == 'build-script-build':
        for output in extra_outputs.get(shortname, []):
            outputs.append(os.path.join(invocation['env']['OUT_DIR'], output))
        for output in objdir_extra_outputs.get(shortname, []):
            outputs.append(os.path.join('$(MOZ_TOPOBJDIR)', output))

    if invocation['target_kind'][0] == 'custom-build' and os.path.basename(invocation['program']) == 'rustc':
        flags = extra_flags.get(shortname, [])
        for flag in flags:
            command.append(flag % {'libloading_outdir': get_libloading_outdir()})

    flags = ''
    if 'rustc' in invocation['program']:
        header = 'RUSTC'
    else:
        inputs.add(invocation['program'])
        header = 'RUN'
        flags = 'o'
    invocation['full-deps'] = set(inputs)
    invocation['full-deps'].update(invocation['outputs'])

    out_group = '$(MOZ_OBJ_ROOT)/<rust-lib>'
    print(": %(input)s | $(MOZ_OBJ_ROOT)/<installed-files> |> ^%(flags)s %(header)s %(display)s^ %(cmd)s |> %(output)s%(extra_outputs)s" % {
        'flags': flags,
        'input': ' '.join(inputs),
        'header': header,
        'display': display_name(invocation),
        'cmd': ' '.join(command),
        'output': ' '.join(outputs),
        'extra_outputs': ' | ' + out_group,
    })
    for dst, link in invocation['links'].iteritems():

        print(": %(input)s |> !tup_ln |> %(output)s%(extra_outputs)s" % {
            'input': link,
            'output': dst,
            'extra_outputs': ' | ' + out_group,
        })

# TODO: if OsString is used
def unpack(val):
    return ''.join(map(chr, val['Unix']))

for key, invocation in enumerate(invocations):
    process(key, invocation)

def get_node(key):
    return key.replace(' ', '_').replace('(', '_').replace(')', '_').replace('.', '_').replace('-', '_').replace(':', '_').replace('/', '_')

#dag = set()
#def mtest2(key, top=True):
#    if top:
#        print("digraph G {")
#    print("%s [label=\"%s\"];" % (get_node(key), key))
#    for dep in invocations[key]['deps']:
#        n1 = get_node(key)
#        n2 = get_node(dep)
#        if ((n1, n2) not in dag):
#            print("%s -> %s [dir=back,arrowtail=\"empty\"];" % (n1, n2))
#            dag.add((n1, n2))
#        mtest2(dep, top=False)
#    if top:
#        print("}")

#def mtest(key, indent=0):
#    print >> sys.stderr, ("%s - %s (%s)" % (' ' * indent, key, invocations[key]['uses']))
#    for dep in invocations[key]['deps']:
#        mtest(dep, indent+4)
#
#mtest('serde_derive v1.0.8 Target(lib) Profile(build) Host')
#mtest('gkrust v0.1.0 (file:///home/mshal/mozilla-central-tmp/toolkit/library/rust) Target(lib) Profile(build) Target')
#mtest2('serde_derive v1.0.8 Target(lib) Profile(build) Host')
#mtest2('gkrust v0.1.0 (file:///home/mshal/mozilla-central-tmp/toolkit/library/rust) Target(lib) Profile(build) Target')