build.gradle
author Gregory Szorc <gps@mozilla.com>
Mon, 08 May 2017 17:19:05 -0700
changeset 361962 11ba6213be5a745d70e5c6fdcd036bafb1ac2718
parent 323735 e20c0b78111ecd40826a7caa5a141fe9c3519a98
child 368847 a419686f707e7a248a3b477556fef8b8ae7d6bae
permissions -rw-r--r--
Bug 1359965 - Support and generate tar.gz WPT archive; r=glandium Several years ago there was a single zip file for all test files. Clients would only extract the files they needed. Thus, zip was a reasonable archive format because it allowed direct access to members without having to decompress the entirety of the stream. We have since split up that monolithic archive into separate, domain-specific archives. e.g. 1 archive for mochitests and one for xpcshell tests. This drastically cut down on network I/O required on testers because they only fetched archives/data that was relevant. It also enabled parallel generation of test archives, we shaved dozens of seconds off builds due to compression being a long pole. Despite the architectural changes to test archive management, we still used zip files. This is not ideal because we no longer access specific files in test archives and thus don't care about single/partial member access performance. This commit implements support for generating tar.gz test archives. And it switches the web-platform archive to a tar.gz file. The performance implications for archive generation are significant: before: 48,321,250 bytes; 6.05s after: 31,844,267 bytes; 4.57s The size is reduced because we have a single compression context so data from 1 file can benefit compression in a subsequent file. CPU usage is reduced because the compressor has to work less with 1 context than it does with N. While I didn't measure it, decompression performance should also be improved for the same reasons. And of course network I/O will be reduced. mozharness consumers use a generic method for handling unarchiving. This method automagically handles multiple file extensions. So as long as downstream consumers aren't hard coding ".zip" this change should "just work." MozReview-Commit-ID: LQa5MIHLsms

import java.util.regex.Pattern

allprojects {
    // Expose the per-object-directory configuration to all projects.
    ext {
        mozconfig = gradle.mozconfig
        topsrcdir = gradle.mozconfig.topsrcdir
        topobjdir = gradle.mozconfig.topobjdir
    }

    repositories {
        if (gradle.mozconfig.substs.GRADLE_MAVEN_REPOSITORY) {
            maven {
                url gradle.mozconfig.substs.GRADLE_MAVEN_REPOSITORY
            }
        }
    }
}

buildDir "${topobjdir}/gradle/build"

buildscript {
    repositories {
        if (gradle.mozconfig.substs.GRADLE_MAVEN_REPOSITORY) {
            maven {
                url gradle.mozconfig.substs.GRADLE_MAVEN_REPOSITORY
            }
        }
        // For android-sdk-manager SNAPSHOT releases.
        maven {
            url "file://${gradle.mozconfig.topsrcdir}/mobile/android/gradle/m2repo"
        }
    }

    dependencies {
        classpath 'com.android.tools.build:gradle:2.1.3'
        classpath('com.stanfy.spoon:spoon-gradle-plugin:1.0.4') {
            // Without these, we get errors linting.
            exclude module: 'guava'
        }
        // Provided in tree.
        classpath 'com.jakewharton.sdkmanager:gradle-plugin:1.5.0-SNAPSHOT'
        classpath 'com.getkeepsafe.dexcount:dexcount-gradle-plugin:0.6.1'
    }
}

task generateCodeAndResources(type:Exec) {
    workingDir "${topobjdir}"

    commandLine mozconfig.substs.GMAKE
    args '-C'
    args "${topobjdir}/mobile/android/base"
    args 'gradle-targets'

    // Only show the output if something went wrong.
    ignoreExitValue = true
    standardOutput = new ByteArrayOutputStream()
    errorOutput = standardOutput
    doLast {
        if (execResult.exitValue != 0) {
            throw new GradleException("Process '${commandLine}' finished with non-zero exit value ${execResult.exitValue}:\n\n${standardOutput.toString()}")
        }
    }
}

// Skip unit test for all build variants, unless if it was specifically requested by user.
// The enabled property for the unit test tasks is reset based on the command line task names just before the task execution.
// I bet there is a easier/cleaner way to do this, but this gets the job done for now.
Pattern pattern = Pattern.compile('.*test(.+UnitTest)?.*')
boolean startTasksIncludeTest = gradle.startParameter.taskNames.any {
    taskName ->
        taskName.matches(pattern)
}
gradle.taskGraph.beforeTask {
    Task task ->
        if (task.name.matches(pattern)) {
            task.enabled = startTasksIncludeTest
        }
}

afterEvaluate {
    subprojects {
        if (!hasProperty('android')) {
            return
        }
        android.applicationVariants.all {
            preBuild.dependsOn rootProject.generateCodeAndResources
        }
        android.libraryVariants.all {
            preBuild.dependsOn rootProject.generateCodeAndResources
        }
    }
}

apply plugin: 'idea'

idea {
    project {
        languageLevel = '1.7'
    }

    module {
        // Object directories take a huge amount of time for IntelliJ to index.
        // Exclude them.  Convention is that object directories start with obj.
        // IntelliJ is clever and will not exclude the parts of the object
        // directory that are referenced, if there are any.  In practice,
        // indexing the entirety of the tree is taking too long, so exclude all
        // but mobile/.
        def topsrcdirURI = file(topsrcdir).toURI()
        excludeDirs += files(file(topsrcdir)
            .listFiles({it.isDirectory()} as FileFilter)
            .collect({topsrcdirURI.relativize(it.toURI()).toString()}) // Relative paths.
            .findAll({!it.equals('mobile/')}))

        // If topobjdir is below topsrcdir, hide only some portions of that tree.
        def topobjdirURI = file(topobjdir).toURI()
        if (!topsrcdirURI.relativize(topobjdirURI).isAbsolute()) {
            excludeDirs -= file(topobjdir)
            excludeDirs += files(file(topobjdir).listFiles())
            excludeDirs -= file("${topobjdir}/gradle")
        }

        if (!mozconfig.substs.MOZ_INSTALL_TRACKING) {
            excludeDirs += file("${topsrcdir}/mobile/android/thirdparty/com/adjust")
        }
    }
}

task wrapper(type: Wrapper) {
}