Skip to content

Instantly share code, notes, and snippets.

@JamesXNelson

JamesXNelson/preload.gradle

Last active December 6, 2019 02:06
Download ZIP

Revisions

  1. JamesXNelson revised this gist Jul 7, 2019. No changes.
  2. JamesXNelson revised this gist Jul 7, 2019. 1 changed file with 4 additions and 2 deletions.
    6 changes: 4 additions & 2 deletions preload.gradle
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,4 @@
    import org.gradle.util.GUtil
    import org.gradle.api.publish.maven.internal.publication.MavenPublicationInternal
    import org.apache.maven.model.io.xpp3.MavenXpp3Reader
    import org.apache.maven.model.io.xpp3.MavenXpp3Writer
    @@ -41,6 +42,7 @@ tasks.register 'primePreload', {
    }

    String repoName = 'Local repo'
    String publicationPrefix = 'pub-'
    if (findProperty('preload') == 'true') {
    PublishingExtension pub = extensions.getByType(PublishingExtension)
    pub.repositories.maven { MavenArtifactRepository repo ->
    @@ -105,7 +107,7 @@ if (findProperty('preload') == 'true') {
    }
    }

    pub.publications.create("pub-$group--$name--$version", MavenPublication) {
    pub.publications.create("$publicationPrefix}$group--$name--$version", MavenPublication) {
    MavenPublication p ->

    p.groupId = group
    @@ -135,7 +137,7 @@ if (findProperty('preload') == 'true') {
    }
    tasks.register('downloadDeps') { dl ->
    dl.dependsOn tasks.withType(PublishToMavenRepository).findResults {
    it.name.startsWith("publishPub-") && it.name.endsWith("Local repoRepository")? it : null
    it.name.startsWith("publish${GUtil.toCamelCase(publicationPrefix)}") && it.name.endsWith("${repoName}Repository")? it : null
    }
    }
    }
  3. JamesXNelson revised this gist Jul 7, 2019. 1 changed file with 92 additions and 120 deletions.
    212 changes: 92 additions & 120 deletions preload.gradle
    View file
    Original file line number Diff line number Diff line change
    @@ -20,145 +20,117 @@ repositories { jcenter() }


    // This is a proof of concept buildscript, suitable for putting into a gist.
    // We have zero actual need for this dependency on apache james, it's just here
    // We have zero actual need for this dependency on wildfly, it's just here
    // as an arbitrary "deep dependency graph" that we can use for prototyping.
    dependencies { api 'org.apache.james:apache-james-mailbox-memory:3.3.0' }
    dependencies { api 'org.wildfly.core:wildfly-server:9.0.1.Final' }



    List<Configuration> sourceConfigs = [configurations.runtimeClasspath] // make this backed by a task property instead

    // make a task that can be used to prime the local gradle jar cache.
    // on a ci system where you provide a pre-primed cache, you may wish to exclude this task.
    tasks.register 'primePreload', {
    Task t ->
    t.doLast {
    // forcibly resolve everything we're supposed to be loading
    Set allFiles = sourceConfigs*.resolvedConfiguration*.resolvedArtifacts*.file.flatten()
    // TODO: make this log level (easily / obviously) configurable (move this code to a class)
    logger.info "Preloading configurations ${->sourceConfigs*.name}; retrieved files $allFiles"
    }
    }

    String repoName = 'Local repo'
    String cheatProp = 'dirty.preload.cheat' // set this as a system property, to cause preloading to run an inline gradle build to download dependencies in parallel...
    String cheatValue = 'noDirtyWork'
    if (findProperty('preload') == 'true') {
    PublishingExtension pub = extensions.getByType(PublishingExtension)
    pub.repositories.maven { MavenArtifactRepository repo ->
    repo.name = repoName
    repo.url = "file://$rootDir/repository"
    }
    List<Configuration> sourceConfigs = [configurations.runtimeClasspath] // make this backed by a task property instead

    // make a task that can be used to prime the local gradle jar cache.
    // on a ci system where you provide a pre-primed cache, you may wish to exclude this task.
    tasks.register 'primePreload', {
    Task t ->
    t.doLast {
    // forcibly resolve everything we're supposed to be loading
    Set allFiles = sourceConfigs*.resolvedConfiguration*.resolvedArtifacts*.file.flatten()
    // TODO: make this log level (easily / obviously) configurable (move this code to a class)
    logger.info "Preloading configurations ${->sourceConfigs*.name}; retrieved files $allFiles"
    }

    Map<ModuleVersionIdentifier, Set<Dependency>> all = [:].withDefault { new LinkedHashSet<>() }
    // we want to download all transitive dependencies, not just the firstLevel (declared) module dependencies.
    sourceConfigs*.resolvedConfiguration*.resolvedArtifacts.flatten().each {
    ResolvedArtifact d ->
    ModuleVersionIdentifier id = d.moduleVersion.id
    Dependency dep = dependencies.create(
    "$id.group:$id.name:$id.version" +
    "${d.classifier ? ":$d.classifier" : '' }" +
    "${ (d.extension ?: d.type) ? "@${d.extension ?: d.type}" : ''}"
    )
    all.get(id).add(dep)
    }


    if (cheatValue != System.getProperty(cheatProp)) {

    // fully resolving any org.gradle.api.artifacts.Configuration during the configuration phase
    // is a "bad practice", as it will download metadata for everything in a serial manner;
    // this is not a problem if the gradle jar cache / dependency metadata cache is fully populated,
    // but in order to *ensure* that we *never* download everything in parallel,
    // this would require immediately running a "cheater build" which just resolves the given configuration in a task.
    //
    // You may request for this dirty hack to occur by setting -Ddirty.preload.cheat=true.
    if ('true' == System.getProperty(cheatProp)) {
    GradleBuild b = new GradleBuild()
    b.startParameter = gradle.startParameter
    b.dir = rootDir
    b.buildFile = rootProject.buildFile
    b.tasks = ['primePreload']
    b.startParameter.systemPropertiesArgs[cheatProp] = cheatValue
    // perform an in-line gradle build whose only task is to download all about-to-be-primed artifacts
    // this will allow the rest of this configuration phase to proceed quickly, when it checks metadata serially
    // (and chooses to not download in serial)
    logger.quiet("\n\n\nInvoking dirty-cheat build")
    b.build()
    logger.quiet("\n\n\nDone dirty-cheat build, carrying on...")
    gradle.startParameter.systemPropertiesArgs[cheatProp] = 'done'
    System.setProperty(cheatProp, 'done')
    }
    ConfigurationContainer configs = project.configurations
    File dlPomDir = new File(project.buildDir, "dl-poms/")
    String gradleVersion = gradle.gradleVersion
    all.each { ModuleVersionIdentifier d, Set<Dependency> c ->
    String group = d.group
    String name = d.name
    String version = d.version
    ResolvedConfiguration resolved = configs.detachedConfiguration(c.toArray(new Dependency[0])).resolvedConfiguration

    Map<ModuleVersionIdentifier, Set<Dependency>> all = [:].withDefault { new LinkedHashSet<>() }
    // we want to download all transitive dependencies, not just the firstLevel (declared) module dependencies.
    sourceConfigs*.resolvedConfiguration*.resolvedArtifacts.flatten().each {
    ResolvedArtifact d ->
    ModuleVersionIdentifier id = d.moduleVersion.id
    TaskProvider pomGen = project.tasks.register("pomGen-$group--$name--$version") {
    Task pomGen ->
    Dependency dep = dependencies.create(
    "$id.group:$id.name:$id.version" +
    "${d.classifier ? ":$d.classifier" : '' }" +
    "${ (d.extension ?: d.type) ? "@${d.extension ?: d.type}" : ''}"
    "$d.group:$d.name:$d.version@pom"
    )
    all.get(id).add(dep)
    }


    ConfigurationContainer configs = project.configurations
    File dlPomDir = new File(project.buildDir, "dl-poms/")
    String gradleVersion = gradle.gradleVersion
    all.each { ModuleVersionIdentifier d, Set<Dependency> c ->
    String group = d.group
    String name = d.name
    String version = d.version
    ResolvedConfiguration resolved = configs.detachedConfiguration(c.toArray(new Dependency[0])).resolvedConfiguration

    TaskProvider pomGen = project.tasks.register("pomGen-$group--$name--$version") {
    Task pomGen ->
    Dependency dep = dependencies.create(
    "$d.group:$d.name:$d.version@pom"
    )
    File f = new File(dlPomDir, "$group/$name-${version}.pom".toString())
    pomGen.outputs.file(f)
    pomGen.doFirst {
    File source = configs.detachedConfiguration(dep).singleFile
    FileReader reader = new FileReader(source)
    org.apache.maven.model.Model pomModel
    try {
    pomModel = new MavenXpp3Reader().read(reader)
    } finally {
    reader.close()
    }
    // sadly, we have to fix the version/groupId for older versions of gradle;
    // this was fixed in May 2019, so if you are using a very recent version of gradle,
    // you could probably ditch alll this mess. TODO: plugin w/ a bifurcating service based on gradle version
    if (!pomModel.version) {
    pomModel.version = pomModel.parent.version
    }
    if (!pomModel.groupId) {
    pomModel.groupId = pomModel.parent.groupId
    }
    FileWriter writer = new FileWriter(f)
    try {
    new MavenXpp3Writer().write(writer, pomModel)
    } finally {
    writer.close()
    }
    File f = new File(dlPomDir, "$group/$name-${version}.pom".toString())
    pomGen.outputs.file(f)
    pomGen.doFirst {
    File source = configs.detachedConfiguration(dep).singleFile
    FileReader reader = new FileReader(source)
    org.apache.maven.model.Model pomModel
    try {
    pomModel = new MavenXpp3Reader().read(reader)
    } finally {
    reader.close()
    }
    }

    pub.publications.create("pub-$group--$name--$version", MavenPublication) {
    MavenPublication p ->

    p.groupId = group
    p.artifactId = name
    p.version = version

    if (gradleVersion < '5.5') {
    (p as MavenPublicationInternal).setPomGenerator(pomGen.get())
    } else {
    (p as MavenPublicationInternal).setPomGenerator(pomGen)
    // sadly, we have to fix the version/groupId for older versions of gradle;
    // this was fixed in May 2019, so if you are using a very recent version of gradle,
    // you could probably ditch alll this mess. TODO: plugin w/ a bifurcating service based on gradle version
    if (!pomModel.version) {
    pomModel.version = pomModel.parent.version
    }
    resolved.resolvedArtifacts.each {
    ResolvedArtifact r ->
    if (r.moduleVersion.id == d) {
    p.artifact provider { r.file }, {
    MavenArtifact m ->
    if (r.classifier) {
    m.classifier = r.classifier
    }
    if (r.extension) {
    m.extension = r.extension
    }
    }
    }
    if (!pomModel.groupId) {
    pomModel.groupId = pomModel.parent.groupId
    }
    }
    FileWriter writer = new FileWriter(f)
    try {
    new MavenXpp3Writer().write(writer, pomModel)
    } finally {
    writer.close()
    }
    }
    }

    pub.publications.create("pub-$group--$name--$version", MavenPublication) {
    MavenPublication p ->

    p.groupId = group
    p.artifactId = name
    p.version = version

    if (gradleVersion < '5.5') {
    (p as MavenPublicationInternal).setPomGenerator(pomGen.get())
    } else {
    (p as MavenPublicationInternal).setPomGenerator(pomGen)
    }
    resolved.resolvedArtifacts.each {
    ResolvedArtifact r ->
    if (r.moduleVersion.id == d) {
    p.artifact provider { r.file }, {
    MavenArtifact m ->
    if (r.classifier) {
    m.classifier = r.classifier
    }
    if (r.extension) {
    m.extension = r.extension
    }
    }
    }
    }
    }
    }
    tasks.register('downloadDeps') { dl ->
  4. JamesXNelson revised this gist Jul 7, 2019. 1 changed file with 145 additions and 53 deletions.
    198 changes: 145 additions & 53 deletions preload.gradle
    View file
    Original file line number Diff line number Diff line change
    @@ -1,76 +1,168 @@
    import org.w3c.dom.Document
    import org.w3c.dom.Element
    import org.gradle.api.publish.maven.internal.publication.MavenPublicationInternal
    import org.apache.maven.model.io.xpp3.MavenXpp3Reader
    import org.apache.maven.model.io.xpp3.MavenXpp3Writer

    buildscript {
    repositories {
    jcenter()
    }
    dependencies {
    // only needed for gradle versions < 5.5... should be resolved w/ different versions of released plugin
    classpath 'org.apache.maven:maven-model:3.6.1'
    }
    }
    plugins {
    id 'java-library'
    id 'maven-publish'
    }
    repositories { jcenter() }
    dependencies { api 'org.springframework.boot:spring-boot:2.1.6.RELEASE' }



    // This is a proof of concept buildscript, suitable for putting into a gist.
    // We have zero actual need for this dependency on apache james, it's just here
    // as an arbitrary "deep dependency graph" that we can use for prototyping.
    dependencies { api 'org.apache.james:apache-james-mailbox-memory:3.3.0' }


    String repoName = 'Local repo'
    String cheatProp = 'dirty.preload.cheat' // set this as a system property, to cause preloading to run an inline gradle build to download dependencies in parallel...
    String cheatValue = 'noDirtyWork'
    if (findProperty('preload') == 'true') {
    PublishingExtension pub = extensions.getByType(PublishingExtension)
    pub.repositories.maven { MavenArtifactRepository repo ->
    repo.name = 'Local repo'
    repo.name = repoName
    repo.url = "file://$rootDir/repository"
    }
    Map<Dependency, Configuration> all = [:]
    configurations.runtimeClasspath.allDependencies.each {
    all[it] = configurations.detachedConfiguration(it)
    List<Configuration> sourceConfigs = [configurations.runtimeClasspath] // make this backed by a task property instead

    // make a task that can be used to prime the local gradle jar cache.
    // on a ci system where you provide a pre-primed cache, you may wish to exclude this task.
    tasks.register 'primePreload', {
    Task t ->
    t.doLast {
    // forcibly resolve everything we're supposed to be loading
    Set allFiles = sourceConfigs*.resolvedConfiguration*.resolvedArtifacts*.file.flatten()
    // TODO: make this log level (easily / obviously) configurable (move this code to a class)
    logger.info "Preloading configurations ${->sourceConfigs*.name}; retrieved files $allFiles"
    }
    }
    all.each { d, c ->
    ResolvedConfiguration resolved = c.resolvedConfiguration
    resolved.resolvedArtifacts.each {
    ResolvedArtifact r ->
    pub.publications.create("pub-$r.moduleVersion.id.group--$r.moduleVersion.id.name", MavenPublication) {
    MavenPublication p ->
    p.artifact(provider { r.file }) {
    MavenArtifact m ->
    if (r.classifier) {
    m.classifier = r.classifier
    }
    if (r.extension) {
    m.extension = r.extension
    }


    if (cheatValue != System.getProperty(cheatProp)) {

    // fully resolving any org.gradle.api.artifacts.Configuration during the configuration phase
    // is a "bad practice", as it will download metadata for everything in a serial manner;
    // this is not a problem if the gradle jar cache / dependency metadata cache is fully populated,
    // but in order to *ensure* that we *never* download everything in parallel,
    // this would require immediately running a "cheater build" which just resolves the given configuration in a task.
    //
    // You may request for this dirty hack to occur by setting -Ddirty.preload.cheat=true.
    if ('true' == System.getProperty(cheatProp)) {
    GradleBuild b = new GradleBuild()
    b.startParameter = gradle.startParameter
    b.dir = rootDir
    b.buildFile = rootProject.buildFile
    b.tasks = ['primePreload']
    b.startParameter.systemPropertiesArgs[cheatProp] = cheatValue
    // perform an in-line gradle build whose only task is to download all about-to-be-primed artifacts
    // this will allow the rest of this configuration phase to proceed quickly, when it checks metadata serially
    // (and chooses to not download in serial)
    logger.quiet("\n\n\nInvoking dirty-cheat build")
    b.build()
    logger.quiet("\n\n\nDone dirty-cheat build, carrying on...")
    gradle.startParameter.systemPropertiesArgs[cheatProp] = 'done'
    System.setProperty(cheatProp, 'done')
    }

    Map<ModuleVersionIdentifier, Set<Dependency>> all = [:].withDefault { new LinkedHashSet<>() }
    // we want to download all transitive dependencies, not just the firstLevel (declared) module dependencies.
    sourceConfigs*.resolvedConfiguration*.resolvedArtifacts.flatten().each {
    ResolvedArtifact d ->
    ModuleVersionIdentifier id = d.moduleVersion.id
    Dependency dep = dependencies.create(
    "$id.group:$id.name:$id.version" +
    "${d.classifier ? ":$d.classifier" : '' }" +
    "${ (d.extension ?: d.type) ? "@${d.extension ?: d.type}" : ''}"
    )
    all.get(id).add(dep)
    }


    ConfigurationContainer configs = project.configurations
    File dlPomDir = new File(project.buildDir, "dl-poms/")
    String gradleVersion = gradle.gradleVersion
    all.each { ModuleVersionIdentifier d, Set<Dependency> c ->
    String group = d.group
    String name = d.name
    String version = d.version
    ResolvedConfiguration resolved = configs.detachedConfiguration(c.toArray(new Dependency[0])).resolvedConfiguration

    TaskProvider pomGen = project.tasks.register("pomGen-$group--$name--$version") {
    Task pomGen ->
    Dependency dep = dependencies.create(
    "$d.group:$d.name:$d.version@pom"
    )
    File f = new File(dlPomDir, "$group/$name-${version}.pom".toString())
    pomGen.outputs.file(f)
    pomGen.doFirst {
    File source = configs.detachedConfiguration(dep).singleFile
    FileReader reader = new FileReader(source)
    org.apache.maven.model.Model pomModel
    try {
    pomModel = new MavenXpp3Reader().read(reader)
    } finally {
    reader.close()
    }
    // sadly, we have to fix the version/groupId for older versions of gradle;
    // this was fixed in May 2019, so if you are using a very recent version of gradle,
    // you could probably ditch alll this mess. TODO: plugin w/ a bifurcating service based on gradle version
    if (!pomModel.version) {
    pomModel.version = pomModel.parent.version
    }
    if (!pomModel.groupId) {
    pomModel.groupId = pomModel.parent.groupId
    }
    p.groupId = r.moduleVersion.id.group
    p.artifactId = r.moduleVersion.id.name
    p.version = r.moduleVersion.id.version
    p.pom.withXml {
    XmlProvider x ->
    Element root = x.asElement()
    Document doc = root.ownerDocument
    Element dependencies = doc.createElement('dependencies')
    root.appendChild(dependencies)

    resolved.firstLevelModuleDependencies.each {
    ResolvedDependency resDep ->
    resDep.children.each {
    ResolvedDependency res ->
    Element dependency = doc.createElement('dependency')
    dependencies.appendChild(dependency)
    Element gId = doc.createElement('groupId')
    gId.setTextContent(res.module.id.group)
    dependency.appendChild(gId)
    Element aId = doc.createElement('artifactId')
    aId.setTextContent(res.module.id.name)
    dependency.appendChild(aId)
    Element v = doc.createElement('version')
    v.setTextContent(res.module.id.version)
    dependency.appendChild(v)
    res.moduleArtifacts.each {
    // TODO: repeat above for a new <dependency /> element,
    // but also set the classifier / type
    }
    FileWriter writer = new FileWriter(f)
    try {
    new MavenXpp3Writer().write(writer, pomModel)
    } finally {
    writer.close()
    }
    }
    }

    pub.publications.create("pub-$group--$name--$version", MavenPublication) {
    MavenPublication p ->

    p.groupId = group
    p.artifactId = name
    p.version = version

    if (gradleVersion < '5.5') {
    (p as MavenPublicationInternal).setPomGenerator(pomGen.get())
    } else {
    (p as MavenPublicationInternal).setPomGenerator(pomGen)
    }
    resolved.resolvedArtifacts.each {
    ResolvedArtifact r ->
    if (r.moduleVersion.id == d) {
    p.artifact provider { r.file }, {
    MavenArtifact m ->
    if (r.classifier) {
    m.classifier = r.classifier
    }
    if (r.extension) {
    m.extension = r.extension
    }
    }
    }
    }
    }
    }
    }
    }
    }
    tasks.register('downloadDeps') { dl ->
    dl.dependsOn tasks.withType(PublishToMavenRepository).findResults {
    println it.name
    it.name.startsWith("publishPub-") && it.name.endsWith("Local repoRepository")? it : null
    }
    }
  5. JamesXNelson created this gist Jul 6, 2019.
    77 changes: 77 additions & 0 deletions preload.gradle
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,77 @@
    import org.w3c.dom.Document
    import org.w3c.dom.Element

    plugins {
    id 'java-library'
    id 'maven-publish'
    }
    repositories { jcenter() }
    dependencies { api 'org.springframework.boot:spring-boot:2.1.6.RELEASE' }

    if (findProperty('preload') == 'true') {
    PublishingExtension pub = extensions.getByType(PublishingExtension)
    pub.repositories.maven { MavenArtifactRepository repo ->
    repo.name = 'Local repo'
    repo.url = "file://$rootDir/repository"
    }
    Map<Dependency, Configuration> all = [:]
    configurations.runtimeClasspath.allDependencies.each {
    all[it] = configurations.detachedConfiguration(it)
    }
    all.each { d, c ->
    ResolvedConfiguration resolved = c.resolvedConfiguration
    resolved.resolvedArtifacts.each {
    ResolvedArtifact r ->
    pub.publications.create("pub-$r.moduleVersion.id.group--$r.moduleVersion.id.name", MavenPublication) {
    MavenPublication p ->
    p.artifact(provider { r.file }) {
    MavenArtifact m ->
    if (r.classifier) {
    m.classifier = r.classifier
    }
    if (r.extension) {
    m.extension = r.extension
    }
    }
    p.groupId = r.moduleVersion.id.group
    p.artifactId = r.moduleVersion.id.name
    p.version = r.moduleVersion.id.version
    p.pom.withXml {
    XmlProvider x ->
    Element root = x.asElement()
    Document doc = root.ownerDocument
    Element dependencies = doc.createElement('dependencies')
    root.appendChild(dependencies)

    resolved.firstLevelModuleDependencies.each {
    ResolvedDependency resDep ->
    resDep.children.each {
    ResolvedDependency res ->
    Element dependency = doc.createElement('dependency')
    dependencies.appendChild(dependency)
    Element gId = doc.createElement('groupId')
    gId.setTextContent(res.module.id.group)
    dependency.appendChild(gId)
    Element aId = doc.createElement('artifactId')
    aId.setTextContent(res.module.id.name)
    dependency.appendChild(aId)
    Element v = doc.createElement('version')
    v.setTextContent(res.module.id.version)
    dependency.appendChild(v)
    res.moduleArtifacts.each {
    // TODO: repeat above for a new <dependency /> element,
    // but also set the classifier / type
    }
    }
    }
    }
    }
    }
    }
    tasks.register('downloadDeps') { dl ->
    dl.dependsOn tasks.withType(PublishToMavenRepository).findResults {
    println it.name
    it.name.startsWith("publishPub-") && it.name.endsWith("Local repoRepository")? it : null
    }
    }
    }