diff --git a/build.gradle.kts b/build.gradle.kts index c3f6a4a..5f52fed 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -43,6 +43,7 @@ repositories { dependencies { // implementation("org.spdx:spdx-tools:2.1.16") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.2.1") testImplementation("org.junit.jupiter:junit-jupiter:5.4.2") } diff --git a/src/main/kotlin/com/github/vlsi/gradle/license/BatchingLoader.kt b/src/main/kotlin/com/github/vlsi/gradle/license/BatchingLoader.kt new file mode 100644 index 0000000..f1fbb9c --- /dev/null +++ b/src/main/kotlin/com/github/vlsi/gradle/license/BatchingLoader.kt @@ -0,0 +1,136 @@ +/* + * Copyright 2019 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.github.vlsi.gradle.license + +import com.github.vlsi.gradle.license.api.License +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.selects.select +import org.gradle.api.artifacts.component.ComponentIdentifier + +class LicenseTag(val name: String, val url: String) +class LicenseesTag(val licenses: List) +class PomContents( + val parentId: ComponentIdentifier?, + val id: ComponentIdentifier, + val licenses: LicenseesTag? +) + +class LicenseDetector( + private val loader: suspend (ComponentIdentifier) -> PomContents +) { + fun LicenseTag.parse(): License = License.`0BSD` + + suspend fun detect(id: ComponentIdentifier): License { + val pom = loader(id) + val licenses = pom.licenses + if (licenses != null) { + return licenses.licenses.first().parse() + } + val parentId = + pom.parentId ?: TODO("License not found for $id, parent pom is missing as well") + return detect(parentId) + } +} + +class BatchBuilder { + private lateinit var handler: (List>>) -> Unit + private val tasks = mutableListOf Response) -> Result>() + + fun handleBatch(handler: (List>>) -> Unit) { + this.handler = handler + } + + fun task(action: suspend (suspend (Request) -> Response) -> Result) { + tasks.add(action) + } + + suspend fun getResult(): List> = coroutineScope { + val loadRequests = Channel>>() + val disconnects = Channel() + launch { + startHandler(tasks.size, loadRequests, disconnects) + } + + supervisorScope { + val results = mutableListOf>() + for (action in tasks) { + val res = async { + try { + action { + val res = CompletableDeferred() + loadRequests.send(it to res) + res.await() + } + } finally { + disconnects.send(Unit) + } + } + results.add(res) + } + results + } + } + + private suspend fun startHandler( + initialClients: Int, + loadRequests: Channel>>, + disconnects: Channel + ) { + var activeClients = initialClients + val requests = mutableListOf>>() + while (true) { + select { + disconnects.onReceive { + activeClients -= 1 + } + loadRequests.onReceive { + requests.add(it) + } + } + if (activeClients == 0) { + // No clients left => exit + return + } + if (activeClients == requests.size) { + handler(requests) + requests.clear() + } + } + } +} + +suspend fun batch(builder: BatchBuilder.() -> Unit): List> { + val scope = BatchBuilder() + builder(scope) + return scope.getResult() +} + +suspend fun loadLicenses(ids: List) = coroutineScope { + batch { + handleBatch { + + } + + for (id in ids) { + task { loader -> + LicenseDetector(loader).detect(id) + } + } + } +} diff --git a/src/main/kotlin/com/github/vlsi/gradle/license/GatherLicenseTask.kt b/src/main/kotlin/com/github/vlsi/gradle/license/GatherLicenseTask.kt index 8e4680a..bb5f9c9 100644 --- a/src/main/kotlin/com/github/vlsi/gradle/license/GatherLicenseTask.kt +++ b/src/main/kotlin/com/github/vlsi/gradle/license/GatherLicenseTask.kt @@ -20,10 +20,13 @@ package com.github.vlsi.gradle.license import com.github.vlsi.gradle.license.api.License import groovy.util.XmlSlurper import groovy.util.slurpersupport.GPathResult +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.selects.select import org.gradle.api.DefaultTask import org.gradle.api.GradleException import org.gradle.api.artifacts.Configuration -import org.gradle.api.artifacts.ResolvedArtifact +import org.gradle.api.artifacts.component.ComponentIdentifier import org.gradle.api.artifacts.component.ModuleComponentIdentifier import org.gradle.api.artifacts.result.ResolvedArtifactResult import org.gradle.api.model.ObjectFactory @@ -34,10 +37,12 @@ import org.gradle.maven.MavenPomArtifact import org.gradle.workers.IsolationMode import org.gradle.workers.WorkerExecutor import java.io.File +import java.net.URI import java.nio.charset.StandardCharsets import java.util.jar.JarFile import javax.inject.Inject +data class LicenseInfo(val license: License?, val file: File?, val licenseFiles: File?) open class GatherLicenseTask @Inject constructor( objectFactory: ObjectFactory, @@ -64,7 +69,7 @@ open class GatherLicenseTask @Inject constructor( @TaskAction fun run() { - val detectedLicenses = mutableMapOf() + val allDependencies = mutableMapOf() val licenseDir = licenseTextDir.get().asFile for (c in configurations.get() @@ -73,16 +78,21 @@ open class GatherLicenseTask @Inject constructor( for (art in c.resolvedConfiguration.resolvedArtifacts) { val compId = art.id.componentIdentifier + if (allDependencies.containsKey(compId)) { + continue + } + val artLicenseTexts = File(licenseDir, art.file.name) - detectedLicenses[art] = LicenseInfo( - license = licenseOverrides.get()[art.moduleVersion.toString()], + allDependencies[compId] = LicenseInfo( + license = licenseOverrides.get()[compId.displayName], + file = art.file, licenseFiles = artLicenseTexts ) if (compId is ModuleComponentIdentifier) { workerExecutor.submit(FindLicense::class) { - displayName = "Analyze ${art.moduleVersion}" + displayName = "Extract licenses for ${compId.displayName}" isolationMode = IsolationMode.NONE - params(art.moduleVersion.toString(), art.file, artLicenseTexts) + params(compId.displayName, art.file, artLicenseTexts) } } } @@ -100,11 +110,11 @@ open class GatherLicenseTask @Inject constructor( }.build() workerExecutor.await() - findManifestLicenses(detectedLicenses) - findPomLicenses(detectedLicenses) - findLicenseFromFiles(detectedLicenses, model) + findManifestLicenses(allDependencies) + findPomLicenses(allDependencies) + findLicenseFromFiles(allDependencies, model) - val missingLicenses = detectedLicenses + val missingLicenses = allDependencies .filter { it.value.license == null } .keys if (missingLicenses.isNotEmpty()) { @@ -115,7 +125,7 @@ open class GatherLicenseTask @Inject constructor( val outFile = outputFile.get().asFile outFile.writer().use { out -> - detectedLicenses + allDependencies .entries .sortedWith(compareBy { it.value.license }) .forEach { @@ -124,18 +134,35 @@ open class GatherLicenseTask @Inject constructor( } } - private fun findManifestLicenses(detectedLicenses: MutableMap) { + private fun findManifestLicenses(detectedLicenses: MutableMap) { for (e in detectedLicenses) { if (e.value.license != null) { continue } - logger.debug("Analyzing {}", e.key) - if (!e.key.file.endsWith(".jar")) { + val file = e.value.file + if (file == null) { + logger.debug( + "No file is specified for artifact {}. Will skip MANIFEST.MF check", + e.key + ) + continue + } + if (!file.endsWith(".jar")) { + logger.debug( + "File {} for artifact {} does not look like a JAR. Will skip MANIFEST.MF check", + file, + e.key + ) continue } - JarFile(e.key.file).use { jar -> + logger.debug( + "Will check if file {} for artifact {} has Bundle-License in MANIFEST.MF", + file, + e.key + ) + JarFile(file).use { jar -> val bundleLicense = jar.manifest.mainAttributes.getValue("Bundle-License") val license = bundleLicense?.substringBefore(";")?.let { License.fromLicenseIdOrNull(it) @@ -150,45 +177,36 @@ open class GatherLicenseTask @Inject constructor( operator fun GPathResult.get(name: String) = getProperty(name) as GPathResult - private fun findPomLicenses(detectedLicenses: MutableMap) { + private fun String.trimTextExtensions() = removeSuffix(".txt").removeSuffix(".md") + + private fun URI.looksTheSame(other: URI) = + schemeSpecificPart == other.schemeSpecificPart || + schemeSpecificPart.trimTextExtensions() == other.schemeSpecificPart.trimTextExtensions() + + private fun findPomLicenses(detectedLicenses: MutableMap) { // TODO: support licenses declared in parent-poms - val componentIds = + val componentsWithUnknownLicenses = detectedLicenses .filter { it.value.license == null } .keys - .associateBy { it.id.componentIdentifier } - if (componentIds.isEmpty()) { + if (componentsWithUnknownLicenses.isEmpty()) { return } + val nameGuesser = TfIdfBuilder().apply { + License.values().forEach { addDocument(it, it.licenseName) } + }.build() + // TODO: support customization of project val result = project.dependencies.createArtifactResolutionQuery() - .forComponents(componentIds.keys) + .forComponents(componentsWithUnknownLicenses) .withArtifacts(MavenModule::class, MavenPomArtifact::class) .execute() - - val nameGuesser = TfIdfBuilder().apply { - License.values() - .forEach { - addDocument( - it, - it.licenseName - ) - } - }.build() + val requiredParents = mutableMapOf>() for (component in result.resolvedComponents) { val id = component.id - if (id !is ModuleComponentIdentifier) { - logger.debug( - "Id {} for component {} is not a ModuleComponentIdentifier. It does not look like a pom file", - id, - component - ) - continue - } - val poms = component.getArtifacts(MavenPomArtifact::class) if (poms.isEmpty()) { logger.debug("No pom files found for component {}", component) @@ -196,10 +214,8 @@ open class GatherLicenseTask @Inject constructor( } val pom = poms.first() as ResolvedArtifactResult val parsedPom = XmlSlurper().parse(pom.file) - var index = 0 - for (l in parsedPom["licenses"]["license"]) { - index += 1 - if (index > 1) { + for ((index, l) in parsedPom["licenses"]["license"].withIndex()) { + if (index > 0) { // TODO: collect all the violations and throw them later throw GradleException( "POM file for component $component declares multiple licenses." + @@ -209,6 +225,7 @@ open class GatherLicenseTask @Inject constructor( l as GPathResult val name = l["name"].toString() val url = l["url"].toString() + val uri = URI(url) val guessList = nameGuesser.predict(name) .entries .sortedByDescending { it.value } @@ -217,17 +234,14 @@ open class GatherLicenseTask @Inject constructor( .asSequence() .take(20) .firstOrNull() { - it.key.seeAlso.any { u -> - u.toString().startsWith(url) || - url.startsWith(u.toString()) - } + it.key.seeAlso.any { u -> u.looksTheSame(uri) } } if (matchingLicense != null) { logger.debug( "Automatically detected license name={} url={} to mean {}", name, url, matchingLicense.key ) - detectedLicenses.compute(componentIds.getValue(id)) { _, v -> + detectedLicenses.compute(id) { _, v -> v!!.copy(license = matchingLicense.key) } continue @@ -238,9 +252,9 @@ open class GatherLicenseTask @Inject constructor( "Automatically detected license {} to mean {}. Other possibilities were {}", name, firstLicense.key, - guessList + guessList.take(10) ) - detectedLicenses.compute(componentIds.getValue(id)) { _, v -> + detectedLicenses.compute(id) { _, v -> v!!.copy(license = firstLicense.key) } continue @@ -251,15 +265,16 @@ open class GatherLicenseTask @Inject constructor( } private fun findLicenseFromFiles( - detectedLicenses: MutableMap, + detectedLicenses: MutableMap, model: Predictor ) { for (e in detectedLicenses) { - if (e.value.license != null) { + val licenseDir = e.value.licenseFiles + if (e.value.license != null || licenseDir == null) { continue } val bestLicenses = - project.fileTree(e.value) { + project.fileTree(licenseDir) { include("**") }.flatMap { f -> // For each file take best 5 predictions @@ -288,5 +303,3 @@ open class GatherLicenseTask @Inject constructor( } } } - -data class LicenseInfo(val license: License?, val licenseFiles: File?) \ No newline at end of file diff --git a/src/test/kotlin/com/github/vlsi/gradle/license/BatchProcessorTest.kt b/src/test/kotlin/com/github/vlsi/gradle/license/BatchProcessorTest.kt new file mode 100644 index 0000000..67444e2 --- /dev/null +++ b/src/test/kotlin/com/github/vlsi/gradle/license/BatchProcessorTest.kt @@ -0,0 +1,65 @@ +/* + * Copyright 2019 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.github.vlsi.gradle.license + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Test + +class BatchProcessorTest { + @Test + fun name2() { + runBlocking { + val batchResults = batch { + handleBatch { + println("Arrived batch of ${it.size} values: ${it.map { it.first }}") + it.forEach { req -> + val input = req.first + if (input == 4) { + req.second.completeExceptionally(IllegalArgumentException("4 is not supported yet")) + return@forEach + } + req.second.complete(if ((input and 1) == 1) input - 1 else input / 2) + } + } + for (i in 1..5) { + task { loader -> + println("Started $i") + var v = i + val sb = StringBuilder("$i") + while (v != 0) { + v = loader(v) + sb.append(" => $v") + println(sb) + } + sb.toString() + } + } + } + + println("batch results: ") + for (r in batchResults) { + try { + println(" ${r.await()}") + } catch (e: Exception) { + println(" exception: $e") + } + } + } + println("done") + } +} diff --git a/src/test/kotlin/com/github/vlsi/gradle/license/GatherLicenseTaskTest.kt b/src/test/kotlin/com/github/vlsi/gradle/license/GatherLicenseTaskTest.kt index a03a452..b183fc7 100644 --- a/src/test/kotlin/com/github/vlsi/gradle/license/GatherLicenseTaskTest.kt +++ b/src/test/kotlin/com/github/vlsi/gradle/license/GatherLicenseTaskTest.kt @@ -68,10 +68,12 @@ class GatherLicenseTaskTest { repositories { mavenCentral() + jcenter() } dependencies { runtime("org.junit.jupiter:junit-jupiter:5.4.2") + runtime("io.ktor:ktor-server-core:1.2.1") } tasks.register("generateLicense", GatherLicenseTask.class) {