build.gradle

buildscript {
	repositories {
		mavenCentral()
	}
	dependencies {
		classpath 'de.thetaphi:forbiddenapis:3.3'
	}
}
plugins {
	// https://github.com/researchgate/gradle-release
	id 'net.researchgate.release' version '2.8.1'
    id 'io.codearte.nexus-staging' version '0.30.0'
}

apply plugin: 'java'
apply plugin: 'de.thetaphi.forbiddenapis'
apply plugin: 'jacoco'
apply plugin: 'maven'
apply plugin: 'signing'
apply plugin: 'io.codearte.nexus-staging'

sourceCompatibility = JavaVersion.VERSION_11
group = 'org.dstadler'
archivesBaseName = 'commoncrawldownload'

repositories {
	mavenCentral()
}

forbiddenApis {
	suppressAnnotations = ['org.dstadler.commons.util.SuppressForbidden']
	bundledSignatures = [ 'jdk-reflection', 'commons-io-unsafe-2.11.0', 'jdk-internal' ]
	signaturesFiles += files('config/forbidden-apis/http-signatures.txt')
}
forbiddenApisMain {
	// 'jdk-unsafe', 'jdk-system-out'
	bundledSignatures += [ 'jdk-deprecated', 'jdk-internal', 'jdk-non-portable' ]
	signaturesFiles += files('config/forbidden-apis/forbidden.signatures.txt')
}

dependencies {
    implementation 'org.dstadler:commons-dost:1.1.0.2'
    implementation 'org.apache.httpcomponents:httpclient:4.5.13'
	implementation 'org.apache.commons:commons-lang3:3.12.0'
	implementation 'commons-io:commons-io:2.11.0'

    // version '0.20.2-cdh3u4' is referenced by webarchive-commons, but "vanished"...
    implementation 'org.apache.hadoop:hadoop-core:0.20.205.0'
    implementation 'org.netpreserve.commons:webarchive-commons:1.1.9'
    implementation 'com.fasterxml.jackson.core:jackson-core:2.14.0'

    // for commoncrawl-source
    implementation 'org.jsoup:jsoup:1.15.3'

    // for @Nullable
    implementation 'com.google.code.findbugs:jsr305:3.0.2'

    testImplementation 'junit:junit:4.13.2'
	testImplementation 'org.dstadler:commons-test:1.0.0.20'
}

wrapper {
    gradleVersion = '6.9.2'
}

test {
	systemProperties = System.properties as Map<String, ?>

	// enable to show standard out and standard error of the test JVM(s) on the console
	// testLogging.showStandardStreams = true

	// http://forums.gradle.org/gradle/topics/jacoco_related_failure_in_multiproject_build
	systemProperties['user.dir'] = workingDir
}

jacoco {
	toolVersion = '0.8.8'
}

jacocoTestReport {
    reports {
        xml.enabled true
	}
}

task sourcesJar(type: Jar) {
    classifier = 'sources'
    from sourceSets.main.allJava
}

task javadocJar(type: Jar) {
    classifier = 'javadoc'
    from javadoc
}

artifacts {
    archives javadocJar, sourcesJar
}

signing {
	setRequired {
		// signing is only required if this is a release version
		// and the artifacts are to be published
		gradle.taskGraph.allTasks.any { it instanceof PublishToMavenRepository }
	}
    sign configurations.archives
}

//provide defaults so we do not need to specify them always
if (!project.hasProperty('ossrhUsername')) {
    ext.ossrhUsername = ''
}
if (!project.hasProperty('ossrhPassword')) {
    ext.ossrhPassword = ''
}

uploadArchives {
  repositories {
    mavenDeployer {
        //noinspection GrUnresolvedAccess
        beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) }

        repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2/') {
        authentication(userName: ossrhUsername, password: ossrhPassword)
      }

      snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots/') {
        authentication(userName: ossrhUsername, password: ossrhPassword)
      }

    pom.project {
        name 'CommonCrawl Document Download'
        packaging 'jar'
        // optionally artifactId can be defined here
        description 'Common utilities I find useful in many of my projects.'
        url 'https://github.com/centic9/CommonCrawlDocumentDownload'

        scm {
            connection 'scm:git@github.com:centic9/CommonCrawlDocumentDownload'
            developerConnection 'scm:git@github.com:centic9/CommonCrawlDocumentDownload'
            url 'https://github.com/centic9/CommonCrawlDocumentDownload'
        }

        licenses {
          license {
            name 'BSD 2-Clause License'
            url 'https://www.opensource.org/licenses/bsd-license.php'
          }
        }

        developers {
          developer {
            id 'centic9	'
            name 'Dominik Stadler'
          }
        }
      }
    }
  }
}

afterReleaseBuild.dependsOn uploadArchives

task lookupURLs(type:JavaExec,dependsOn: compileJava) {
	description = 'Reads the current Common Crawl URL index data and extracts all URLs for interesting mime-types or file extensions'

    jvmArgs = [
            '-Xmx128m'
    ]

    main = 'org.dstadler.commoncrawl.index.DownloadURLIndex'
	classpath = sourceSets.main.runtimeClasspath
}


task downloadOldIndex(type:JavaExec,dependsOn: compileJava) {
	description = 'Reading blocks at beginning or the last stored block-number and download binary data from the common crawl archives'

    jvmArgs = [
            '-Xmx128m'
    ]

    main = 'org.dstadler.commoncrawl.oldindex.ReadAndDownload'
	classpath = sourceSets.main.runtimeClasspath
}

task downloadDocuments(type:JavaExec,dependsOn: compileJava) {
	description = 'Uses the URLs listed in commoncrawl.txt to download the documents from the Common Crawl'

    jvmArgs = [
            '-Xmx128m'
    ]

    main = 'org.dstadler.commoncrawl.index.DownloadFromCommonCrawl'
	classpath = sourceSets.main.runtimeClasspath
}

task deduplicate(type:JavaExec,dependsOn: compileJava) {
	description = 'Compares files by size and hash and moves away identical files into a backup-directory'

    jvmArgs = [
            '-Xmx1024m'
    ]

    main = 'org.dstadler.commoncrawl.Deduplicate'
	classpath = sourceSets.main.runtimeClasspath
}