Merge pull request #115 from marklogic/release/2.1.0

Merging 2.1.0 into master

Author: rjrudin

.gitignore

@@ -15,3 +15,5 @@ gradle-local.properties
 logs
 .ipynb_checkpoints
 venv
+.venv
+docker

CONTRIBUTING.md

@@ -5,18 +5,22 @@ have cloned this repository to your local workstation.
 
 In order to develop and/or test the connector, or to try out the PySpark instructions below, you first 
 need to deploy the test application in this project to MarkLogic. You can do so either on your own installation of 
-MarkLogic, or you can use `docker-compose` to install a 3-node MarkLogic cluster with a load balancer in front of it. 
+MarkLogic, or you can use `docker-compose` to install MarkLogic, optionally as a 3-node cluster with a load balancer
+in front of it.
 
-## Installing a 3-node cluster with docker-compose
+## Installing MarkLogic with docker-compose
 
 If you wish to use `docker-compose`, perform the following steps before deploying the test application.
 
 1. [Install Docker](https://docs.docker.com/get-docker/).
 2. Ensure that you don't have a MarkLogic instance running locally (if you do, you may run into port conflicts in 
    the next step).
-3. Run `./gradlew dockerUp` (Gradle tasks are included as shortcuts for running Docker commands). This will start up
-   a 3-node cluster with a load balancer in front of it. Additionally, the 8000/8001/8002 ports are available on the 
-   "bootstrap" node of the cluster for accessing the out-of-the-box MarkLogic applications.
+3. Run `docker-compose up -d --build`.
+
+The above will result in a new MarkLogic instance with a single node. 
+
+Alternatively, if you would like to test against a 3-node MarkLogic cluster with a load balancer in front of it, 
+run `docker-compose -f docker-compose-3nodes.yaml up -d --build`.
 
 ### Accessing MarkLogic logs in Grafana
 
@@ -85,7 +89,7 @@ This will produce a single jar file for the connector in the `./build/libs` dire
 
 You can then launch PySpark with the connector available via:
 
-    pyspark --jars build/libs/marklogic-spark-connector-2.0-SNAPSHOT.jar
+    pyspark --jars build/libs/marklogic-spark-connector-2.1.0.jar
 
 The below command is an example of loading data from the test application deployed via the instructions at the top of 
 this page. 
@@ -114,3 +118,7 @@ more commands you can try out.
 
 See the section with the same name in the 
 [MarkLogic Koop contributing guide](https://github.com/koopjs/koop-provider-marklogic/blob/master/CONTRIBUTING.md).
+
+If you are looking to test the examples in the documentation, please be sure to follow the instructions in the 
+"Getting Started" guide. That involves creating an application in MarkLogic that has an app server listening on port
+8003. You will use that app server instead of the test-app server on port 8016. 

build.gradle

@@ -1,14 +1,14 @@
 plugins {
   id 'java-library'
   id 'net.saliman.properties' version '1.5.2'
-  id 'com.github.johnrengelman.shadow' version '7.1.2'
-  id "com.marklogic.ml-gradle" version "4.5.2"
+  id 'com.github.johnrengelman.shadow' version '8.1.1'
+  id "com.marklogic.ml-gradle" version "4.6.0"
   id 'maven-publish'
   id 'signing'
 }
 
 group 'com.marklogic'
-version '2.0.0'
+version '2.1.0'
 
 java {
   sourceCompatibility = 1.8
@@ -21,7 +21,15 @@ repositories {
 
 dependencies {
   compileOnly 'org.apache.spark:spark-sql_2.12:' + sparkVersion
-  implementation "com.marklogic:marklogic-client-api:6.2.1"
+  implementation ("com.marklogic:marklogic-client-api:6.4.0") {
+    // The Java Client uses Jackson 2.15.2; Scala 3.4.x does not yet support that and will throw the following error:
+    // Scala module 2.14.2 requires Jackson Databind version >= 2.14.0 and < 2.15.0 - Found jackson-databind version 2.15.2
+    // So the 4 Jackson modules are excluded to allow for Spark's to be used.
+    exclude module: 'jackson-core'
+    exclude module: 'jackson-databind'
+    exclude module: 'jackson-annotations'
+    exclude module: 'jackson-dataformat-csv'
+  }
 
   // Makes it possible to use lambdas in Java 8 to implement Spark's Function1 and Function2 interfaces
   // See https://github.com/scala/scala-java8-compat for more information
@@ -31,8 +39,21 @@ dependencies {
   }
 
   testImplementation 'org.apache.spark:spark-sql_2.12:' + sparkVersion
-  testImplementation 'com.marklogic:ml-app-deployer:4.5.2'
-  testImplementation 'com.marklogic:marklogic-junit5:1.3.0'
+
+  // The exclusions in these two modules ensure that we use the Jackson libraries from spark-sql when running the tests.
+  testImplementation ('com.marklogic:ml-app-deployer:4.6.0') {
+    exclude module: 'jackson-core'
+    exclude module: 'jackson-databind'
+    exclude module: 'jackson-annotations'
+    exclude module: 'jackson-dataformat-csv'
+  }
+  testImplementation ('com.marklogic:marklogic-junit5:1.4.0') {
+    exclude module: 'jackson-core'
+    exclude module: 'jackson-databind'
+    exclude module: 'jackson-annotations'
+    exclude module: 'jackson-dataformat-csv'
+  }
+
   testImplementation "ch.qos.logback:logback-classic:1.3.5"
   testImplementation "org.slf4j:jcl-over-slf4j:1.7.36"
   testImplementation "org.skyscreamer:jsonassert:1.5.1"
@@ -44,6 +65,14 @@ test {
   environment "mlHost", mlHost
 }
 
+task reloadTestData(type: com.marklogic.gradle.task.MarkLogicTask) {
+  description = "Convenience task for clearing the test database and reloading the test data; only intended for a connector developer to use."
+  doLast {
+    new com.marklogic.mgmt.resource.databases.DatabaseManager(getManageClient()).clearDatabase("spark-test-test-content")
+  }
+}
+reloadTestData.finalizedBy mlLoadData
+
 if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17)) {
   test {
     // See https://stackoverflow.com/questions/72724816/running-unit-tests-with-spark-3-3-0-on-java-17-fails-with-illegalaccesserror-cl
@@ -67,16 +96,11 @@ shadowJar {
 }
 
 task perfTest(type: JavaExec) {
-  main = "com.marklogic.spark.reader.PerformanceTester"
+  mainClass = "com.marklogic.spark.reader.PerformanceTester"
   classpath = sourceSets.test.runtimeClasspath
   args mlHost
 }
 
-task dockerUp(type: Exec) {
-  description = "Creates and starts a 3 node MarkLogic cluster."
-  commandLine "docker-compose", "up", "-d", "--build"
-}
-
 task dockerBuildCache(type: Exec) {
   description = "Creates an image named 'marklogic-spark-cache' containing a cache of the Gradle dependencies."
   commandLine 'docker', 'build', '--no-cache', '-t', 'marklogic-spark-cache', '.'
@@ -110,12 +134,12 @@ task dockerPerfTest(type: Exec) {
 }
 
 task sourcesJar(type: Jar, dependsOn: classes) {
-  classifier 'sources'
+  archiveClassifier = "sources"
   from sourceSets.main.allSource
 }
 
 task javadocJar(type: Jar, dependsOn: javadoc) {
-  classifier "javadoc"
+  archiveClassifier = "javadoc"
   from javadoc
 }
 javadoc.failOnError = false
@@ -190,6 +214,6 @@ task gettingStartedZip(type: Zip) {
   from "examples/getting-started"
   exclude "build", ".gradle", "gradle-*.properties"
   into "marklogic-spark-getting-started-${version}"
-  archiveName "marklogic-spark-getting-started-${version}.zip"
-  destinationDir(file('build'))
+  archiveFileName = "marklogic-spark-getting-started-${version}.zip"
+  destinationDirectory = file("build")
 }

docker-compose-3nodes.yaml

@@ -0,0 +1,145 @@
+version: '3.8'
+name: marklogic_spark_3nodes
+
+services:
+
+  # See https://caddyserver.com/docs/quick-starts/reverse-proxy for more information on using Caddy as a reverse proxy
+  # and load balancer.
+  caddy-load-balancer:
+    image: caddy:2-alpine
+    volumes:
+      - ./caddy/data:/data
+      - ./caddy/config/Caddyfile:/etc/caddy/Caddyfile
+    depends_on:
+      - bootstrap_3n
+      - node2
+      - node3
+    ports:
+      # For running marklogic-spark-connector tests against the 3-node cluster
+      - 8016:8816
+      - 8015:8815
+      # For running performance tests against quick-table data
+      - 8009:8809
+      # For the getting-started project
+      - 8020:8820
+    networks:
+      - external_net
+      - internal_net
+
+  # Named "bootstrap" as it's the first MarkLogic host created. Other MarkLogic hosts will then join the cluster created
+  # by this host. Note that each MarkLogic host has its 8000-8002 ports exposed externally so that the apps on those
+  # ports can each be accessed if needed.
+  bootstrap_3n:
+    image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0"
+    platform: linux/amd64
+    container_name: bootstrap_3n
+    hostname: bootstrap_3n.local
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+    volumes:
+      - ./docker/marklogic/logs/bootstrap_3n:/var/opt/MarkLogic/Logs
+    ports:
+      - 8000-8002:8000-8002
+    networks:
+      - external_net
+      - internal_net
+
+  node2:
+    image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0"
+    platform: linux/amd64
+    container_name: node2
+    hostname: node2.local
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n.local
+    volumes:
+      - ./docker/marklogic/logs/node2:/var/opt/MarkLogic/Logs
+    depends_on:
+      - bootstrap_3n
+    ports:
+      - 8100-8102:8000-8002
+    networks:
+      - external_net
+      - internal_net
+
+  node3:
+    image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0"
+    platform: linux/amd64
+    container_name: node3
+    hostname: node3.local
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
+    environment:
+      - MARKLOGIC_INIT=true
+      - MARKLOGIC_ADMIN_USERNAME=admin
+      - MARKLOGIC_ADMIN_PASSWORD=admin
+      - MARKLOGIC_JOIN_CLUSTER=true
+      - MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n.local
+    volumes:
+      - ./docker/marklogic/logs/node3:/var/opt/MarkLogic/Logs
+    depends_on:
+      - bootstrap_3n
+    ports:
+      - 8200-8202:8000-8002
+    networks:
+      - external_net
+      - internal_net
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - 3000:3000
+    volumes:
+      - ./config/grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yaml
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+    networks:
+      - external_net
+      - internal_net
+
+  loki:
+    image: grafana/loki:latest
+    ports:
+      - 3100:3100
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      - external_net
+      - internal_net
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: promtail
+    volumes:
+      - ./config/promtail.yaml:/etc/promtail/docker-config.yaml
+      - ./docker/marklogic/logs:/var/log/marklogic
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: -config.file=/etc/promtail/docker-config.yaml
+    depends_on:
+      - loki
+      - bootstrap_3n
+      - node2
+      - node3
+    networks:
+      - internal_net
+
+networks:
+  external_net: { }
+  internal_net:
+    internal: true
+    driver_opts:
+      com.docker.network.bridge.enable_icc: "true"