test: issue http request with a valid user agent

Some suites (e.g., rdf-streaming-updater, rdf-spark-tools)
were failing because they accessed https://schema.wikimedia.org
with an invalid user agent, resulting in a 403.

This patch updates the default WDQS UA string
and ensures all requests use it.

Bug: T406064
Change-Id: If294b2ea71b441778e7533c235e9523817f3f300

Author: gmodena

rdf-spark-tools/src/test/scala/org/wikidata/query/rdf/updater/reconcile/ReconciliationSenderUnitTest.scala

@@ -3,12 +3,10 @@ package org.wikidata.query.rdf.updater.reconcile
 import java.io.IOException
 import java.net.URI
 import java.time.Instant
-
 import java.util.{UUID, function}
 import scala.collection.JavaConverters._
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
-
 import com.fasterxml.jackson.databind.node.ArrayNode
 import org.apache.commons.io.IOUtils
 import org.apache.http.HttpVersion
@@ -20,7 +18,7 @@ import org.scalamock.matchers.ArgCapture.CaptureAll
 import org.scalamock.scalatest.MockFactory
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
-import org.wikidata.query.rdf.tool.{EntityId, MapperUtils}
+import org.wikidata.query.rdf.tool.{EntityId, HttpClientUtils, MapperUtils}
 import org.wikidata.query.rdf.tool.change.events.{EventInfo, EventsMeta, ReconcileEvent}
 import org.wikidata.query.rdf.tool.change.events.ReconcileEvent.Action
 import org.wikimedia.eventutilities.core.event.{EventSchemaLoader, EventSchemaValidator}
@@ -35,7 +33,11 @@ class ReconciliationSenderUnitTest extends AnyFlatSpec with Matchers with MockFa
   private val domain: String = "mydomain"
   private val mystream: String = "mystream"
   private val defaultLoader: function.Function[URI, Array[Byte]] = new function.Function[URI, Array[Byte]]() {
-    override def apply(u: URI): Array[Byte] = IOUtils.toByteArray(u)
+    override def apply(u: URI): Array[Byte] = {
+      val connection = u.toURL.openConnection()
+      connection.setRequestProperty("User-Agent", HttpClientUtils.WDQS_DEFAULT_UA)
+      IOUtils.toByteArray(connection.getInputStream)
+    }
   }
   private val resLoader: ResourceLoader = ResourceLoader.builder()
     .setDefaultLoader(defaultLoader)
@@ -65,6 +67,7 @@ class ReconciliationSenderUnitTest extends AnyFlatSpec with Matchers with MockFa
 
   "ReconciliationSender" should "send batch of events compatible with their schema" in {
     val httpClient = mock[HttpClient]
+
     val validResponse = stub[CloseableHttpResponse]
     val captureAllSuccess = CaptureAll[HttpUriRequest]()
 

streaming-updater-common/src/test/java/org/wikidata/query/rdf/updater/MutationEventDataJsonSerializationUnitTest.java

@@ -24,10 +24,12 @@
 import org.openrdf.model.Statement;
 import org.openrdf.rio.RDFFormat;
 import org.openrdf.rio.RDFWriterRegistry;
+import org.wikidata.query.rdf.tool.HttpClientUtils;
 import org.wikidata.query.rdf.tool.MapperUtils;
 import org.wikidata.query.rdf.tool.change.events.EventsMeta;
 import org.wikimedia.eventutilities.core.event.EventSchemaLoader;
 import org.wikimedia.eventutilities.core.event.EventSchemaValidator;
+import org.wikimedia.eventutilities.core.http.BasicHttpClient;
 import org.wikimedia.eventutilities.core.json.JsonLoadingException;
 import org.wikimedia.eventutilities.core.json.JsonSchemaLoader;
 import org.wikimedia.eventutilities.core.util.ResourceLoader;
@@ -56,10 +58,14 @@ public static Collection<Object[]> params() {
     }
 
     public MutationEventDataJsonSerializationUnitTest(String version) throws MalformedURLException {
+        BasicHttpClient.Builder builder = BasicHttpClient.builder();
+        builder.httpClientBuilder().setUserAgent(HttpClientUtils.WDQS_DEFAULT_UA);
+        BasicHttpClient httpClient = builder.build();
         eventSchemaLoader = EventSchemaLoader
                 .builder()
                 .setJsonSchemaLoader(JsonSchemaLoader.build(ResourceLoader
                         .builder()
+                        .withHttpClient(httpClient)
                         .setBaseUrls(Collections.singletonList(new URL("https://schema.wikimedia.org/repositories/primary/jsonschema")))
                         .build()))
                 .build();

streaming-updater-producer/src/test/scala/org/wikidata/query/rdf/updater/IncomingEventStreamsIntegrationTest.scala

@@ -4,11 +4,13 @@ import org.apache.flink.api.connector.sink2.{Sink, SinkWriter}
 import org.apache.flink.core.fs.Path
 import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
 import org.scalatest.{FlatSpec, Matchers}
+import org.wikidata.query.rdf.tool.HttpClientUtils
 import org.wikidata.query.rdf.tool.change.events.{EventInfo, EventsMeta}
 import org.wikidata.query.rdf.tool.wikibase.WikibaseRepository
 import org.wikidata.query.rdf.tool.wikibase.WikibaseRepository.Uris
 import org.wikidata.query.rdf.updater.config.{FilteredReconciliationStream, InputStreams, UpdaterPipelineInputEventStreamConfig}
 import org.wikimedia.eventutilities.core.event.{EventSchemaLoader, EventStreamConfig, EventStreamFactory, StaticEventStreamConfigLoader}
+import org.wikimedia.eventutilities.core.http.BasicHttpClient
 import org.wikimedia.eventutilities.core.json.{JsonLoader, JsonSchemaLoader}
 import org.wikimedia.eventutilities.core.util.ResourceLoader
 import org.wikimedia.eventutilities.flink.stream.EventDataStreamFactory
@@ -26,8 +28,15 @@ class IncomingEventStreamsIntegrationTest extends FlatSpec with FlinkTestCluster
     // useful to put test schemas while changes are being reviewed on the schemas repos
     this.getClass.getResource("/schema_repo/").toString)
 
+  private val httpClient: BasicHttpClient = {
+    val builder = BasicHttpClient.builder()
+    builder.httpClientBuilder().setUserAgent(HttpClientUtils.WDQS_DEFAULT_UA)
+    builder.build()
+  }
+
   private val resourceLoader = ResourceLoader.builder()
     .setBaseUrls(ResourceLoader.asURLs(schemaRepos.asJava))
+    .withHttpClient(httpClient)
     .build()
   private val jsonLoader: JsonLoader = new JsonLoader(resourceLoader)
   private val eventStreamConfigLoader = new StaticEventStreamConfigLoader(

streaming-updater-producer/src/test/scala/org/wikidata/query/rdf/updater/IncomingEventStreamsUnitTest.scala

@@ -2,9 +2,11 @@ package org.wikidata.query.rdf.updater
 
 import org.apache.flink.types.Row
 import org.scalatest.{FlatSpec, Matchers}
+import org.wikidata.query.rdf.tool.HttpClientUtils
 import org.wikidata.query.rdf.tool.change.events.{EventInfo, EventsMeta}
 import org.wikidata.query.rdf.tool.wikibase.WikibaseRepository.Uris
 import org.wikimedia.eventutilities.core.event.{EventSchemaLoader, EventStreamConfig, EventStreamFactory, StaticEventStreamConfigLoader}
+import org.wikimedia.eventutilities.core.http.BasicHttpClient
 import org.wikimedia.eventutilities.core.json.{JsonLoader, JsonSchemaLoader}
 import org.wikimedia.eventutilities.core.util.ResourceLoader
 import org.wikimedia.eventutilities.flink.stream.EventDataStreamFactory
@@ -22,8 +24,15 @@ class IncomingEventStreamsUnitTest extends FlatSpec with Matchers {
     // useful to put test schemas while changes are being reviewed on the schemas repos
     this.getClass.getResource("/schema_repo/").toString)
 
+  private val httpClient: BasicHttpClient = {
+    val builder = BasicHttpClient.builder()
+    builder.httpClientBuilder().setUserAgent(HttpClientUtils.WDQS_DEFAULT_UA)
+    builder.build()
+  }
+
   private val resourceLoader = ResourceLoader.builder()
     .setBaseUrls(ResourceLoader.asURLs(schemaRepos.asJava))
+    .withHttpClient(httpClient)
     .build()
   private val jsonLoader: JsonLoader = new JsonLoader(resourceLoader)
   private val eventStreamConfigLoader = new StaticEventStreamConfigLoader(

tools/src/main/java/org/wikidata/query/rdf/tool/HttpClientUtils.java

@@ -91,7 +91,7 @@ public final class HttpClientUtils {
     /**
      * Default HTTP User-Agent used by the system.
      */
-    public static final String WDQS_DEFAULT_UA = "Wikidata Query Service Updater Bot";
+    public static final String WDQS_DEFAULT_UA = "Wikidata-Query-RDF/1.0 (https://github.com/wikimedia/wikidata-query-rdf) Bot/1.0";
 
     /**
      * Max number of connection pooled per route.

tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryWireIntegrationTest.java

@@ -28,6 +28,7 @@
 import org.junit.Test;
 import org.openrdf.model.Statement;
 import org.wikidata.query.rdf.test.SystemPropertyContext;
+import org.wikidata.query.rdf.tool.HttpClientUtils;
 import org.wikidata.query.rdf.tool.change.Change;
 import org.wikidata.query.rdf.tool.exception.RetryableException;
 import org.wikidata.query.rdf.tool.rdf.RDFParserSuppliers;
@@ -159,7 +160,7 @@ public void defaultUserAgentIsSet() throws RetryableException {
 
         repository.fetchRdfForEntity("Q1");
 
-        verify(getRequestedFor(anyUrl()).withHeader("User-Agent", containing("Wikidata Query Service Updater")));
+        verify(getRequestedFor(anyUrl()).withHeader("User-Agent", containing(HttpClientUtils.WDQS_DEFAULT_UA)));
     }
 
     @Test