Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions CreateSnapshot/build.gradle
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
plugins {
id 'application'
id 'java'
id 'io.freefair.lombok'
id 'org.opensearch.migrations.java-application-conventions'
}
Expand Down
3 changes: 1 addition & 2 deletions DataGenerator/build.gradle
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
plugins {
id 'application'
id 'java'
id 'org.opensearch.migrations.java-application-conventions'
id 'io.freefair.lombok'
}

Expand Down
3 changes: 1 addition & 2 deletions DocumentsFromSnapshotMigration/build.gradle
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
plugins {
id 'application'
id 'java'
id 'org.opensearch.migrations.java-application-conventions'
id 'io.freefair.lombok'
id 'com.avast.gradle.docker-compose'
id 'com.bmuschko.docker-remote-api'
Expand Down
3 changes: 1 addition & 2 deletions MetadataMigration/build.gradle
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
plugins {
id 'application'
id 'java'
id 'org.opensearch.migrations.java-application-conventions'
id 'io.freefair.lombok'
}

Expand Down
1 change: 0 additions & 1 deletion RFS/build.gradle
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
plugins {
id 'org.opensearch.migrations.java-library-conventions'
id 'java'
id 'io.freefair.lombok'
id 'java-test-fixtures'
id 'me.champeau.jmh'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@

class RestClientTest {

private static final int SEND_CREATE_GET_SNAPSHOT_SIZE = 82;
private static final int SEND_CREATE_SNAPSHOT_SIZE = 139;
private static final int SEND_TOTAL_SIZE = SEND_CREATE_GET_SNAPSHOT_SIZE + SEND_CREATE_SNAPSHOT_SIZE;
private static final int READ_RESPONSE_SIZE = 66;
private static final int READ_TOTAL_SIZE = READ_RESPONSE_SIZE + READ_RESPONSE_SIZE;

private static HttpClient makeSingleConnectionHttpClient() {
var provider = ConnectionProvider.builder("singleConnection").maxConnections(1).build();
return HttpClient.create(provider);
Expand All @@ -46,11 +52,11 @@ public void testGetEmitsInstrumentation() throws Exception {

for (var kvp : Map.of(
"createGetSnapshotContext",
new long[] { 101, 66 },
new long[] { SEND_CREATE_GET_SNAPSHOT_SIZE, READ_RESPONSE_SIZE },
"createSnapshotContext",
new long[] { 139, 66 },
new long[] { SEND_CREATE_SNAPSHOT_SIZE, READ_RESPONSE_SIZE },
"",
new long[] { 240, 132 }
new long[] { SEND_TOTAL_SIZE, READ_TOTAL_SIZE }
).entrySet()) {
long bytesSent = allMetricData.stream()
.filter(md -> md.getName().startsWith("bytesSent"))
Expand Down Expand Up @@ -109,7 +115,7 @@ public void testGetEmitsInstrumentation() throws Exception {
.sorted(Comparator.comparing(SpanData::getEndEpochNanos))
.collect(Collectors.toList());
int i = 0;
for (var expectedBytes : List.of(new long[] { 139, 66 }, new long[] { 101, 66 })) {
for (var expectedBytes : List.of(new long[] { SEND_CREATE_SNAPSHOT_SIZE, READ_RESPONSE_SIZE }, new long[] { SEND_CREATE_GET_SNAPSHOT_SIZE, READ_RESPONSE_SIZE })) {
var span = httpRequestSpansByTime.get(i++);
long bytesSent = span.getAttributes().get(RfsContexts.GenericRequestContext.BYTES_SENT_ATTR);
long bytesRead = span.getAttributes().get(RfsContexts.GenericRequestContext.BYTES_READ_ATTR);
Expand Down
1 change: 1 addition & 0 deletions TrafficCapture/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ subprojects {
options.compilerArgs += [
"-XDcompilePolicy=simple",
"-Xplugin:ErrorProne -XepDisableAllChecks -Xep:MustBeClosed:ERROR -XepDisableWarningsInGeneratedCode",
"--should-stop=ifError=FLOW"
]
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import io.netty.buffer.Unpooled;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.ApiVersions;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.MockProducer;
import org.apache.kafka.clients.producer.Producer;
Expand All @@ -29,6 +28,7 @@
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.record.AbstractRecords;
import org.apache.kafka.common.record.CompressionType;
import org.apache.kafka.common.record.RecordBatch;
import org.apache.kafka.common.serialization.ByteArraySerializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.Assertions;
Expand Down Expand Up @@ -58,6 +58,7 @@ public void testLargeRequestIsWithinKafkaMessageSizeLimit() throws IOException,
int maxAllowableMessageSize = 1024 * 1024;
MockProducer<String, byte[]> producer = new MockProducer<>(
true,
null,
new StringSerializer(),
new ByteArraySerializer()
);
Expand Down Expand Up @@ -95,7 +96,7 @@ private static ConnectionContext createCtx() {
/**
* This size calculation is based off the KafkaProducer client request size validation check done when Producer
* records are sent. This validation appears to be consistent for several versions now, here is a reference to
* version 3.5 at the time of writing this: https://github.com/apache/kafka/blob/3.5/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java#L1030-L1032.
* version 3.5 at the time of writing this: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java#L1002-L1003.
* It is, however, subject to change which may make this test scenario more suited for an integration test where
* a KafkaProducer does not need to be mocked.
*/
Expand All @@ -105,11 +106,10 @@ private int calculateRecordSize(ProducerRecord<String, byte[]> record, String re
String recordKey = recordKeySubstitute == null ? record.key() : recordKeySubstitute;
byte[] serializedKey = stringSerializer.serialize(record.topic(), record.headers(), recordKey);
byte[] serializedValue = byteArraySerializer.serialize(record.topic(), record.headers(), record.value());
ApiVersions apiVersions = new ApiVersions();
stringSerializer.close();
byteArraySerializer.close();
return AbstractRecords.estimateSizeInBytesUpperBound(
apiVersions.maxUsableProduceMagic(),
RecordBatch.CURRENT_MAGIC_VALUE,
CompressionType.NONE,
serializedKey,
serializedValue,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.opensearch.migrations.replay.datatypes;

import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;

public interface ISourceTrafficChannelKey {
Expand All @@ -10,6 +11,7 @@ public interface ISourceTrafficChannelKey {

@Getter
@AllArgsConstructor
@EqualsAndHashCode
class PojoImpl implements ISourceTrafficChannelKey {
String nodeId;
String connectionId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import lombok.EqualsAndHashCode;

@EqualsAndHashCode()
@EqualsAndHashCode(callSuper = true)
public abstract class PojoTrafficStreamKey extends ISourceTrafficChannelKey.PojoImpl implements ITrafficStreamKey {
protected final int trafficStreamIndex;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.util.Map;
import java.util.Optional;
import java.util.PriorityQueue;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
Expand Down Expand Up @@ -337,7 +338,7 @@ private ConsumerRecords<String, byte[]> safePollWithSwallowedRuntimeExceptions(
log.atTrace().setMessage("All previously COMMITTED positions: {{}}")
.addArgument(() -> kafkaConsumer.assignment()
.stream()
.map(tp -> tp + ": " + kafkaConsumer.committed(tp))
.map(tp -> tp + ": " + kafkaConsumer.committed(Set.of(tp)))
.collect(Collectors.joining(",")))
.log();
return records;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,23 +1,21 @@
package org.opensearch.migrations.replay.util;

import javax.annotation.Nullable;

import com.google.errorprone.annotations.MustBeClosed;
import io.netty.util.ReferenceCountUtil;

public class RefSafeHolder<T> implements AutoCloseable {
private final T resource;

private RefSafeHolder(@Nullable T resource) {
private RefSafeHolder(T resource) {
this.resource = resource;
}

@MustBeClosed
public static <T> RefSafeHolder<T> create(@Nullable T resource) {
public static <T> RefSafeHolder<T> create(T resource) {
return new RefSafeHolder<>(resource);
}

public @Nullable T get() {
public T get() {
return resource;
}

Expand Down
1 change: 0 additions & 1 deletion awsUtilities/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
plugins {
id 'org.opensearch.migrations.java-library-conventions'
id 'io.freefair.lombok'
id 'java'
}

dependencies {
Expand Down
20 changes: 10 additions & 10 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ plugins {
// Observing higher memory usage with task-tree plugin. Disabling except if needed
// id "com.dorongold.task-tree" version "2.1.1"
id "com.diffplug.spotless" version '7.0.2'
id 'io.freefair.lombok' version '8.12' apply false
id 'io.freefair.lombok' version '8.13' apply false
id 'jacoco'
id 'me.champeau.jmh' version '0.7.2' apply false
id 'com.gradleup.shadow' version '8.3.5' apply false
id 'me.champeau.jmh' version '0.7.3' apply false
id 'com.gradleup.shadow' version '8.3.6' apply false
id 'com.avast.gradle.docker-compose' version "0.17.12" apply false
id 'com.google.protobuf' version "0.9.4" apply false
}
Expand Down Expand Up @@ -117,14 +117,14 @@ subprojects {
force 'net.minidev:json-smart:2.5.2'
force 'xalan:xalan:2.7.3'
force 'com.thoughtworks.xstream:xstream:1.4.21'
force 'commons-net:commons-net:3.9.0'
force 'org.jsoup:jsoup:1.15.3'
force 'org.apache.tika:tika-core:1.28.4'
force 'commons-net:commons-net:3.11.1'
force 'org.jsoup:jsoup:1.19.1'
force 'org.apache.tika:tika-core:3.1.0'
force 'com.jayway.jsonpath:json-path:2.9.0'
force 'dnsjava:dnsjava:3.6.0'
force 'org.xerial.snappy:snappy-java:1.1.10.6'
force 'org.glassfish.jersey.core:jersey-common:2.34'
force 'org.apache.httpcomponents:httpclient:4.5.13'
force 'dnsjava:dnsjava:3.6.3'
force 'org.xerial.snappy:snappy-java:1.1.10.7'
force 'org.glassfish.jersey.core:jersey-common:3.1.10'
force 'org.apache.httpcomponents:httpclient:4.5.14'

preferProjectModules()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,9 @@ plugins {
}

dependencies {
constraints {
// Define dependency versions as constraints
implementation 'org.apache.commons:commons-text:1.10.0'
}

// Use JUnit Jupiter for testing.
testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3'
testImplementation libs.junit.jupiter
testRuntimeOnly libs.junit.platform.launcher
}

tasks.named('test') {
Expand Down
1 change: 0 additions & 1 deletion coreUtilities/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
plugins {
id 'org.opensearch.migrations.java-library-conventions'
id 'io.freefair.lombok'
id 'java'
id 'java-test-fixtures'
}

Expand Down
4 changes: 1 addition & 3 deletions dashboardsSanitizer/build.gradle
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
plugins {
id 'application'
id 'org.opensearch.migrations.java-library-conventions'
id 'org.opensearch.migrations.java-application-conventions'
id 'io.freefair.lombok'
id 'java'
}

repositories {
Expand Down
72 changes: 38 additions & 34 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
@@ -1,44 +1,45 @@
[versions]
aws-crt = "0.29.18"
aws-msk-iam-auth = "2.0.3"
aws-sdk = "2.25.19"
bouncycastle = "1.78"
aws-crt = "0.36.3"
aws-msk-iam-auth = "2.3.1"
aws-sdk = "2.31.4"
bouncycastle = "1.80"
commons-collections = "4.4"
commons-compress = "1.26.0"
commons-compress = "1.27.1"
disruptor = "4.0.0"
dockerapi = "3.3.6"
errorprone = "2.32.0"
graalvm = "24.1.1"
groovy = "3.0.23"
guava = "32.0.1-jre"
hamcrest = "2.2"
httpclient5 = "5.2.1"
jackson = "2.16.2"
jcommander = "1.85"
dockerapi = "3.4.2"
errorprone = "2.37.0"
graalvm = "24.2.0"
groovy = "3.0.24"
guava = "33.4.5-jre"
hamcrest = "3.0"
httpclient5 = "5.4.2"
jackson = "2.18.3"
jcommander = "2.0"
jenkins-pipeline-unit = "1.23"
jmespath-core = "0.6.0"
jmeter = "5.6.3"
jmh-core = "1.36"
jolt-core = "0.1.7"
json = "20240303"
jupiter = "5.10.2"
kafka-clients = "3.6.0"
log4j = "2.23.1"
lombok = "1.18.30"
jmh-core = "1.37"
jolt-core = "0.1.8"
json = "20250107"
junit-platform = "1.12.1"
junit-jupiter = "5.12.1"
kafka-clients = "4.0.0"
log4j = "2.24.3"
lombok = "1.18.36"
lucene6 = '6.6.6'
lucene7 = '7.7.3'
lucene9 = "9.12.1"
mockito = "5.11.0"
netty = "4.1.108.Final"
mockito = "5.16.1"
netty = "4.1.119.Final"
opensearch-testcontainers = "2.0.1"
opentelemetry = "1.34.1"
opentelemetry = "1.48.0"
opentelemetry-semconv = "1.23.1-alpha"
protobuf = "3.25.5"
reactor-netty = "1.1.18"
reactor-test = "3.7.3"
semver4j = "5.3.0"
slf4j = "2.0.13"
testcontainers = "1.19.7"
protobuf = "4.30.1"
reactor-netty = "1.2.4"
reactor-test = "3.7.4"
semver4j = "5.6.0"
slf4j = "2.0.17"
testcontainers = "1.20.6"
toxiproxy = "2.1.7"

[libraries]
Expand Down Expand Up @@ -95,10 +96,13 @@ jolt-core = { module = "com.bazaarvoice.jolt:jolt-core", version.ref = "jolt-cor

json = { module = "org.json:json", version.ref = "json" }

junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "jupiter" }
junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "jupiter" }
junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "jupiter" }
junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "jupiter" }
junit-platform-launcher = { module = "org.junit.platform:junit-platform-launcher", version.ref = "junit-platform" }

junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit-jupiter" }
junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit-jupiter" }
junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit-jupiter" }
junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit-jupiter" }


kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafka-clients" }

Expand Down
1 change: 0 additions & 1 deletion transformation/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
plugins {
id 'org.opensearch.migrations.java-library-conventions'
id 'io.freefair.lombok'
id 'java'
id 'java-test-fixtures'
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
plugins {
id 'org.opensearch.migrations.java-library-conventions'
id 'io.freefair.lombok'
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
plugins {
id 'org.opensearch.migrations.java-library-conventions'
id 'io.freefair.lombok'
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
plugins {
id 'org.opensearch.migrations.java-library-conventions'
id 'io.freefair.lombok'
}

Expand Down
Loading
Loading