diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java index 6908b35e6..c7e895914 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java @@ -16,7 +16,9 @@ */ package com.google.edwmigration.dumper.application.dumper.connector; -import com.google.common.base.Preconditions; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.handle.Handle; import com.google.edwmigration.dumper.application.dumper.task.Task; @@ -50,25 +52,21 @@ default String getDescription() { * @param arguments cli params * @throws RuntimeException if incorrect set of arguments passed to the particular connector */ - default void validate(ConnectorArguments arguments) {} + default void validate(@Nonnull ConnectorArguments arguments) {} - default void validateDateRange(ConnectorArguments arguments) { - ZonedDateTime startDate = arguments.getStartDate(); - ZonedDateTime endDate = arguments.getEndDate(); + static void validateDateRange(@Nonnull ConnectorArguments arguments) { + ZonedDateTime start = arguments.getStartDate(); + ZonedDateTime end = arguments.getEndDate(); - if (startDate != null) { - Preconditions.checkNotNull( - endDate, "End date must be specified with start date, but was null."); - Preconditions.checkState( - startDate.isBefore(endDate), - "Start date [%s] must be before end date [%s].", - startDate, - endDate); - } else { - Preconditions.checkState( - endDate == null, - "End date can be specified only with start date, but start date was null."); + if (start == null && end == null) { + return; } + checkNotNull(start, "End date can be specified only with start date, but start date was null."); + // The assignment makes 'end' recognized as @Nonnull. + end = checkNotNull(end, "End date must be specified with start date, but was null."); + + String message = String.format("Start date [%s] must be before end date [%s].", start, end); + checkState(end.isAfter(start), message); } void addTasksTo(@Nonnull List> out, @Nonnull ConnectorArguments arguments) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/airflow/AirflowConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/airflow/AirflowConnector.java index a166f176e..60f801591 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/airflow/AirflowConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/airflow/AirflowConnector.java @@ -17,6 +17,7 @@ package com.google.edwmigration.dumper.application.dumper.connector.airflow; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.edwmigration.dumper.application.dumper.connector.Connector.validateDateRange; import static com.google.edwmigration.dumper.application.dumper.connector.airflow.AirflowDatabaseDriverClasses.jdbcPrefixForClassName; import com.google.auto.service.AutoService; @@ -209,7 +210,7 @@ private static void addQueryTask( } @Override - public void validate(ConnectorArguments arguments) { + public final void validate(@Nonnull ConnectorArguments arguments) { Preconditions.checkState(arguments.isAssessment(), "--assessment flag is required"); Preconditions.checkState( arguments.getDriverPaths() != null && !arguments.getDriverPaths().isEmpty(), diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java index d2b4ae136..2c6535b1f 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java @@ -16,6 +16,7 @@ */ package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager; +import static com.google.edwmigration.dumper.application.dumper.connector.Connector.validateDateRange; import static com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager.AbstractClouderaTimeSeriesTask.TimeSeriesAggregation.DAILY; import static com.google.edwmigration.dumper.application.dumper.task.TaskCategory.OPTIONAL; @@ -129,7 +130,7 @@ public ClouderaManagerHandle open(@Nonnull ConnectorArguments arguments) throws } @Override - public void validate(ConnectorArguments arguments) { + public final void validate(@Nonnull ConnectorArguments arguments) { String clouderaUri = arguments.getUri(); Preconditions.checkNotNull(clouderaUri, "--url for Cloudera Manager API is required"); diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/oozie/OozieConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/oozie/OozieConnector.java index ecaf53040..56e61b18f 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/oozie/OozieConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/oozie/OozieConnector.java @@ -16,6 +16,8 @@ */ package com.google.edwmigration.dumper.application.dumper.connector.hadoop.oozie; +import static com.google.edwmigration.dumper.application.dumper.connector.Connector.validateDateRange; + import com.google.auto.service.AutoService; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -79,7 +81,7 @@ public String getDefaultFileName(boolean isAssessment, Clock clock) { } @Override - public void validate(ConnectorArguments arguments) { + public final void validate(@Nonnull ConnectorArguments arguments) { Preconditions.checkState(arguments.isAssessment(), "--assessment flag is required"); validateDateRange(arguments); diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/AbstractSnowflakeConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/AbstractSnowflakeConnector.java index 12b86ca74..2c79ccefa 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/AbstractSnowflakeConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/AbstractSnowflakeConnector.java @@ -101,9 +101,7 @@ public Handle open(@Nonnull ConnectorArguments arguments) } @Override - public void validate(ConnectorArguments arguments) { - super.validate(arguments); - + public final void validate(@Nonnull ConnectorArguments arguments) { ArrayList messages = new ArrayList<>(); MetadataDumperUsageException exception = null; @@ -133,8 +131,18 @@ public void validate(ConnectorArguments arguments) { exception = new MetadataDumperUsageException(unsupportedFilter, messages); } removeDuplicateMessageAndThrow(exception); + validateForConnector(arguments); } + /** + * Called by {@link #validate} to perform connector-specific checks. + * + *

Subclasses should override this with logic to run after the common validation for Snowflake. + * + * @param arguments User-provided arguments of the Dumper run. + */ + protected abstract void validateForConnector(@Nonnull ConnectorArguments arguments); + private static void removeDuplicateMessageAndThrow( @Nullable MetadataDumperUsageException exception) { if (exception != null) { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeLiteConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeLiteConnector.java index 374c8aac3..2ada797b5 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeLiteConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeLiteConnector.java @@ -20,15 +20,12 @@ import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.connector.Connector; -import com.google.edwmigration.dumper.application.dumper.handle.Handle; import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask; import com.google.edwmigration.dumper.application.dumper.task.FormatTask; import com.google.edwmigration.dumper.application.dumper.task.Task; import com.google.edwmigration.dumper.application.dumper.utils.ArchiveNameUtil; -import java.sql.SQLException; import java.time.Clock; import java.util.List; -import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.annotation.ParametersAreNonnullByDefault; @@ -46,13 +43,6 @@ public SnowflakeLiteConnector() { super(NAME); } - @Override - @Nonnull - public Handle open(ConnectorArguments arguments) - throws MetadataDumperUsageException, SQLException { - return super.open(arguments); - } - @Override @Nonnull public String getDefaultFileName(boolean isAssessment, @Nullable Clock clock) { @@ -73,10 +63,7 @@ public final void addTasksTo(List> out, ConnectorArguments argum } @Override - public final void validate(@Nullable ConnectorArguments arguments) { - super.validate(arguments); - - Objects.requireNonNull(arguments); + protected void validateForConnector(ConnectorArguments arguments) { if (!arguments.isAssessment()) { throw noAssessmentException(); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeLogsConnector.java index 888a4f9a8..e3145eaf6 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeLogsConnector.java @@ -161,9 +161,7 @@ private TaskDescription( } @Override - public final void validate(ConnectorArguments arguments) { - super.validate(arguments); - + protected final void validateForConnector(@Nonnull ConnectorArguments arguments) { if (arguments.isAssessment() && arguments.hasQueryLogEarliestTimestamp()) { throw unsupportedOption(ConnectorArguments.OPT_QUERY_LOG_EARLIEST_TIMESTAMP); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeMetadataConnector.java index 70db23e0f..fb64e98ea 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeMetadataConnector.java @@ -129,6 +129,9 @@ public Iterable getPropertyConstants() { return builder.build(); } + @Override + protected final void validateForConnector(@Nonnull ConnectorArguments arguments) {} + private void addSqlTasksWithInfoSchemaFallback( @Nonnull List> out, @Nonnull Class> header, diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/ConnectorTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/ConnectorTest.java index 9e5ee0b37..176bc6794 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/ConnectorTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/ConnectorTest.java @@ -16,58 +16,55 @@ */ package com.google.edwmigration.dumper.application.dumper.connector; +import static com.google.edwmigration.dumper.application.dumper.connector.Connector.validateDateRange; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; -import com.google.edwmigration.dumper.application.dumper.handle.Handle; -import com.google.edwmigration.dumper.application.dumper.task.Task; -import java.time.Clock; -import java.util.List; -import javax.annotation.Nonnull; import org.junit.Test; public class ConnectorTest { - private final Connector connector = new EmptyConnector(); @Test public void validateDateRange_startDateAndEndDate_success() throws Exception { - String argsStr = "--connector test --start-date=2001-02-20 --end-date=2001-02-25"; + ConnectorArguments args = + new ConnectorArguments( + "--connector", "test", "--start-date=2001-02-20", "--end-date=2001-02-25"); // Act - connector.validateDateRange(toArgs(argsStr)); + validateDateRange(args); } @Test - public void validateDateRange_startDateAfterEndDate_throws() { - String argsStr = "--connector test --start-date=2001-02-20 --end-date=2001-02-20"; + public void validateDateRange_startDateAfterEndDate_throws() throws Exception { + ConnectorArguments args = + new ConnectorArguments( + "--connector", "test", "--start-date=2001-02-20", "--end-date=2001-02-20"); Exception exception = - assertThrows( - IllegalStateException.class, () -> connector.validateDateRange(toArgs(argsStr))); + assertThrows(RuntimeException.class, () -> Connector.validateDateRange(args)); assertEquals( "Start date [2001-02-20T00:00Z] must be before end date [2001-02-20T00:00Z].", exception.getMessage()); } @Test - public void validateDateRange_endDateAlone_throws() { - String argsStr = "--connector test --end-date=2001-02-20"; + public void validateDateRange_endDateAlone_throws() throws Exception { + ConnectorArguments args = + new ConnectorArguments("--connector", "test", "--end-date=2001-02-20"); - Exception exception = - assertThrows( - IllegalStateException.class, () -> connector.validateDateRange(toArgs(argsStr))); + Exception exception = assertThrows(RuntimeException.class, () -> validateDateRange(args)); assertEquals( "End date can be specified only with start date, but start date was null.", exception.getMessage()); } @Test - public void validateDateRange_startDateAlone_throws() { - String argsStr = "--connector test --start-date=2001-02-20"; + public void validateDateRange_startDateAlone_throws() throws Exception { + ConnectorArguments args = + new ConnectorArguments("--connector", "test", "--start-date=2001-02-20"); - Exception exception = - assertThrows(RuntimeException.class, () -> connector.validateDateRange(toArgs(argsStr))); + Exception exception = assertThrows(RuntimeException.class, () -> validateDateRange(args)); assertEquals( "End date must be specified with start date, but was null.", exception.getMessage()); } @@ -75,42 +72,6 @@ public void validateDateRange_startDateAlone_throws() { @Test public void validateDateRange_requiredArgs_success() throws Exception { // Act - connector.validateDateRange(toArgs("--connector test")); - } - - private static class EmptyConnector implements Connector { - - @Nonnull - @Override - public String getName() { - return null; - } - - @Nonnull - @Override - public String getDefaultFileName(boolean isAssessment, Clock clock) { - return null; - } - - @Override - public void addTasksTo( - @Nonnull List> out, @Nonnull ConnectorArguments arguments) - throws Exception {} - - @Nonnull - @Override - public Handle open(@Nonnull ConnectorArguments arguments) throws Exception { - return null; - } - - @Nonnull - @Override - public Iterable getPropertyConstants() { - return null; - } - } - - private static ConnectorArguments toArgs(String args) throws Exception { - return new ConnectorArguments(args.split(" ")); + validateDateRange(new ConnectorArguments("--connector", "test")); } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/airflow/AirflowConnectorTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/airflow/AirflowConnectorTest.java index 027697d64..c771ef5c5 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/airflow/AirflowConnectorTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/airflow/AirflowConnectorTest.java @@ -58,7 +58,7 @@ public void validate_startDateAfterEndDate_throws() { String argsStr = validRequiredArgs + " --start-date=2001-02-20 --end-date=2001-02-20"; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsStr))); + assertThrows(RuntimeException.class, () -> connector.validate(args(argsStr))); assertEquals( "Start date [2001-02-20T00:00Z] must be before end date [2001-02-20T00:00Z].", exception.getMessage()); @@ -69,7 +69,7 @@ public void validate_endDateAlone_throws() { String argsStr = validRequiredArgs + " --end-date=2001-02-20"; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsStr))); + assertThrows(RuntimeException.class, () -> connector.validate(args(argsStr))); assertEquals( "End date can be specified only with start date, but start date was null.", exception.getMessage()); @@ -94,7 +94,7 @@ public void validate_databaseParam_isNotSupported() throws Exception { // Act Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsStr))); + assertThrows(RuntimeException.class, () -> connector.validate(args(argsStr))); assertEquals("--database is not supported, use --schema or --url", exception.getMessage()); } @@ -117,19 +117,19 @@ public void validate_hostParam_brokenState() throws Exception { { String argsWithHost = argsStr + " --host localhost "; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsWithHost))); + assertThrows(RuntimeException.class, () -> connector.validate(args(argsWithHost))); assertEquals("--port is required with --host", exception.getMessage()); } { String argsWithPort = argsStr + " --host localhost --port 8080"; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsWithPort))); + assertThrows(RuntimeException.class, () -> connector.validate(args(argsWithPort))); assertEquals("--schema is required with --host", exception.getMessage()); } { String argsWithPort = argsStr + " --host localhost --schema 8080"; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsWithPort))); + assertThrows(RuntimeException.class, () -> connector.validate(args(argsWithPort))); assertEquals("--port is required with --host", exception.getMessage()); } } @@ -153,19 +153,19 @@ public void validate_jdbcParam_brokenState() throws Exception { { String argsWithPort = argsStr + " --port 8080"; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsWithPort))); + assertThrows(RuntimeException.class, () -> connector.validate(args(argsWithPort))); assertEquals("--port param should not be used with --url", exception.getMessage()); } { String argsWithSchema = argsStr + " --schema 8080"; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsWithSchema))); + assertThrows(RuntimeException.class, () -> connector.validate(args(argsWithSchema))); assertEquals("--schema param should not be used with --url", exception.getMessage()); } { String argsWithSchema = argsStr + " --host localhost"; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsWithSchema))); + assertThrows(RuntimeException.class, () -> connector.validate(args(argsWithSchema))); assertEquals( "--url either --host must be provided (both parameters at once are not acceptable)", exception.getMessage()); @@ -178,33 +178,28 @@ public void validate_requiredArguments() throws Exception { { String argsStr = "--connector airflow "; - exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsStr))); + exception = assertThrows(RuntimeException.class, () -> connector.validate(args(argsStr))); assertEquals("--assessment flag is required", exception.getMessage()); } { String argsStr = "--connector airflow --assessment"; - exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsStr))); + exception = assertThrows(RuntimeException.class, () -> connector.validate(args(argsStr))); assertEquals("Path to jdbc driver is required in --driver param", exception.getMessage()); } { String argsStr = "--connector airflow --assessment --driver /home/dir/"; - exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsStr))); + exception = assertThrows(RuntimeException.class, () -> connector.validate(args(argsStr))); assertEquals("--user param is required", exception.getMessage()); } { String argsStr = "--connector airflow --driver /home/dir/ --user dbusername --assessment"; - exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsStr))); + exception = assertThrows(RuntimeException.class, () -> connector.validate(args(argsStr))); assertEquals("--password param is required", exception.getMessage()); } { String argsStr = "--connector airflow " + "--driver /home/dir/ --user dbusername --assessment --password"; - exception = - assertThrows(IllegalStateException.class, () -> connector.validate(args(argsStr))); + exception = assertThrows(RuntimeException.class, () -> connector.validate(args(argsStr))); assertEquals( "--url either --host must be provided (both parameters at once are not acceptable)", exception.getMessage()); diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/oozie/OozieConnectorTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/oozie/OozieConnectorTest.java index 08addf39f..807f4f679 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/oozie/OozieConnectorTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/oozie/OozieConnectorTest.java @@ -60,7 +60,7 @@ public void validate_startDateAfterEndDate_throws() { String argsStr = validRequiredArgs + " --start-date=2001-02-20 --end-date=2001-02-20"; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(toArgs(argsStr))); + assertThrows(RuntimeException.class, () -> connector.validate(toArgs(argsStr))); assertEquals( "Start date [2001-02-20T00:00Z] must be before end date [2001-02-20T00:00Z].", exception.getMessage()); @@ -71,7 +71,7 @@ public void validate_endDateAlone_throws() { String argsStr = validRequiredArgs + " --end-date=2001-02-20"; Exception exception = - assertThrows(IllegalStateException.class, () -> connector.validate(toArgs(argsStr))); + assertThrows(RuntimeException.class, () -> connector.validate(toArgs(argsStr))); assertEquals( "End date can be specified only with start date, but start date was null.", exception.getMessage()); @@ -97,8 +97,7 @@ public void validate_requiredArgs_success() throws Exception { public void validate_AssessmentIsRequired_throw() throws Exception { Exception exception = - assertThrows( - IllegalStateException.class, () -> connector.validate(toArgs("--connector oozie"))); + assertThrows(RuntimeException.class, () -> connector.validate(toArgs("--connector oozie"))); assertEquals("--assessment flag is required", exception.getMessage()); }