diff --git a/morf-avro/pom.xml b/morf-avro/pom.xml new file mode 100644 index 000000000..8f4551824 --- /dev/null +++ b/morf-avro/pom.xml @@ -0,0 +1,71 @@ + + + + morf-parent + org.alfasoftware + 2.0.5-SNAPSHOT + + 4.0.0 + Morf - Avro + + morf-avro + + + + org.apache.maven.plugins + maven-compiler-plugin + + 9 + 9 + + + + + + + + org.alfasoftware + morf-directory-spi + + + + org.apache.avro + avro + 1.11.0 + + + junit + junit + test + + + + com.google.jimfs + jimfs + test + 1.2 + + + org.mockito + mockito-core + test + + + org.slf4j + slf4j-api + 2.0.6 + + + + com.google.code.gson + gson + 2.9.0 + + + + + + + \ No newline at end of file diff --git a/morf-avro/src/main/java/org/alfasoftware/morf/avro/AvroDataSetConsumer.java b/morf-avro/src/main/java/org/alfasoftware/morf/avro/AvroDataSetConsumer.java new file mode 100644 index 000000000..03afa7e33 --- /dev/null +++ b/morf-avro/src/main/java/org/alfasoftware/morf/avro/AvroDataSetConsumer.java @@ -0,0 +1,71 @@ +package org.alfasoftware.morf.avro; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.alfasoftware.morf.dataset.DataSetConsumer; +import org.alfasoftware.morf.dataset.Record; +import org.alfasoftware.morf.directory.DirectoryDataSetConsumer; +import org.alfasoftware.morf.metadata.Column; +import org.alfasoftware.morf.metadata.DataType; +import org.alfasoftware.morf.metadata.DataValueLookup; +import org.alfasoftware.morf.metadata.Table; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumWriter; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.EnumMap; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +public class AvroDataSetConsumer extends DirectoryDataSetConsumer implements DataSetConsumer { + + private static final Gson GSON_FOR_TABLE_DATA = new GsonBuilder().setPrettyPrinting().registerTypeAdapter(Table.class, new JsonTableSerializer()).create(); + private static final Gson GSON_FOR_COLUMN_DATA = new GsonBuilder().setPrettyPrinting().registerTypeAdapter(Table.class, new JsonColumnSetSerializer()).create(); + private final Map> recordMappingFunctions = new EnumMap<>(DataType.class); + + public AvroDataSetConsumer(Path path) { + super("avro", path); + recordMappingFunctions.put(DataType.DATE, DataValueLookup::getDate); + recordMappingFunctions.put(DataType.BIG_INTEGER, DataValueLookup::getLong); + recordMappingFunctions.put(DataType.BLOB, DataValueLookup::getByteArray); + recordMappingFunctions.put(DataType.BOOLEAN, DataValueLookup::getBoolean); + recordMappingFunctions.put(DataType.CLOB, DataValueLookup::getByteArray); + recordMappingFunctions.put(DataType.DECIMAL, DataValueLookup::getBigDecimal); + recordMappingFunctions.put(DataType.INTEGER, DataValueLookup::getInteger); + recordMappingFunctions.put(DataType.STRING, DataValueLookup::getString); + } + + @Override + public void open() { + super.directoryOutputStreamProvider.open(); + } + + @Override + public void close(CloseState closeState) { + super.directoryOutputStreamProvider.close(); + } + + @Override + public void table(Table table, Iterable records) { + String gsonSchema = GSON_FOR_COLUMN_DATA.toJson(table, Table.class); + Map columnNamesAndTypes = table.columns().stream().collect(Collectors.toMap(Column::getName, Column::getType)); + Schema avroSchema = new Schema.Parser().parse(gsonSchema); + DatumWriter datumWriter = new GenericDatumWriter<>(avroSchema); + try (DataFileWriter dataFileWriter = new DataFileWriter<>(datumWriter)) { + dataFileWriter.create(avroSchema, super.directoryOutputStreamProvider.openOutputStreamForTable(table.getName())); + for (Record r : records) { + GenericRecord row = new GenericData.Record(avroSchema); + columnNamesAndTypes.entrySet().forEach(entry -> row.put(entry.getKey(), recordMappingFunctions.get(entry.getValue()).apply(r, entry.getKey()))); + dataFileWriter.append(row); + } + } catch (IOException e) { + throw new RuntimeException("Exception occurred upon trying to write avro data file for table [" + table.getName() + "]", e); + } + } +} diff --git a/morf-avro/src/main/java/org/alfasoftware/morf/avro/AvroDataSetProducer.java b/morf-avro/src/main/java/org/alfasoftware/morf/avro/AvroDataSetProducer.java new file mode 100644 index 000000000..89ca336a2 --- /dev/null +++ b/morf-avro/src/main/java/org/alfasoftware/morf/avro/AvroDataSetProducer.java @@ -0,0 +1,32 @@ +package org.alfasoftware.morf.avro; + +import org.alfasoftware.morf.dataset.DataSetProducer; +import org.alfasoftware.morf.dataset.Record; +import org.alfasoftware.morf.metadata.Schema; + +public class AvroDataSetProducer implements DataSetProducer { + @Override + public void open() { + + } + + @Override + public void close() { + + } + + @Override + public Schema getSchema() { + return null; + } + + @Override + public Iterable records(String tableName) { + return null; + } + + @Override + public boolean isTableEmpty(String tableName) { + return false; + } +} diff --git a/morf-avro/src/main/java/org/alfasoftware/morf/avro/JsonColumnSetSerializer.java b/morf-avro/src/main/java/org/alfasoftware/morf/avro/JsonColumnSetSerializer.java new file mode 100644 index 000000000..fce30f6d0 --- /dev/null +++ b/morf-avro/src/main/java/org/alfasoftware/morf/avro/JsonColumnSetSerializer.java @@ -0,0 +1,54 @@ +package org.alfasoftware.morf.avro; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import org.alfasoftware.morf.metadata.Column; +import org.alfasoftware.morf.metadata.DataType; +import org.alfasoftware.morf.metadata.Table; + +import java.lang.reflect.Type; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +public class JsonColumnSetSerializer implements JsonSerializer { + + private final Map DATA_TYPE_TO_AVRO_TYPE_MAP; + + JsonColumnSetSerializer() { + DATA_TYPE_TO_AVRO_TYPE_MAP = new EnumMap(DataType.class); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.DATE, "int"); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.BIG_INTEGER, "long"); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.BLOB, "string"); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.BOOLEAN, "boolean"); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.CLOB, "string"); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.DECIMAL, "double"); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.INTEGER, "int"); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.STRING, "string"); + } + + @Override + public JsonElement serialize(Table table, Type type, JsonSerializationContext jsonSerializationContext) { + JsonElement jsonElement = new JsonObject(); + jsonElement.getAsJsonObject().addProperty("namespace", "org.alfasoftware.morf"); + jsonElement.getAsJsonObject().addProperty("name", table.getName() + "_columns"); + jsonElement.getAsJsonObject().addProperty("type", "record"); + JsonElement fields = getColumnsAsFieldsFor(table.columns()); + jsonElement.getAsJsonObject().add("fields", fields); + return jsonElement; + } + + private JsonArray getColumnsAsFieldsFor(List columns) { + JsonArray array = new JsonArray(); + for (Column c : columns) { + JsonObject object = new JsonObject(); + object.getAsJsonObject().addProperty("name", c.getName()); + object.getAsJsonObject().addProperty("type", DATA_TYPE_TO_AVRO_TYPE_MAP.get(c.getType())); + array.getAsJsonArray().add(object); + } + return array; + } +} diff --git a/morf-avro/src/main/java/org/alfasoftware/morf/avro/JsonTableSerializer.java b/morf-avro/src/main/java/org/alfasoftware/morf/avro/JsonTableSerializer.java new file mode 100644 index 000000000..4369513b2 --- /dev/null +++ b/morf-avro/src/main/java/org/alfasoftware/morf/avro/JsonTableSerializer.java @@ -0,0 +1,66 @@ +package org.alfasoftware.morf.avro; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import org.alfasoftware.morf.metadata.Column; +import org.alfasoftware.morf.metadata.DataType; +import org.alfasoftware.morf.metadata.Table; + +import java.lang.reflect.Type; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +public class JsonTableSerializer implements JsonSerializer
{ + + private final Map DATA_TYPE_TO_AVRO_TYPE_MAP; + + JsonTableSerializer() { + DATA_TYPE_TO_AVRO_TYPE_MAP = new EnumMap(DataType.class); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.DATE, "int"); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.BIG_INTEGER, "long"); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.BLOB, "string"); +// recordMappingFunctions.put(DataType.BOOLEAN, (record, columnName) -> record.getBoolean(columnName)); +// recordMappingFunctions.put(DataType.CLOB, (record, columnName) -> record.getByteArray(columnName)); +// recordMappingFunctions.put(DataType.DECIMAL, (record, columnName) -> record.getBigDecimal(columnName)); +// recordMappingFunctions.put(DataType.INTEGER, (record, columnName) -> record.getInteger(columnName)); + DATA_TYPE_TO_AVRO_TYPE_MAP.put(DataType.STRING, "string"); + } + + @Override + public JsonElement serialize(Table table, Type type, JsonSerializationContext jsonSerializationContext) { + JsonElement jsonElement = new JsonObject(); + jsonElement.getAsJsonObject().addProperty("namespace", "org.alfasoftware.morf"); + jsonElement.getAsJsonObject().addProperty("name", "table"); + jsonElement.getAsJsonObject().addProperty("type", "record"); + JsonElement fields = new JsonArray(); + JsonElement isTemporary = new JsonObject(); + isTemporary.getAsJsonObject().addProperty("name", "isTemporary"); + isTemporary.getAsJsonObject().addProperty("type", "boolean"); + fields.getAsJsonArray().add(isTemporary); + JsonElement columnType = new JsonObject(); + columnType.getAsJsonObject().addProperty("name", "column"); + columnType.getAsJsonObject().addProperty("type", "record"); + columnType.getAsJsonObject().add("fields", getColumnsAsFieldsFor(table.columns())); + JsonElement columns = new JsonObject(); + columns.getAsJsonObject().addProperty("name", "columns"); + columns.getAsJsonObject().add("type", columnType); + fields.getAsJsonArray().add(columns); + jsonElement.getAsJsonObject().add("fields", fields); + return jsonElement; + } + + private JsonArray getColumnsAsFieldsFor(List columns) { + JsonArray array = new JsonArray(); + for (Column c : columns) { + JsonObject object = new JsonObject(); + object.getAsJsonObject().addProperty("name", c.getName()); + object.getAsJsonObject().addProperty("type", DATA_TYPE_TO_AVRO_TYPE_MAP.get(c.getType())); + array.getAsJsonArray().add(object); + } + return array; + } +} diff --git a/morf-avro/src/test/java/org/alfasoftware/morf/avro/TestAvroDataSetConsumer.java b/morf-avro/src/test/java/org/alfasoftware/morf/avro/TestAvroDataSetConsumer.java new file mode 100644 index 000000000..b7ff3c331 --- /dev/null +++ b/morf-avro/src/test/java/org/alfasoftware/morf/avro/TestAvroDataSetConsumer.java @@ -0,0 +1,305 @@ +package org.alfasoftware.morf.avro; + +import com.google.common.collect.Lists; +import com.google.common.jimfs.Jimfs; +import org.alfasoftware.morf.dataset.DataSetConnector; +import org.alfasoftware.morf.dataset.DataSetProducer; +import org.alfasoftware.morf.dataset.Record; +import org.alfasoftware.morf.metadata.Column; +import org.alfasoftware.morf.metadata.DataSetUtils; +import org.alfasoftware.morf.metadata.DataType; +import org.alfasoftware.morf.metadata.Index; +import org.alfasoftware.morf.metadata.Schema; +import org.alfasoftware.morf.metadata.Table; +import org.alfasoftware.morf.metadata.View; +import org.apache.avro.file.DataFileReader; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumReader; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import static java.nio.file.StandardOpenOption.CREATE_NEW; +import static org.junit.Assert.assertEquals; + +public class TestAvroDataSetConsumer { + + private static final String SUB_DIR_FOR_TEST_CLASS = TestAvroDataSetConsumer.class.getName(); + private FileSystem fileSystem; + private Path root; + + + @Before + public void setup() throws IOException { + fileSystem = Jimfs.newFileSystem(); + root = fileSystem.getPath("/" + SUB_DIR_FOR_TEST_CLASS); + Files.createDirectory(root); + } + + @Test + public void testOutputOneFile() throws IOException { + // Given + Path pathForThisTest = root.resolve("testOutputOneFile"); + Files.createDirectory(pathForThisTest); + AvroDataSetConsumer sut = new AvroDataSetConsumer(pathForThisTest); + LinkedHashMap inputs = new LinkedHashMap<>(); + inputs.put(1L, "Bran"); + inputs.put(2L, "Brannan"); + DataSetProducer producer = new StubDataSetProducer(inputs); + + // When + new DataSetConnector(producer, sut).connect(); + + // Then - read content from in memory file system. + DatumReader datumReader = new GenericDatumReader<>(); + Files.createDirectories(new File(System.getProperty("user.dir")).toPath().resolve("target/testOutputOneFile")); + // copy out of the in memory file system as avro doesn't support the java.nio paths api :( + Files.copy(pathForThisTest.resolve("avrotable.avro"), Files.newOutputStream(new File(System.getProperty("user.dir")).toPath().resolve("target/testOutputOneFile/avrotable.avro"))); + DataFileReader dataFileReader = new DataFileReader<>(new File(new File(System.getProperty("user.dir")).toPath().resolve("target/testOutputOneFile/avrotable.avro").toString()), datumReader); + List rows = new ArrayList<>(); + while (dataFileReader.hasNext()) { + rows.add(dataFileReader.next()); + } + assertEquals("Column1 for row 1 should be correct", 1L, rows.get(0).get("Column1")); + assertEquals("Column2 for row 1 should be correct", inputs.get(1L), rows.get(0).get("Column2").toString()); + assertEquals("Column1 for row 2 should be correct", 2L, rows.get(1).get("Column1")); + assertEquals("Column2 for row 2 should be correct", inputs.get(2L), rows.get(1).get("Column2").toString()); + } + + + @Test + public void testOutputOneFileAsArchive() throws IOException { + // Given + Path pathForThisTest = root.resolve("testOutputOneFileAsArchive"); + AvroDataSetConsumer sut = new AvroDataSetConsumer(pathForThisTest); + LinkedHashMap inputs = new LinkedHashMap<>(); + inputs.put(1L, "Bran"); + inputs.put(2L, "Brannan"); + DataSetProducer producer = new StubDataSetProducer(inputs); + + // When + new DataSetConnector(producer, sut).connect(); + + // Then - read content from in memory file system. + DatumReader datumReader = new GenericDatumReader<>(); + Files.createDirectories(new File(System.getProperty("user.dir")).toPath().resolve("target/testOutputOneFileAsArchive")); + // copy out of the in memory file system as avro doesn't support the java.nio paths api :( + Path outputPath = new File(System.getProperty("user.dir")).toPath().resolve("target/testOutputOneFileAsArchive"); + Files.copy(pathForThisTest, Files.newOutputStream(outputPath.resolve("output.zip"))); + + ZipFile zipFile = new ZipFile(outputPath.resolve("output.zip").toFile()); + ZipEntry zipEntry = zipFile.getEntry("avrotable.avro"); + zipFile.getInputStream(zipEntry).transferTo(Files.newOutputStream(outputPath.resolve("extractedFromZip.avro"), CREATE_NEW)); + DataFileReader dataFileReader = new DataFileReader<>(new File(new File(System.getProperty("user.dir")).toPath().resolve("target/testOutputOneFileAsArchive/extractedFromZip.avro").toString()), datumReader); + List rows = new ArrayList<>(); + while (dataFileReader.hasNext()) { + rows.add(dataFileReader.next()); + } + assertEquals("Column1 for row 1 should be correct", 1L, rows.get(0).get("Column1")); + assertEquals("Column2 for row 1 should be correct", inputs.get(1L), rows.get(0).get("Column2").toString()); + assertEquals("Column1 for row 2 should be correct", 2L, rows.get(1).get("Column1")); + assertEquals("Column2 for row 2 should be correct", inputs.get(2L), rows.get(1).get("Column2").toString()); + } + + private static class StubDataSetProducer implements DataSetProducer { + + private final Map inputRecords; + + StubDataSetProducer(Map inputRecords) { + this.inputRecords = inputRecords; + } + + @Override + public void open() { + // no-op + } + + @Override + public void close() { + // no-op + } + + @Override + public Schema getSchema() { + return new Schema() { + @Override + public boolean isEmptyDatabase() { + return false; + } + + @Override + public boolean tableExists(String name) { + return name.equalsIgnoreCase("avrotable"); + } + + @Override + public Table getTable(String name) { + if (!name.equalsIgnoreCase("avrotable")) { + throw new IllegalStateException("Asked for a table which doesn't exist: [" + name + "]"); + } + return new AvroTable(); + } + + @Override + public Collection tableNames() { + return Lists.newArrayList("avrotable"); + } + + @Override + public Collection
tables() { + return Lists.newArrayList(new AvroTable()); + } + + @Override + public boolean viewExists(String name) { + return false; + } + + @Override + public View getView(String name) { + throw new IllegalStateException("Asked for a view which doesn't exist: [" + name + "]"); + } + + @Override + public Collection viewNames() { + return new ArrayList<>(); + } + + @Override + public Collection views() { + return new ArrayList<>(); + } + }; + } + + @Override + public Iterable records(String tableName) { + if (!tableName.equalsIgnoreCase("avrotable")) { + throw new RuntimeException("Asked for record of table which doesn't exist: [" + tableName + "]"); + } + return inputRecords.entrySet().stream() + .map(entry -> DataSetUtils.record().setLong("Column1", entry.getKey()) + .setString("Column2", entry.getValue())) + .collect(Collectors.toList()); + } + + @Override + public boolean isTableEmpty(String tableName) { + if (!tableName.equalsIgnoreCase("avrotable")) { + throw new RuntimeException("Asked for record of table which doesn't exist: [" + tableName + "]"); + } + return false; + } + } + + private static class StubColumn implements Column { + + private final String name; + private final boolean primaryKey; + private final String defaultValue; + private final boolean autoNumbered; + private final int autoNumberStart; + private final DataType dataType; + private final int width; + private final int scale; + private final boolean nullable; + + + StubColumn(String name, boolean primaryKey, String defaultValue, boolean autoNumbered, int autoNumberStart, DataType dataType, int width, int scale, boolean nullable) { + + this.name = name; + this.primaryKey = primaryKey; + this.defaultValue = defaultValue; + this.autoNumbered = autoNumbered; + this.autoNumberStart = autoNumberStart; + this.dataType = dataType; + this.width = width; + this.scale = scale; + this.nullable = nullable; + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isPrimaryKey() { + return primaryKey; + } + + @Override + public String getDefaultValue() { + return defaultValue; + } + + @Override + public boolean isAutoNumbered() { + return autoNumbered; + } + + @Override + public int getAutoNumberStart() { + return autoNumberStart; + } + + @Override + public DataType getType() { + return dataType; + } + + @Override + public int getWidth() { + return width; + } + + @Override + public int getScale() { + return scale; + } + + @Override + public boolean isNullable() { + return nullable; + } + } + + private static class AvroTable implements Table { + @Override + public String getName() { + return "avrotable"; + } + + @Override + public List columns() { + return Lists.newArrayList( + new StubColumn("Column1", true, "1", true, 1, DataType.BIG_INTEGER, 12, 0, false), + new StubColumn("Column2", false, "Hello", false, 1, DataType.STRING, 20, 0, false) + ); + } + + @Override + public List indexes() { + return new ArrayList<>(); + } + + @Override + public boolean isTemporary() { + return false; + } + } +} diff --git a/morf-core/pom.xml b/morf-core/pom.xml index 5e8a4ba05..c9bcdf7a4 100755 --- a/morf-core/pom.xml +++ b/morf-core/pom.xml @@ -73,10 +73,6 @@ joda-time joda-time - - xalan - serializer - org.hamcrest hamcrest-all diff --git a/morf-directory-spi/pom.xml b/morf-directory-spi/pom.xml new file mode 100644 index 000000000..9dd7c3793 --- /dev/null +++ b/morf-directory-spi/pom.xml @@ -0,0 +1,22 @@ + + + + morf-parent + org.alfasoftware + 2.0.5-SNAPSHOT + + 4.0.0 + + morf-directory-spi + Morf - Directory SPI + + + + org.alfasoftware + morf-core + + + + \ No newline at end of file diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/ArchiveDataSetReader.java b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/ArchiveDataSetReader.java similarity index 88% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/ArchiveDataSetReader.java rename to morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/ArchiveDataSetReader.java index 4338771c7..533bb043e 100755 --- a/morf-core/src/main/java/org/alfasoftware/morf/xml/ArchiveDataSetReader.java +++ b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/ArchiveDataSetReader.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package org.alfasoftware.morf.xml; +package org.alfasoftware.morf.directory; import java.io.File; import java.io.IOException; @@ -25,35 +25,36 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlInputStreamProvider; /** * Allows reading of data sets based on an archive (zip) file. * * @author Copyright (c) Alfa Financial Software 2010 */ -class ArchiveDataSetReader extends BaseDataSetReader implements XmlInputStreamProvider { +public class ArchiveDataSetReader extends BaseDataSetReader implements DirectoryStreamProvider.DirectoryInputStreamProvider { /** * The file to read the archive from. */ private final File file; + /** * References the zip archive. */ private ZipFile zipFile; - private final Pattern filenamePattern = Pattern.compile("(\\w+)\\.xml"); + private final Pattern filenamePattern; /** * Creates an archive data set linked to the specified file. * * @param file The archive file to use. */ - public ArchiveDataSetReader(File file) { + public ArchiveDataSetReader(String suffix, File file) { super(); this.file = file; + this.filenamePattern = Pattern.compile("(\\w+)\\." + suffix); } diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/ArchiveDataSetWriter.java b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/ArchiveDataSetWriter.java similarity index 74% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/ArchiveDataSetWriter.java rename to morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/ArchiveDataSetWriter.java index 70981e668..a1a8ed77f 100755 --- a/morf-core/src/main/java/org/alfasoftware/morf/xml/ArchiveDataSetWriter.java +++ b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/ArchiveDataSetWriter.java @@ -13,26 +13,24 @@ * limitations under the License. */ -package org.alfasoftware.morf.xml; +package org.alfasoftware.morf.directory; + +import com.google.common.io.ByteStreams; import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider; - -import com.google.common.io.ByteStreams; - /** * Allows reading of data sets based on an archive (zip) file. * * @author Copyright (c) Alfa Financial Software 2010 */ -class ArchiveDataSetWriter implements XmlOutputStreamProvider { +class ArchiveDataSetWriter implements DirectoryStreamProvider.DirectoryOutputStreamProvider { /** * A read me entry to be included in all created zip files. @@ -42,7 +40,9 @@ class ArchiveDataSetWriter implements XmlOutputStreamProvider { /** * Identifies the archive to access. */ - private final File file; + private final Path path; + + private final String suffix; /** * References the zip archive. @@ -50,18 +50,19 @@ class ArchiveDataSetWriter implements XmlOutputStreamProvider { private AdaptedZipOutputStream zipOutput; /** - * Creates an archive data set linked to the specified file. + * Creates an archive data set linked to the specified path. * - * @param file The archive file to use. + * @param path The archive file to use. */ - public ArchiveDataSetWriter(File file) { + ArchiveDataSetWriter(String suffix, Path path) { super(); - this.file = file; + this.suffix = suffix; + this.path = path; } /** - * @see org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider#clearDestination() + * @see org.alfasoftware.morf.directory.DirectoryStreamProvider.DirectoryOutputStreamProvider#clearDestination() */ @Override public void clearDestination() { @@ -70,23 +71,23 @@ public void clearDestination() { /** - * @see org.alfasoftware.morf.xml.XmlStreamProvider#open() + * @see org.alfasoftware.morf.directory.DirectoryStreamProvider#open() */ @Override public void open() { if (zipOutput != null) { - throw new IllegalStateException("Archive data set instance for [" + file + "] already open"); + throw new IllegalStateException("Archive data set instance for [" + path + "] already open"); } try { - zipOutput = new AdaptedZipOutputStream(new FileOutputStream(file)); + zipOutput = new AdaptedZipOutputStream(Files.newOutputStream(path)); // Put the read me entry in ZipEntry entry = new ZipEntry("_ReadMe.txt"); zipOutput.putNextEntry(entry); ByteStreams.copy(new ByteArrayInputStream(READ_ME.getBytes("UTF-8")), zipOutput); } catch (Exception e) { - throw new RuntimeException("Error opening zip archive [" + file + "]", e); + throw new RuntimeException("Error opening zip archive [" + path + "]", e); } } @@ -103,13 +104,13 @@ public void close() { try { zipOutput.reallyClose(); } catch (IOException e) { - throw new RuntimeException("Error closing zip archive [" + file + "]", e); + throw new RuntimeException("Error closing zip archive [" + path + "]", e); } } /** - * @see org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider#openOutputStreamForTable(java.lang.String) + * @see org.alfasoftware.morf.directory.DirectoryStreamProvider.DirectoryOutputStreamProvider#openOutputStreamForTable(java.lang.String) */ @Override public OutputStream openOutputStreamForTable(String tableName) { @@ -118,13 +119,13 @@ public OutputStream openOutputStreamForTable(String tableName) { } try { - ZipEntry entry = new ZipEntry(tableName + ".xml"); + ZipEntry entry = new ZipEntry(tableName + "." + suffix); zipOutput.putNextEntry(entry); // Make sure the caller can't actually close the underlying stream return zipOutput; } catch (IOException e) { - throw new RuntimeException("Error creating new zip entry in archive [" + file + "]", e); + throw new RuntimeException("Error creating new zip entry in archive [" + path + "]", e); } } diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/BaseDataSetReader.java b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/BaseDataSetReader.java similarity index 87% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/BaseDataSetReader.java rename to morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/BaseDataSetReader.java index 83af12502..2246eac2b 100755 --- a/morf-core/src/main/java/org/alfasoftware/morf/xml/BaseDataSetReader.java +++ b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/BaseDataSetReader.java @@ -13,13 +13,12 @@ * limitations under the License. */ -package org.alfasoftware.morf.xml; +package org.alfasoftware.morf.directory; import java.util.Collection; import java.util.List; import java.util.Map; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlInputStreamProvider; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -28,7 +27,7 @@ * * @author Copyright (c) Alfa Financial Software 2012 */ -abstract class BaseDataSetReader implements XmlInputStreamProvider { +public abstract class BaseDataSetReader implements DirectoryStreamProvider.DirectoryInputStreamProvider { /** * Maps table names to upper case @@ -71,7 +70,7 @@ protected final String fileNameForTable(String tableName) { /** - * @see org.alfasoftware.morf.xml.XmlStreamProvider.XmlInputStreamProvider#availableStreamNames() + * @see org.alfasoftware.morf.directory.DirectoryStreamProvider.DirectoryInputStreamProvider#availableStreamNames() */ @Override public Collection availableStreamNames() { diff --git a/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DirectoryDataSet.java b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DirectoryDataSet.java new file mode 100755 index 000000000..7e6e2b85d --- /dev/null +++ b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DirectoryDataSet.java @@ -0,0 +1,129 @@ +/* Copyright 2017 Alfa Financial Software + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.alfasoftware.morf.directory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Allows reading of a data set from a directory. + * + * @author Copyright (c) Alfa Financial Software 2010 + */ +public class DirectoryDataSet extends BaseDataSetReader implements DirectoryStreamProvider.DirectoryInputStreamProvider, DirectoryStreamProvider.DirectoryOutputStreamProvider { + + /** + * The directory to read from. + */ + private final Path directory; + private final String suffix; + + /** + * Creates a directory based data set reader based on the directory file. + * + * @param directory The directory to access. + */ + public DirectoryDataSet(String suffix, Path directory) { + super(); + this.suffix = suffix; + this.directory = directory; + + if (!Files.isDirectory(directory)) { + throw new IllegalArgumentException("[" + directory + "] is not a directory"); + } + + // read the files in the directory + // Do it here because DirectoryDataSet historically did not have to be "open" to be used. + try { + Files.list(directory).filter(path -> path.endsWith(suffix)) + .forEach(path -> addTableName( + path.getFileName().toString().replaceAll("\\." + suffix, ""), + path.getFileName().toString())); + } catch (IOException e) { + e.printStackTrace(); + } + } + + + /** + * @see DirectoryOutputStreamProvider#clearDestination() + */ + @Override + public void clearDestination() { + try { + Files.list(directory) + .filter(path -> !path.getFileName().startsWith(".")) + .forEach(this::deleteFileOrDirectory); + // skip files/folders that start with . such as .svn + } catch (IOException e) { + throw new RuntimeException("Exception occurred when trying to clear existing directory", e); + } + } + + + private void deleteFileOrDirectory(Path path) { + if (Files.isDirectory(path)) { + // clear it out (recursively) if needed... + if (!Files.isSymbolicLink(path)) { + try { + Files.list(path).forEach(this::deleteFileOrDirectory); + } catch (IOException ex) { + throw new RuntimeException("Exception occurred when trying to clear existing directory", ex); + } + } + } + try { + Files.delete(path); + } catch (IOException e) { + throw new RuntimeException("Exception cleaning output directory, file [" + path + "]"); + } + + } + + + /** + * @see DirectoryInputStreamProvider#openInputStreamForTable(String) + */ + @Override + public InputStream openInputStreamForTable(String tableName) { + try { + return Files.newInputStream(directory.resolve(fileNameForTable(tableName))); + } catch (FileNotFoundException e) { + throw new RuntimeException("Error opening output stream", e); + } catch (IOException e) { + throw new RuntimeException("Error opening output stream", e); + } + } + + + /** + * @see DirectoryOutputStreamProvider#openOutputStreamForTable(String) + */ + @Override + public OutputStream openOutputStreamForTable(String tableName) { + try { + return Files.newOutputStream(directory.resolve(tableName + "." + suffix)); + } catch (FileNotFoundException e) { + throw new RuntimeException("Error opening output stream", e); + } catch (IOException e) { + throw new RuntimeException("Error opening output stream", e); + } + } +} diff --git a/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DirectoryDataSetConsumer.java b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DirectoryDataSetConsumer.java new file mode 100644 index 000000000..8d36fa9ea --- /dev/null +++ b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DirectoryDataSetConsumer.java @@ -0,0 +1,113 @@ +package org.alfasoftware.morf.directory; + +import org.alfasoftware.morf.dataset.DataSetConsumer; + +import java.nio.file.Files; +import java.nio.file.Path; + +public abstract class DirectoryDataSetConsumer implements DataSetConsumer { + + /** + * Source of content handlers to send data to. + */ + protected final DirectoryStreamProvider.DirectoryOutputStreamProvider directoryOutputStreamProvider; + + /** + * What to do about clearing the destination. + */ + protected final ClearDestinationBehaviour clearDestinationBehaviour; + + protected final String suffix; + + /** + * Creates a data set consumer that will pipe the data set to the file system location + * specified by file. + * + *

The serialised output can be written to a single archive or multiple data files:

+ *
    + *
  • If file identifies a directory then each table in the data set is + * serialised to a separate file within that directory.
    • + *
  • If file identifies a file name then the file will be created or replaced with + * a zip archive containing one file per table in the data set.
    • + *
+ * + * @param path to file system location to receive the data set. + * @param clearDestinationBehaviour Whether to clear the destination directory or not. + */ + public DirectoryDataSetConsumer(String suffix, + Path path, + DirectoryDataSetConsumer.ClearDestinationBehaviour clearDestinationBehaviour) { + super(); + this.suffix = suffix; + if (Files.isDirectory(path)) { + this.directoryOutputStreamProvider = new DirectoryDataSet(suffix, path); + } else { + this.directoryOutputStreamProvider = new ArchiveDataSetWriter(suffix, path); + } + this.clearDestinationBehaviour = clearDestinationBehaviour; + } + + /** + * Creates a data set consumer that will pipe the data set to the file system location + * specified by file. + * + *

The serialised output can be written to a single archive or multiple data files:

+ *
    + *
  • If file identifies a directory then each table in the data set is + * serialised to a separate XML file within that directory.
    • + *
  • If file identifies a file name then the file will be created or replaced with + * a zip archive containing one XML file per table in the data set.
    • + *
+ * + * @param path to file system location to receive the data set. + */ + public DirectoryDataSetConsumer(String suffix, Path path) { + this(suffix, path, DirectoryDataSetConsumer.ClearDestinationBehaviour.CLEAR); + } + + public DirectoryDataSetConsumer(String suffix, DirectoryStreamProvider.DirectoryOutputStreamProvider directoryOutputStreamProvider, ClearDestinationBehaviour clearDestinationBehaviour) { + this.suffix = suffix; + this.directoryOutputStreamProvider = directoryOutputStreamProvider; + this.clearDestinationBehaviour = clearDestinationBehaviour; + } + + + /** + * @see org.alfasoftware.morf.dataset.DataSetConsumer#open() + */ + @Override + public void open() { + directoryOutputStreamProvider.open(); + + if (clearDestinationBehaviour.equals(DirectoryDataSetConsumer.ClearDestinationBehaviour.CLEAR)) { + // we're outputting, so clear the destination of any previous runs + directoryOutputStreamProvider.clearDestination(); + } + } + + /** + * Fired when a dataset has ended. + * + * @see org.alfasoftware.morf.dataset.DataSetConsumer#close(org.alfasoftware.morf.dataset.DataSetConsumer.CloseState) + */ + @Override + public void close(CloseState closeState) { + directoryOutputStreamProvider.close(); + } + + + /** + * Controls the behaviour of the consumer when running against a directory. + */ + public enum ClearDestinationBehaviour { + /** + * Clear the destination out before extracting (the default) + */ + CLEAR, + + /** + * Overwrite the destination + */ + OVERWRITE + } +} diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/XmlStreamProvider.java b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DirectoryStreamProvider.java similarity index 88% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/XmlStreamProvider.java rename to morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DirectoryStreamProvider.java index 01977e439..dd4f52dd8 100755 --- a/morf-core/src/main/java/org/alfasoftware/morf/xml/XmlStreamProvider.java +++ b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DirectoryStreamProvider.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package org.alfasoftware.morf.xml; +package org.alfasoftware.morf.directory; import java.io.InputStream; import java.io.OutputStream; @@ -24,7 +24,7 @@ * * @author Copyright (c) Alfa Financial Software 2010 */ -interface XmlStreamProvider { +public interface DirectoryStreamProvider { /** * Opens or creates any resources required to provide content handlers. @@ -41,7 +41,7 @@ interface XmlStreamProvider { * * @author Copyright (c) Alfa Financial Software 2010 */ - interface XmlInputStreamProvider extends XmlStreamProvider { + interface DirectoryInputStreamProvider extends DirectoryStreamProvider { /** * Provides an input stream to read XML for a specific table. @@ -71,7 +71,7 @@ interface XmlInputStreamProvider extends XmlStreamProvider { * * @author Copyright (c) Alfa Financial Software 2010 */ - interface XmlOutputStreamProvider extends XmlStreamProvider { + interface DirectoryOutputStreamProvider extends DirectoryStreamProvider { /** * Provides an output stream to write XML for a specific table. diff --git a/morf-core/src/test/java/org/alfasoftware/morf/xml/DummyXmlOutputStreamProvider.java b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DummyXmlOutputStreamProvider.java old mode 100644 new mode 100755 similarity index 71% rename from morf-core/src/test/java/org/alfasoftware/morf/xml/DummyXmlOutputStreamProvider.java rename to morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DummyXmlOutputStreamProvider.java index 79a1b2f79..528a7e2f3 --- a/morf-core/src/test/java/org/alfasoftware/morf/xml/DummyXmlOutputStreamProvider.java +++ b/morf-directory-spi/src/main/java/org/alfasoftware/morf/directory/DummyXmlOutputStreamProvider.java @@ -13,19 +13,17 @@ * limitations under the License. */ -package org.alfasoftware.morf.xml; +package org.alfasoftware.morf.directory; import java.io.ByteArrayOutputStream; import java.io.OutputStream; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider; - /** * Testing implementation to catch result XML. * * @author Copyright (c) Alfa Financial Software 2010 */ -public final class DummyXmlOutputStreamProvider implements XmlOutputStreamProvider { +public final class DummyXmlOutputStreamProvider implements DirectoryStreamProvider.DirectoryOutputStreamProvider { /** * Holds the output stream for test data. @@ -38,7 +36,7 @@ public final class DummyXmlOutputStreamProvider implements XmlOutputStreamProvid private boolean cleared; /** - * @see org.alfasoftware.morf.xml.XmlStreamProvider#close() + * @see org.alfasoftware.morf.directory.DirectoryStreamProvider#close() */ @Override public void close() { @@ -47,7 +45,7 @@ public void close() { /** - * @see org.alfasoftware.morf.xml.XmlStreamProvider#open() + * @see org.alfasoftware.morf.directory.DirectoryStreamProvider#open() */ @Override public void open() { @@ -56,7 +54,7 @@ public void open() { /** - * @see org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider#openOutputStreamForTable(java.lang.String) + * @see org.alfasoftware.morf.directory.DirectoryStreamProvider.DirectoryOutputStreamProvider#openOutputStreamForTable(java.lang.String) */ @Override public OutputStream openOutputStreamForTable(String tableName) { @@ -65,7 +63,7 @@ public OutputStream openOutputStreamForTable(String tableName) { /** - * @see org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider#clearDestination() + * @see org.alfasoftware.morf.directory.DirectoryStreamProvider.DirectoryOutputStreamProvider#clearDestination() */ @Override public void clearDestination() { diff --git a/morf-excel/pom.xml b/morf-excel/pom.xml index 356f0a4bd..eaefd4814 100755 --- a/morf-excel/pom.xml +++ b/morf-excel/pom.xml @@ -37,7 +37,7 @@ org.alfasoftware - morf-core + morf-directory-spi net.sourceforge.jexcelapi diff --git a/morf-integration-test/pom.xml b/morf-integration-test/pom.xml index 497c8505d..cb85fd5f8 100755 --- a/morf-integration-test/pom.xml +++ b/morf-integration-test/pom.xml @@ -101,6 +101,10 @@ morf-testsupport test + + org.alfasoftware + morf-xml + pl.pragmatists JUnitParams diff --git a/morf-xml/pom.xml b/morf-xml/pom.xml new file mode 100644 index 000000000..0793446f8 --- /dev/null +++ b/morf-xml/pom.xml @@ -0,0 +1,49 @@ + + + + morf-parent + org.alfasoftware + 2.0.5-SNAPSHOT + + 4.0.0 + + morf-xml + Morf - XML + + + 2.7.1 + + + + + + xalan + serializer + ${serializer.version} + + + + + + org.alfasoftware + morf-directory-spi + + + xalan + serializer + + + junit + junit + test + + + org.mockito + mockito-core + test + + + + \ No newline at end of file diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/DataMaskingXmlDataSetConsumer.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/DataMaskingXmlDataSetConsumer.java similarity index 90% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/DataMaskingXmlDataSetConsumer.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/DataMaskingXmlDataSetConsumer.java index 87606651d..61f330889 100755 --- a/morf-core/src/main/java/org/alfasoftware/morf/xml/DataMaskingXmlDataSetConsumer.java +++ b/morf-xml/src/main/java/org/alfasoftware/morf/xml/DataMaskingXmlDataSetConsumer.java @@ -21,8 +21,8 @@ import org.alfasoftware.morf.dataset.DataSetConsumer; import org.alfasoftware.morf.dataset.Record; +import org.alfasoftware.morf.directory.DirectoryStreamProvider; import org.alfasoftware.morf.metadata.Column; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider; /** * Implementation of {@linkplain DataSetConsumer} which discards specific fields when consuming @@ -63,7 +63,7 @@ public DataMaskingXmlDataSetConsumer(File file, Map> tableCo * for the data in the data set. * @param tableColumnsToMask the table columns to mask when consuming a dataset */ - public DataMaskingXmlDataSetConsumer(XmlOutputStreamProvider xmlOutputStreamProvider, Map> tableColumnsToMask) { + public DataMaskingXmlDataSetConsumer(DirectoryStreamProvider.DirectoryOutputStreamProvider xmlOutputStreamProvider, Map> tableColumnsToMask) { super(xmlOutputStreamProvider); this.tableColumnsToMask = tableColumnsToMask; } diff --git a/morf-testsupport/src/main/java/org/alfasoftware/morf/diagnostics/DatabaseDumper.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/DatabaseDumper.java similarity index 95% rename from morf-testsupport/src/main/java/org/alfasoftware/morf/diagnostics/DatabaseDumper.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/DatabaseDumper.java index 1821bf929..6a9c612bf 100755 --- a/morf-testsupport/src/main/java/org/alfasoftware/morf/diagnostics/DatabaseDumper.java +++ b/morf-xml/src/main/java/org/alfasoftware/morf/xml/DatabaseDumper.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package org.alfasoftware.morf.diagnostics; +package org.alfasoftware.morf.xml; import java.io.File; import java.io.IOException; @@ -25,7 +25,6 @@ import org.alfasoftware.morf.dataset.DataSetProducer; import org.alfasoftware.morf.jdbc.ConnectionResources; import org.alfasoftware.morf.jdbc.DatabaseDataSetProducer; -import org.alfasoftware.morf.xml.XmlDataSetConsumer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/DirectoryDataSet.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/DirectoryDataSet.java similarity index 86% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/DirectoryDataSet.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/DirectoryDataSet.java index ffe3a2588..558ba3623 100755 --- a/morf-core/src/main/java/org/alfasoftware/morf/xml/DirectoryDataSet.java +++ b/morf-xml/src/main/java/org/alfasoftware/morf/xml/DirectoryDataSet.java @@ -24,15 +24,19 @@ import java.nio.file.Files; import java.util.Arrays; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlInputStreamProvider; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider; +import org.alfasoftware.morf.directory.BaseDataSetReader; +import org.alfasoftware.morf.directory.DirectoryStreamProvider; /** * Allows reading of a data set from a directory. * + * Deprecated - use org.alfasoftware.morf.directory.DirectoryDataSet + * * @author Copyright (c) Alfa Financial Software 2010 */ -class DirectoryDataSet extends BaseDataSetReader implements XmlInputStreamProvider, XmlOutputStreamProvider { +@Deprecated +class DirectoryDataSet extends BaseDataSetReader implements DirectoryStreamProvider.DirectoryInputStreamProvider, + DirectoryStreamProvider.DirectoryOutputStreamProvider { /** * The directory to read from. diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/Escaping.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/Escaping.java similarity index 100% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/Escaping.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/Escaping.java diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/Version2to4TransformingReader.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/Version2to4TransformingReader.java similarity index 100% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/Version2to4TransformingReader.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/Version2to4TransformingReader.java diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/ViewURLAsFile.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/ViewURLAsFile.java similarity index 100% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/ViewURLAsFile.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/ViewURLAsFile.java diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/XmlDataSetConsumer.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlDataSetConsumer.java similarity index 84% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/XmlDataSetConsumer.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlDataSetConsumer.java index df594cccb..292a6a6cf 100755 --- a/morf-core/src/main/java/org/alfasoftware/morf/xml/XmlDataSetConsumer.java +++ b/morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlDataSetConsumer.java @@ -15,22 +15,14 @@ package org.alfasoftware.morf.xml; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Properties; - import org.alfasoftware.morf.dataset.DataSetConsumer; import org.alfasoftware.morf.dataset.Record; +import org.alfasoftware.morf.directory.DirectoryDataSetConsumer; +import org.alfasoftware.morf.directory.DirectoryStreamProvider; import org.alfasoftware.morf.metadata.Column; import org.alfasoftware.morf.metadata.DataType; import org.alfasoftware.morf.metadata.Index; import org.alfasoftware.morf.metadata.Table; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider; import org.apache.commons.lang3.StringUtils; import org.apache.xml.serializer.Method; import org.apache.xml.serializer.OutputPropertiesFactory; @@ -41,18 +33,27 @@ import org.xml.sax.SAXException; import org.xml.sax.helpers.AttributesImpl; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Properties; + /** * Serialises data sets to XML. * *

The output from this class can be sent directly to the file system by using the * constructor {@link #XmlDataSetConsumer(File)}. Alternatively the output from this * class can be sent to one or many streams by calling the constructor - * accepting the {@link XmlOutputStreamProvider}. + * accepting the {@link DirectoryStreamProvider.DirectoryOutputStreamProvider}. *

* * @author Copyright (c) Alfa Financial Software 2009 */ -public class XmlDataSetConsumer implements DataSetConsumer { +public class XmlDataSetConsumer extends DirectoryDataSetConsumer implements DataSetConsumer { /** * Attributes implementation for use when no attributes are present on a node. @@ -66,7 +67,10 @@ public class XmlDataSetConsumer implements DataSetConsumer { /** * Controls the behaviour of the consumer when running against a directory. + * + * Deprecated - use FileSystemDataSetConsumer#ClearDestinationBehaviour instead. Will be removed in 2.0.0. */ + @Deprecated public enum ClearDestinationBehaviour { /** * Clear the destination out before extracting (the default) @@ -79,16 +83,6 @@ public enum ClearDestinationBehaviour { OVERWRITE } - /** - * Source of content handlers to send data to. - */ - private final XmlOutputStreamProvider xmlStreamProvider; - - /** - * What to do about clearing the destination. - */ - private final ClearDestinationBehaviour clearDestinationBehaviour; - /** * Creates a data set consumer that will pipe the data set to the file system location * specified by file. @@ -104,7 +98,7 @@ public enum ClearDestinationBehaviour { * @param file The file system location to receive the data set. */ public XmlDataSetConsumer(File file) { - this(file, ClearDestinationBehaviour.CLEAR); + this(file, DirectoryDataSetConsumer.ClearDestinationBehaviour.CLEAR); } @@ -123,14 +117,8 @@ public XmlDataSetConsumer(File file) { * @param file The file system location to receive the data set. * @param clearDestinationBehaviour Whether to clear the destination directory or not. */ - public XmlDataSetConsumer(File file, ClearDestinationBehaviour clearDestinationBehaviour) { - super(); - if (file.isDirectory()) { - this.xmlStreamProvider = new DirectoryDataSet(file); - } else { - this.xmlStreamProvider = new ArchiveDataSetWriter(file); - } - this.clearDestinationBehaviour = clearDestinationBehaviour; + public XmlDataSetConsumer(File file, DirectoryDataSetConsumer.ClearDestinationBehaviour clearDestinationBehaviour) { + super("xml", file.toPath(), clearDestinationBehaviour); } @@ -141,8 +129,8 @@ public XmlDataSetConsumer(File file, ClearDestinationBehaviour clearDestinationB * @param xmlOutputStreamProvider Provides streams to receive the XML content * for the data in the data set. */ - public XmlDataSetConsumer(XmlOutputStreamProvider xmlOutputStreamProvider) { - this(xmlOutputStreamProvider, ClearDestinationBehaviour.CLEAR); + public XmlDataSetConsumer(DirectoryStreamProvider.DirectoryOutputStreamProvider xmlOutputStreamProvider) { + this(xmlOutputStreamProvider, DirectoryDataSetConsumer.ClearDestinationBehaviour.CLEAR); } @@ -154,35 +142,8 @@ public XmlDataSetConsumer(XmlOutputStreamProvider xmlOutputStreamProvider) { * for the data in the data set. * @param clearDestinationBehaviour The behaviour of the consumer when running against a directory. */ - public XmlDataSetConsumer(XmlOutputStreamProvider xmlOutputStreamProvider, ClearDestinationBehaviour clearDestinationBehaviour) { - super(); - this.xmlStreamProvider = xmlOutputStreamProvider; - this.clearDestinationBehaviour = clearDestinationBehaviour; - } - - - /** - * @see org.alfasoftware.morf.dataset.DataSetConsumer#open() - */ - @Override - public void open() { - xmlStreamProvider.open(); - - if (clearDestinationBehaviour.equals(ClearDestinationBehaviour.CLEAR)) { - // we're outputting, so clear the destination of any previous runs - xmlStreamProvider.clearDestination(); - } - } - - - /** - * Fired when a dataset has ended. - * - * @see org.alfasoftware.morf.dataset.DataSetConsumer#close(org.alfasoftware.morf.dataset.DataSetConsumer.CloseState) - */ - @Override - public void close(CloseState closeState) { - xmlStreamProvider.close(); + public XmlDataSetConsumer(DirectoryStreamProvider.DirectoryOutputStreamProvider xmlOutputStreamProvider, DirectoryDataSetConsumer.ClearDestinationBehaviour clearDestinationBehaviour) { + super(".xml", xmlOutputStreamProvider, clearDestinationBehaviour); } @@ -197,7 +158,7 @@ public void table(final Table table, final Iterable records) { // Get a content handler for this table try { - OutputStream outputStream = xmlStreamProvider.openOutputStreamForTable(table.getName()); + OutputStream outputStream = directoryOutputStreamProvider.openOutputStreamForTable(table.getName()); try { ContentHandler contentHandler = createContentHandler(outputStream); diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/XmlDataSetNode.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlDataSetNode.java similarity index 100% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/XmlDataSetNode.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlDataSetNode.java diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/XmlDataSetProducer.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlDataSetProducer.java similarity index 92% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/XmlDataSetProducer.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlDataSetProducer.java index 15a7cda03..164775b87 100755 --- a/morf-core/src/main/java/org/alfasoftware/morf/xml/XmlDataSetProducer.java +++ b/morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlDataSetProducer.java @@ -15,6 +15,29 @@ package org.alfasoftware.morf.xml; +import com.google.common.base.Charsets; +import com.google.common.io.Closeables; +import org.alfasoftware.morf.dataset.DataSetProducer; +import org.alfasoftware.morf.dataset.Record; +import org.alfasoftware.morf.directory.ArchiveDataSetReader; +import org.alfasoftware.morf.directory.DirectoryStreamProvider; +import org.alfasoftware.morf.metadata.Column; +import org.alfasoftware.morf.metadata.DataSetUtils; +import org.alfasoftware.morf.metadata.DataSetUtils.RecordBuilder; +import org.alfasoftware.morf.metadata.DataType; +import org.alfasoftware.morf.metadata.Index; +import org.alfasoftware.morf.metadata.Schema; +import org.alfasoftware.morf.metadata.SchemaUtils; +import org.alfasoftware.morf.metadata.Table; +import org.alfasoftware.morf.metadata.View; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import javax.xml.stream.FactoryConfigurationError; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -32,30 +55,6 @@ import java.util.NoSuchElementException; import java.util.Set; -import javax.xml.stream.FactoryConfigurationError; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; - -import org.alfasoftware.morf.dataset.DataSetProducer; -import org.alfasoftware.morf.dataset.Record; -import org.alfasoftware.morf.metadata.Column; -import org.alfasoftware.morf.metadata.DataSetUtils; -import org.alfasoftware.morf.metadata.DataSetUtils.RecordBuilder; -import org.alfasoftware.morf.metadata.DataType; -import org.alfasoftware.morf.metadata.Index; -import org.alfasoftware.morf.metadata.Schema; -import org.alfasoftware.morf.metadata.SchemaUtils; -import org.alfasoftware.morf.metadata.Table; -import org.alfasoftware.morf.metadata.View; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlInputStreamProvider; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import com.google.common.base.Charsets; -import com.google.common.io.Closeables; - /** * Reads XML and provides it as a data set. Uses XML pull-processing to get the * XML efficiently. @@ -69,7 +68,7 @@ public class XmlDataSetProducer implements DataSetProducer { /** * Source of streams from which to read XMl. */ - private final XmlInputStreamProvider xmlStreamProvider; + private final DirectoryStreamProvider.DirectoryInputStreamProvider xmlStreamProvider; /** @@ -114,7 +113,7 @@ public XmlDataSetProducer(URL source, String sourceUsername, String sourcePasswo this.xmlStreamProvider = new DirectoryDataSet(file); } else if (file.isFile()) { - this.xmlStreamProvider = new ArchiveDataSetReader(file); + this.xmlStreamProvider = new ArchiveDataSetReader("xml", file); } else { throw new RuntimeException("Could not find [" + file + "] from [" + source + "]"); @@ -127,7 +126,7 @@ else if (file.isFile()) { * * @param xmlStreamProvider Provides streams from which to read XML data. */ - public XmlDataSetProducer(XmlInputStreamProvider xmlStreamProvider) { + public XmlDataSetProducer(DirectoryStreamProvider.DirectoryInputStreamProvider xmlStreamProvider) { super(); this.xmlStreamProvider = xmlStreamProvider; this.urlHandler = new ViewURLAsFile(); @@ -243,7 +242,7 @@ private static XMLStreamReader openPullParser(InputStream inputStream) { /** - * Provides meta data based on an {@link XmlInputStreamProvider}. + * Provides meta data based on an {@link DirectoryStreamProvider.DirectoryInputStreamProvider}. * * @author Copyright (c) Alfa Financial Software 2010 */ @@ -252,13 +251,13 @@ private static final class PullProcessorMetaDataProvider implements Schema { /** * The source for data. */ - private final XmlInputStreamProvider xmlStreamProvider; + private final DirectoryStreamProvider.DirectoryInputStreamProvider xmlStreamProvider; /** * @param xmlStreamProvider The source stream provider. */ - public PullProcessorMetaDataProvider(XmlInputStreamProvider xmlStreamProvider) { + public PullProcessorMetaDataProvider(DirectoryStreamProvider.DirectoryInputStreamProvider xmlStreamProvider) { super(); this.xmlStreamProvider = xmlStreamProvider; } @@ -389,7 +388,7 @@ private static final class PullProcessorTableMetaData extends XmlPullProcessor i /** - * @param xmlPullParser pull parser that provides the xml data + * @param xmlStreamReader pull parser that provides the xml data * @param xmlFormatVersion The format version. */ public PullProcessorTableMetaData(XMLStreamReader xmlStreamReader, int xmlFormatVersion) { @@ -737,7 +736,7 @@ private static class PullProcessorRecordIterator extends XmlPullProcessor implem /** - * @param xmlPullParser Input stream containing the source XML data. + * @param xmlStreamReader Input stream containing the source XML data. */ public PullProcessorRecordIterator(XMLStreamReader xmlStreamReader) { super(xmlStreamReader); diff --git a/morf-core/src/main/java/org/alfasoftware/morf/xml/XmlPullProcessor.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlPullProcessor.java similarity index 100% rename from morf-core/src/main/java/org/alfasoftware/morf/xml/XmlPullProcessor.java rename to morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlPullProcessor.java diff --git a/morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlStreamProvider.java b/morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlStreamProvider.java new file mode 100755 index 000000000..a4c1bf184 --- /dev/null +++ b/morf-xml/src/main/java/org/alfasoftware/morf/xml/XmlStreamProvider.java @@ -0,0 +1,50 @@ +/* Copyright 2017 Alfa Financial Software + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.alfasoftware.morf.xml; + +import org.alfasoftware.morf.directory.DirectoryStreamProvider; + +/** + * Provides streams for accessing XML data sets. + * + * Deprecated - use DirectoryStreamProvider. This class will be removed from 2.0.0. + * + * @author Copyright (c) Alfa Financial Software 2010 + */ +@Deprecated +public interface XmlStreamProvider extends DirectoryStreamProvider { + + + + /** + * Provides input streams for reading data sets from XML. + * + * @author Copyright (c) Alfa Financial Software 2010 + */ + @Deprecated + interface XmlInputStreamProvider extends DirectoryStreamProvider.DirectoryInputStreamProvider { + } + + + /** + * Provides output streams for writing XML. + * + * @author Copyright (c) Alfa Financial Software 2010 + */ + @Deprecated + interface XmlOutputStreamProvider extends DirectoryStreamProvider.DirectoryOutputStreamProvider { + } +} diff --git a/morf-core/src/main/resources/org/alfasoftware/morf/xml/package.html b/morf-xml/src/main/resources/org.alfasoftware.morf.xml/package.html similarity index 100% rename from morf-core/src/main/resources/org/alfasoftware/morf/xml/package.html rename to morf-xml/src/main/resources/org.alfasoftware.morf.xml/package.html diff --git a/morf-testsupport/src/main/java/org/alfasoftware/morf/xml/DummyXmlOutputStreamProvider.java b/morf-xml/src/test/java/org/alfasoftware/morf/xml/DummyXmlOutputStreamProvider.java old mode 100755 new mode 100644 similarity index 88% rename from morf-testsupport/src/main/java/org/alfasoftware/morf/xml/DummyXmlOutputStreamProvider.java rename to morf-xml/src/test/java/org/alfasoftware/morf/xml/DummyXmlOutputStreamProvider.java index 79a1b2f79..8e08aa3d4 --- a/morf-testsupport/src/main/java/org/alfasoftware/morf/xml/DummyXmlOutputStreamProvider.java +++ b/morf-xml/src/test/java/org/alfasoftware/morf/xml/DummyXmlOutputStreamProvider.java @@ -18,14 +18,14 @@ import java.io.ByteArrayOutputStream; import java.io.OutputStream; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlOutputStreamProvider; +import org.alfasoftware.morf.directory.DirectoryStreamProvider; /** * Testing implementation to catch result XML. * * @author Copyright (c) Alfa Financial Software 2010 */ -public final class DummyXmlOutputStreamProvider implements XmlOutputStreamProvider { +public final class DummyXmlOutputStreamProvider implements DirectoryStreamProvider.DirectoryOutputStreamProvider { /** * Holds the output stream for test data. diff --git a/morf-core/src/test/java/org/alfasoftware/morf/xml/SourceXML.java b/morf-xml/src/test/java/org/alfasoftware/morf/xml/SourceXML.java similarity index 100% rename from morf-core/src/test/java/org/alfasoftware/morf/xml/SourceXML.java rename to morf-xml/src/test/java/org/alfasoftware/morf/xml/SourceXML.java diff --git a/morf-core/src/test/java/org/alfasoftware/morf/xml/TestDataMaskingXmlDataSetConsumer.java b/morf-xml/src/test/java/org/alfasoftware/morf/xml/TestDataMaskingXmlDataSetConsumer.java similarity index 73% rename from morf-core/src/test/java/org/alfasoftware/morf/xml/TestDataMaskingXmlDataSetConsumer.java rename to morf-xml/src/test/java/org/alfasoftware/morf/xml/TestDataMaskingXmlDataSetConsumer.java index 95a5a2cbf..c132a2d20 100755 --- a/morf-core/src/test/java/org/alfasoftware/morf/xml/TestDataMaskingXmlDataSetConsumer.java +++ b/morf-xml/src/test/java/org/alfasoftware/morf/xml/TestDataMaskingXmlDataSetConsumer.java @@ -15,27 +15,23 @@ package org.alfasoftware.morf.xml; -import static org.alfasoftware.morf.metadata.DataSetUtils.record; -import static org.alfasoftware.morf.metadata.SchemaUtils.column; -import static org.alfasoftware.morf.metadata.SchemaUtils.index; -import static org.alfasoftware.morf.metadata.SchemaUtils.table; -import static org.alfasoftware.morf.metadata.SchemaUtils.versionColumn; -import static org.junit.Assert.assertEquals; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.alfasoftware.morf.dataset.DataSetConsumer; import org.alfasoftware.morf.dataset.DataSetConsumer.CloseState; import org.alfasoftware.morf.dataset.Record; +import org.alfasoftware.morf.metadata.DataSetUtils; import org.alfasoftware.morf.metadata.DataType; +import org.alfasoftware.morf.metadata.SchemaUtils; import org.alfasoftware.morf.metadata.Table; import org.junit.Test; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; /** * Tests for {@linkplain DataMaskingXmlDataSetConsumer}. @@ -58,19 +54,19 @@ public void getValue() { DummyXmlOutputStreamProvider dummyXmlOutputStreamProvider = new DummyXmlOutputStreamProvider(); DataSetConsumer testConsumer = new DataMaskingXmlDataSetConsumer(dummyXmlOutputStreamProvider, toMask); - Table metaData = table("Test").columns( - column("id", DataType.BIG_INTEGER).primaryKey().autoNumbered(123), - versionColumn(), - column("bar", DataType.STRING, 10).nullable(), - column("baz", DataType.STRING, 10).nullable(), - column("bob", DataType.DECIMAL, 13, 2).nullable() + Table metaData = SchemaUtils.table("Test").columns( + SchemaUtils.column("id", DataType.BIG_INTEGER).primaryKey().autoNumbered(123), + SchemaUtils.versionColumn(), + SchemaUtils.column("bar", DataType.STRING, 10).nullable(), + SchemaUtils.column("baz", DataType.STRING, 10).nullable(), + SchemaUtils.column("bob", DataType.DECIMAL, 13, 2).nullable() ).indexes( - index("fizz").unique().columns("bar", "baz") + SchemaUtils.index("fizz").unique().columns("bar", "baz") ); testConsumer.open(); List mockRecords = new ArrayList<>(); - mockRecords.add(record() + mockRecords.add(DataSetUtils.record() .setInteger("id", 1) .setInteger("version", 1) .setString("bar", "abc") diff --git a/morf-core/src/test/java/org/alfasoftware/morf/xml/TestEscaping.java b/morf-xml/src/test/java/org/alfasoftware/morf/xml/TestEscaping.java similarity index 100% rename from morf-core/src/test/java/org/alfasoftware/morf/xml/TestEscaping.java rename to morf-xml/src/test/java/org/alfasoftware/morf/xml/TestEscaping.java diff --git a/morf-core/src/test/java/org/alfasoftware/morf/xml/TestVersion2to4TransformingReader.java b/morf-xml/src/test/java/org/alfasoftware/morf/xml/TestVersion2to4TransformingReader.java similarity index 100% rename from morf-core/src/test/java/org/alfasoftware/morf/xml/TestVersion2to4TransformingReader.java rename to morf-xml/src/test/java/org/alfasoftware/morf/xml/TestVersion2to4TransformingReader.java diff --git a/morf-core/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetConsumer.java b/morf-xml/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetConsumer.java similarity index 62% rename from morf-core/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetConsumer.java rename to morf-xml/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetConsumer.java index 8a7ece783..4348ec6a9 100755 --- a/morf-core/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetConsumer.java +++ b/morf-xml/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetConsumer.java @@ -15,29 +15,23 @@ package org.alfasoftware.morf.xml; -import static org.alfasoftware.morf.metadata.DataSetUtils.record; -import static org.alfasoftware.morf.metadata.SchemaUtils.column; -import static org.alfasoftware.morf.metadata.SchemaUtils.idColumn; -import static org.alfasoftware.morf.metadata.SchemaUtils.index; -import static org.alfasoftware.morf.metadata.SchemaUtils.table; -import static org.alfasoftware.morf.metadata.SchemaUtils.versionColumn; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - +import com.google.common.collect.ImmutableList; import org.alfasoftware.morf.dataset.DataSetConsumer; import org.alfasoftware.morf.dataset.DataSetConsumer.CloseState; import org.alfasoftware.morf.dataset.Record; +import org.alfasoftware.morf.directory.DirectoryDataSetConsumer; +import org.alfasoftware.morf.metadata.DataSetUtils; import org.alfasoftware.morf.metadata.DataType; +import org.alfasoftware.morf.metadata.SchemaUtils; import org.alfasoftware.morf.metadata.Table; -import org.alfasoftware.morf.xml.XmlDataSetConsumer.ClearDestinationBehaviour; +import org.junit.Assert; import org.junit.Test; -import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; /** * Test that we can convert data sets to xml. @@ -54,20 +48,21 @@ public void testBasicSerialisation() { DummyXmlOutputStreamProvider dummyXmlOutputStreamProvider = new DummyXmlOutputStreamProvider(); DataSetConsumer testConsumer = new XmlDataSetConsumer(dummyXmlOutputStreamProvider); - Table metaData = table("Test").columns( - column("id", DataType.BIG_INTEGER).primaryKey().autoNumbered(123), - versionColumn(), - column("bar", DataType.STRING, 10).nullable(), - column("baz", DataType.STRING, 10).nullable(), - column("bob", DataType.DECIMAL, 13, 2).nullable() - ).indexes( index("fizz").unique().columns("bar", "baz") + Table metaData = SchemaUtils.table("Test").columns( + SchemaUtils.column("id", DataType.BIG_INTEGER).primaryKey().autoNumbered(123), + SchemaUtils.versionColumn(), + SchemaUtils.column("bar", DataType.STRING, 10).nullable(), + SchemaUtils.column("baz", DataType.STRING, 10).nullable(), + SchemaUtils.column("bob", DataType.DECIMAL, 13, 2).nullable() + ).indexes( + SchemaUtils.index("fizz").unique().columns("bar", "baz") ); testConsumer.open(); List mockRecords = new ArrayList<>(); - mockRecords.add(record() - .setInteger(idColumn().getName(), 1) - .setInteger(versionColumn().getName(), 1) + mockRecords.add(DataSetUtils.record() + .setInteger(SchemaUtils.idColumn().getName(), 1) + .setInteger(SchemaUtils.versionColumn().getName(), 1) .setString("bar", "abc") .setString("baz", "123") .setString("bob", "456.78")); @@ -86,19 +81,19 @@ public void testSerialisationWithBlob() { DummyXmlOutputStreamProvider dummyXmlOutputStreamProvider = new DummyXmlOutputStreamProvider(); DataSetConsumer testConsumer = new XmlDataSetConsumer(dummyXmlOutputStreamProvider); - Table metaData = table("Test").columns( - idColumn(), - versionColumn(), - column("noel", DataType.STRING, 10).nullable(), - column("edmonds", DataType.STRING, 10).nullable(), - column("blobby", DataType.BLOB).nullable() + Table metaData = SchemaUtils.table("Test").columns( + SchemaUtils.idColumn(), + SchemaUtils.versionColumn(), + SchemaUtils.column("noel", DataType.STRING, 10).nullable(), + SchemaUtils.column("edmonds", DataType.STRING, 10).nullable(), + SchemaUtils.column("blobby", DataType.BLOB).nullable() ); testConsumer.open(); List mockRecords = new ArrayList<>(); - mockRecords.add(record() - .setInteger(idColumn().getName(), 1) - .setInteger(versionColumn().getName(), 1) + mockRecords.add(DataSetUtils.record() + .setInteger(SchemaUtils.idColumn().getName(), 1) + .setInteger(SchemaUtils.versionColumn().getName(), 1) .setString("noel", "noel") .setString("edmonds", "edmonds") .setString("blobby", "YmxvYmJ5")); @@ -117,11 +112,11 @@ public void testCompositePrimaryKeys() { DummyXmlOutputStreamProvider dummyXmlOutputStreamProvider = new DummyXmlOutputStreamProvider(); DataSetConsumer testConsumer = new XmlDataSetConsumer(dummyXmlOutputStreamProvider); - Table metaData = table("Test").columns( - idColumn(), - versionColumn(), - column("normalColumn", DataType.STRING, 10).nullable(), - column("col2", DataType.STRING, 10).primaryKey() + Table metaData = SchemaUtils.table("Test").columns( + SchemaUtils.idColumn(), + SchemaUtils.versionColumn(), + SchemaUtils.column("normalColumn", DataType.STRING, 10).nullable(), + SchemaUtils.column("col2", DataType.STRING, 10).primaryKey() ); testConsumer.open(); @@ -144,23 +139,23 @@ public void testClearDestinationBehaviour() { testConsumer.open(); testConsumer.close(CloseState.COMPLETE); - assertTrue("cleared", dummyXmlOutputStreamProvider.cleared()); + Assert.assertTrue("cleared", dummyXmlOutputStreamProvider.cleared()); } { DummyXmlOutputStreamProvider dummyXmlOutputStreamProvider = new DummyXmlOutputStreamProvider(); - DataSetConsumer testConsumer = new XmlDataSetConsumer(dummyXmlOutputStreamProvider, ClearDestinationBehaviour.CLEAR); + DataSetConsumer testConsumer = new XmlDataSetConsumer(dummyXmlOutputStreamProvider, DirectoryDataSetConsumer.ClearDestinationBehaviour.CLEAR); testConsumer.open(); testConsumer.close(CloseState.COMPLETE); - assertTrue("cleared", dummyXmlOutputStreamProvider.cleared()); + Assert.assertTrue("cleared", dummyXmlOutputStreamProvider.cleared()); } { DummyXmlOutputStreamProvider dummyXmlOutputStreamProvider = new DummyXmlOutputStreamProvider(); - DataSetConsumer testConsumer = new XmlDataSetConsumer(dummyXmlOutputStreamProvider, ClearDestinationBehaviour.OVERWRITE); + DataSetConsumer testConsumer = new XmlDataSetConsumer(dummyXmlOutputStreamProvider, DirectoryDataSetConsumer.ClearDestinationBehaviour.OVERWRITE); testConsumer.open(); testConsumer.close(CloseState.COMPLETE); - assertFalse("not cleared", dummyXmlOutputStreamProvider.cleared()); + Assert.assertFalse("not cleared", dummyXmlOutputStreamProvider.cleared()); } } @@ -173,16 +168,16 @@ public void testIndexOrdering() throws IOException { DummyXmlOutputStreamProvider dummyXmlOutputStreamProvider = new DummyXmlOutputStreamProvider(); DataSetConsumer testConsumer = new XmlDataSetConsumer(dummyXmlOutputStreamProvider); - Table testTable = table("TestTable") + Table testTable = SchemaUtils.table("TestTable") .columns( - column("x", DataType.DECIMAL, 10), - column("y", DataType.DECIMAL, 10), - column("z", DataType.DECIMAL, 10) + SchemaUtils.column("x", DataType.DECIMAL, 10), + SchemaUtils.column("y", DataType.DECIMAL, 10), + SchemaUtils.column("z", DataType.DECIMAL, 10) ).indexes( - index("CCC").columns("x"), // these will get re-ordered on export - index("AAA").columns("y"), - index("DDD").columns("z"), - index("BBB").columns("x", "y") + SchemaUtils.index("CCC").columns("x"), // these will get re-ordered on export + SchemaUtils.index("AAA").columns("y"), + SchemaUtils.index("DDD").columns("z"), + SchemaUtils.index("BBB").columns("x", "y") ); testConsumer.open(); @@ -193,7 +188,7 @@ public void testIndexOrdering() throws IOException { String expectedXML = SourceXML.readResource("testIndexOrdering.xml"); String actualXML = dummyXmlOutputStreamProvider.getXmlString().trim(); - assertEquals("Serialised data set", expectedXML, actualXML); + Assert.assertEquals("Serialised data set", expectedXML, actualXML); } @@ -201,19 +196,19 @@ public void testIndexOrdering() throws IOException { public void testWithNullCharacters() { DummyXmlOutputStreamProvider dummyXmlOutputStreamProvider = new DummyXmlOutputStreamProvider(); DataSetConsumer testConsumer = new XmlDataSetConsumer(dummyXmlOutputStreamProvider); - Table testTable = table("TestTable") + Table testTable = SchemaUtils.table("TestTable") .columns( - column("x", DataType.STRING, 10) + SchemaUtils.column("x", DataType.STRING, 10) ); testConsumer.open(); List records = ImmutableList.of( - record().setString("x", "foo"), - record().setString("x", new String(new char[] {'a', 0, 'c'})), - record().setString("x", new String(new char[] {0})), - record().setString("x", "string with a \\ in it"), - record().setString("x", "some \r valid \n things \t in this one") + DataSetUtils.record().setString("x", "foo"), + DataSetUtils.record().setString("x", new String(new char[] {'a', 0, 'c'})), + DataSetUtils.record().setString("x", new String(new char[] {0})), + DataSetUtils.record().setString("x", "string with a \\ in it"), + DataSetUtils.record().setString("x", "some \r valid \n things \t in this one") ); testConsumer.table(testTable, records); @@ -222,7 +217,7 @@ public void testWithNullCharacters() { String actualXML = dummyXmlOutputStreamProvider.getXmlString().trim(); String expectedXML = SourceXML.readResource("testWithNullCharacters.xml"); - assertEquals("Serialised data set", expectedXML, actualXML); + Assert.assertEquals("Serialised data set", expectedXML, actualXML); } } diff --git a/morf-core/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetProducer.java b/morf-xml/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetProducer.java similarity index 73% rename from morf-core/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetProducer.java rename to morf-xml/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetProducer.java index 87d1d20df..da2a4ea63 100755 --- a/morf-core/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetProducer.java +++ b/morf-xml/src/test/java/org/alfasoftware/morf/xml/TestXmlDataSetProducer.java @@ -15,44 +15,39 @@ package org.alfasoftware.morf.xml; -import static org.alfasoftware.morf.metadata.SchemaUtils.column; -import static org.alfasoftware.morf.metadata.SchemaUtils.table; -import static org.alfasoftware.morf.xml.SourceXML.readResource; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; import org.alfasoftware.morf.dataset.DataSetConnector; import org.alfasoftware.morf.dataset.DataSetConsumer; import org.alfasoftware.morf.dataset.DataSetConsumer.CloseState; import org.alfasoftware.morf.dataset.DataSetProducer; import org.alfasoftware.morf.dataset.Record; +import org.alfasoftware.morf.directory.DirectoryStreamProvider; import org.alfasoftware.morf.metadata.DataType; import org.alfasoftware.morf.metadata.SchemaHomology; +import org.alfasoftware.morf.metadata.SchemaUtils; import org.alfasoftware.morf.metadata.Table; -import org.alfasoftware.morf.xml.XmlStreamProvider.XmlInputStreamProvider; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import com.google.common.base.Charsets; -import com.google.common.collect.ImmutableList; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.alfasoftware.morf.xml.SourceXML.readResource; +import static org.junit.Assert.assertEquals; /** * Test cases to check XML can be parsed to a data set consumer. @@ -81,7 +76,7 @@ public void testFullXMLRoundTrip() { new DataSetConnector(producer, testConsumer).connect(); - assertEquals("output should be the same as input", SourceXML.FULL_SAMPLE, dummyXmlOutputStreamProvider.getXmlString().trim()); + Assert.assertEquals("output should be the same as input", SourceXML.FULL_SAMPLE, dummyXmlOutputStreamProvider.getXmlString().trim()); } @@ -99,7 +94,7 @@ public void testVersion1Xml() { new DataSetConnector(producer, testConsumer).connect(); - assertEquals("output should be the same as input", SourceXML.FULL_SAMPLE_V1_UPGRADED, dummyXmlOutputStreamProvider.getXmlString().trim()); + Assert.assertEquals("output should be the same as input", SourceXML.FULL_SAMPLE_V1_UPGRADED, dummyXmlOutputStreamProvider.getXmlString().trim()); } @@ -111,7 +106,7 @@ public void testVersion1Xml() { public void testReducedXMLRoundTrip() { DataSetProducer producer = new XmlDataSetProducer(new TestXmlInputStreamProvider(SourceXML.REDUCED_SAMPLE)); - DataSetConsumer consumer = mock(DataSetConsumer.class); + DataSetConsumer consumer = Mockito.mock(DataSetConsumer.class); new DataSetConnector(producer, consumer).connect(); @@ -123,16 +118,16 @@ public void testReducedXMLRoundTrip() { Mockito.verify(consumer).table(tableCaptor.capture(), iterableCaptor.capture()); SchemaHomology schemaHomology = new SchemaHomology(new SchemaHomology.ThrowingDifferenceWriter()); - assertTrue( + Assert.assertTrue( schemaHomology.tablesMatch( tableCaptor.getValue(), - table("Test") + SchemaUtils.table("Test") .columns( - column("id", DataType.BIG_INTEGER).primaryKey(), - column("version", DataType.INTEGER).defaultValue("0"), - column("bar", DataType.STRING), - column("baz", DataType.STRING), - column("bob", DataType.DECIMAL) + SchemaUtils.column("id", DataType.BIG_INTEGER).primaryKey(), + SchemaUtils.column("version", DataType.INTEGER).defaultValue("0"), + SchemaUtils.column("bar", DataType.STRING), + SchemaUtils.column("baz", DataType.STRING), + SchemaUtils.column("bob", DataType.DECIMAL) ) ) ); @@ -148,12 +143,12 @@ public void testReducedXMLRoundTrip() { public void testBlobXML() { DataSetProducer producer = new XmlDataSetProducer(new TestXmlInputStreamProvider(SourceXML.BLOBBY_SAMPLE)); - assertEquals("exactly 1 table expected in schema", 1, producer.getSchema().tables().size()); - assertEquals("exactly 5 columns expected in table", 5, producer.getSchema().getTable("Test").columns().size()); + Assert.assertEquals("exactly 1 table expected in schema", 1, producer.getSchema().tables().size()); + Assert.assertEquals("exactly 5 columns expected in table", 5, producer.getSchema().getTable("Test").columns().size()); for (Record record : producer.records("Test")) { - assertEquals("record value incorrect", "noel", record.getString("noel")); - assertEquals("record value incorrect", "edmonds", record.getString("edmonds")); - assertEquals("record value incorrect", "YmxvYmJ5", record.getString("blobby")); + Assert.assertEquals("record value incorrect", "noel", record.getString("noel")); + Assert.assertEquals("record value incorrect", "edmonds", record.getString("edmonds")); + Assert.assertEquals("record value incorrect", "YmxvYmJ5", record.getString("blobby")); } } @@ -206,13 +201,13 @@ public void testTableNamesAgainstArchive() throws MalformedURLException { */ private void testTableNamesAgainstProducer(XmlDataSetProducer producer) { producer.open(); - assertTrue("EntityOne", producer.getSchema().tableNames().contains("EntityOne")); - assertTrue("EntityTwo", producer.getSchema().tableNames().contains("EntityTwo")); + Assert.assertTrue("EntityOne", producer.getSchema().tableNames().contains("EntityOne")); + Assert.assertTrue("EntityTwo", producer.getSchema().tableNames().contains("EntityTwo")); use(producer.records("EntityOne")); - assertTrue("eNTITYoNE", producer.getSchema().tableExists("eNTITYoNE")); + Assert.assertTrue("eNTITYoNE", producer.getSchema().tableExists("eNTITYoNE")); use(producer.records("eNTITYoNE")); - assertFalse("Non existant table", producer.getSchema().tableNames().contains("NotExist")); + Assert.assertFalse("Non existant table", producer.getSchema().tableNames().contains("NotExist")); producer.close(); } @@ -238,8 +233,8 @@ public void testCompositePrimaryKey() { producer.open(); assertEquals("exactly 1 table expected in schema", 1, producer.getSchema().tables().size()); assertEquals("exactly 4 columns expected in table", 4, producer.getSchema().getTable("Test").columns().size()); - assertTrue("first column is primary key", producer.getSchema().getTable("Test").columns().get(0).isPrimaryKey()); - assertTrue("last column is primary key", producer.getSchema().getTable("Test").columns().get(3).isPrimaryKey()); + Assert.assertTrue("first column is primary key", producer.getSchema().getTable("Test").columns().get(0).isPrimaryKey()); + Assert.assertTrue("last column is primary key", producer.getSchema().getTable("Test").columns().get(3).isPrimaryKey()); producer.close(); } @@ -256,19 +251,19 @@ public void testWithComments() throws IOException { Table fooTable = producer.getSchema().getTable("Foo"); - assertTrue( + Assert.assertTrue( new SchemaHomology().tablesMatch( fooTable, - table("Foo") + SchemaUtils.table("Foo") .columns( - column("id", DataType.BIG_INTEGER).primaryKey()) + SchemaUtils.column("id", DataType.BIG_INTEGER).primaryKey()) ) ); List records = ImmutableList.copyOf(producer.records("Foo")); - assertEquals(40646L, records.get(0).getLong("id").longValue()); - assertEquals(40641L, records.get(3).getLong("id").longValue()); + Assert.assertEquals(40646L, records.get(0).getLong("id").longValue()); + Assert.assertEquals(40641L, records.get(3).getLong("id").longValue()); producer.close(); } @@ -281,11 +276,11 @@ public void testPartialData() throws IOException { List records = ImmutableList.copyOf(producer.records("TestTable")); - assertEquals("1", records.get(0).getString("x")); - assertEquals("2", records.get(1).getString("x")); + Assert.assertEquals("1", records.get(0).getString("x")); + Assert.assertEquals("2", records.get(1).getString("x")); - assertEquals("ABC", records.get(0).getString("y")); - assertEquals(null, records.get(1).getString("y")); + Assert.assertEquals("ABC", records.get(0).getString("y")); + Assert.assertEquals(null, records.get(1).getString("y")); producer.close(); } @@ -299,7 +294,7 @@ public void testFutureFormatFails() throws IOException { try { ImmutableList.copyOf(producer.records("TestTable")); - fail(); + Assert.fail(); } catch(IllegalStateException ise) { // ok } finally { @@ -311,9 +306,9 @@ public void testFutureFormatFails() throws IOException { private void validateDataSetProducerWithNullsAndBackslashes(DataSetProducer dataSetProducer) { dataSetProducer.open(); ImmutableList records = ImmutableList.copyOf(dataSetProducer.records("Foo")); - assertEquals(new String(new char[] {'A', 0 /*null*/, 'C'}), records.get(0).getString("val")); - assertEquals(new String(new char[] { 0 /*null*/ }), records.get(1).getString("val")); - assertEquals("escape\\it", records.get(2).getString("val")); + Assert.assertEquals(new String(new char[] {'A', 0 /*null*/, 'C'}), records.get(0).getString("val")); + Assert.assertEquals(new String(new char[] { 0 /*null*/ }), records.get(1).getString("val")); + Assert.assertEquals("escape\\it", records.get(2).getString("val")); dataSetProducer.close(); } @@ -337,16 +332,16 @@ public void testWithUnusualCharacters() { try { ImmutableList r = ImmutableList.copyOf(dataSetProducer.records("testWithUnusualCharacters")); - assertEquals("\u0000", r.get(0).getString("characterValue")); - assertEquals("&", r.get(38).getString("characterValue")); - assertEquals(">", r.get(62).getString("characterValue")); - assertEquals("A", r.get(65).getString("characterValue")); - assertEquals("\n", r.get(10).getString("characterValue")); - assertEquals("\r", r.get(13).getString("characterValue")); + Assert.assertEquals("\u0000", r.get(0).getString("characterValue")); + Assert.assertEquals("&", r.get(38).getString("characterValue")); + Assert.assertEquals(">", r.get(62).getString("characterValue")); + Assert.assertEquals("A", r.get(65).getString("characterValue")); + Assert.assertEquals("\n", r.get(10).getString("characterValue")); + Assert.assertEquals("\r", r.get(13).getString("characterValue")); - assertEquals("\ufffd", r.get(344).getString("characterValue")); - assertEquals("\ufffe", r.get(345).getString("characterValue")); - assertEquals("\uffff", r.get(346).getString("characterValue")); + Assert.assertEquals("\ufffd", r.get(344).getString("characterValue")); + Assert.assertEquals("\ufffe", r.get(345).getString("characterValue")); + Assert.assertEquals("\uffff", r.get(346).getString("characterValue")); } finally { dataSetProducer.close(); @@ -359,7 +354,7 @@ public void testWithUnusualCharacters() { * * @author Copyright (c) Alfa Financial Software 2010 */ - private static final class TestXmlInputStreamProvider implements XmlInputStreamProvider { + private static final class TestXmlInputStreamProvider implements DirectoryStreamProvider.DirectoryInputStreamProvider { private final String content; private final String tableName; @@ -407,7 +402,7 @@ public void open() { */ @Override public InputStream openInputStreamForTable(String tableName) { - assertEquals("Table name", this.tableName.toUpperCase(), tableName.toUpperCase()); + Assert.assertEquals("Table name", this.tableName.toUpperCase(), tableName.toUpperCase()); return new ByteArrayInputStream(content.getBytes(Charsets.UTF_8)); } diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/dataset.zip b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/dataset.zip similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/dataset.zip rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/dataset.zip diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/test.zip b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/test.zip similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/test.zip rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/test.zip diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/testFutureFormatFails.xml b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/testFutureFormatFails.xml similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/testFutureFormatFails.xml rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/testFutureFormatFails.xml diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/testIndexOrdering.xml b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/testIndexOrdering.xml similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/testIndexOrdering.xml rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/testIndexOrdering.xml diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/testPartialData.xml b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/testPartialData.xml similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/testPartialData.xml rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/testPartialData.xml diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithComments.xml b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithComments.xml similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithComments.xml rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithComments.xml diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacterReferencesV2.xml b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacterReferencesV2.xml similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacterReferencesV2.xml rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacterReferencesV2.xml diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacterReferencesV3.xml b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacterReferencesV3.xml similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacterReferencesV3.xml rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacterReferencesV3.xml diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacters.xml b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacters.xml similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacters.xml rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithNullCharacters.xml diff --git a/morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithUnusualCharacters.xml b/morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithUnusualCharacters.xml similarity index 100% rename from morf-core/src/test/resources/org/alfasoftware/morf/xml/testWithUnusualCharacters.xml rename to morf-xml/src/test/resources/org/alfasoftware/morf/xml/testWithUnusualCharacters.xml diff --git a/pom.xml b/pom.xml index 6660f2888..d18811a8b 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ + morf-avro morf-core morf-testsupport morf-excel @@ -26,6 +27,8 @@ morf-postgresql morf-sqlserver morf-integration-test + morf-directory-spi + morf-xml @@ -62,9 +65,8 @@ 1.2.15 3.9.0 2.6.9 - 2.7.1 2.6.4 - 1.5.0 + 2.0.6 morf-testsupport/** @@ -242,11 +244,21 @@ + + org.alfasoftware + morf-avro + ${project.version} + org.alfasoftware morf-core ${project.version} + + org.alfasoftware + morf-directory-spi + ${project.version} + org.alfasoftware morf-testsupport @@ -287,7 +299,11 @@ morf-sqlserver ${project.version} - + + org.alfasoftware + morf-xml + ${project.version} + org.apache.commons commons-lang3 @@ -387,11 +403,6 @@ slf4j-simple ${slf4j.version} - - xalan - serializer - ${serializer.version} -