Import Jore 3 ids for lines and scheduled stop points

docker/docker-compose.custom.yml

@@ -36,13 +36,13 @@ services:
 
   jore4-mssqltestdb:
     # pin compatible version of mssql schema
-    image: "hsldevcom/jore4-mssql-testdb:schema-only-jr_linja_vaatimus-table--20220523-1854abfa889ee6432878c5203a115ad4945506d3"
+    image: "hsldevcom/jore4-mssql-testdb:schema-only-fix-jore-schema--20230420-9851067c8d776b75af133c48e8be1f277d861403"
     # volume to easily use jore3 database dump
     volumes:
       - ../jore3dump:/mnt/jore3dump
 
   jore4-hasura:
     # pin compatible version of jore4 data model
-    image: "hsldevcom/jore4-hasura:hsl-main--20230324-beb6ecbde2b638b79af499e7f18a568d85f1541e"
+    image: "hsldevcom/jore4-hasura:hsl-add-numeric-id-fields--20230417-7193ce6efeb1979dde29720dd899f90b548df4e5"
 networks:
     jore4:

src/main/java/fi/hsl/jore/importer/feature/batch/line/LineExportMapper.java

@@ -31,6 +31,7 @@ public LineExportMapper(final IJsonbConverter jsonConverter) {
     public ImporterLine mapRow(final ResultSet resultSet, final int rowNumber) throws SQLException {
         return ImporterLine.of(
                 ExternalId.of(resultSet.getString("external_id")),
+                resultSet.getString("export_id"),
                 resultSet.getString("line_number"),
                 jsonConverter.fromJson(resultSet.getString("name"), MultilingualString.class),
                 NetworkType.of(resultSet.getString("network_type")),

src/main/java/fi/hsl/jore/importer/feature/batch/line/LineExportProcessor.java

@@ -25,10 +25,24 @@ public class LineExportProcessor implements ItemProcessor<ImporterLine, Jore4Lin
     @Override
     public Jore4Line process(final ImporterLine input) throws Exception {
         LOGGER.debug("Processing line: {}", input);
+        final String externalId = input.externalId().value();
+
+        final Short exportId = tryParseExportId(input.exportId());
+        if (exportId == null) {
+            // The id field is supposed to be a 4-digit number stored as string.
+            // The field is required, so abort processing of whole line if we can't handle it.
+            LOGGER.warn(
+                "Skipping importing line {} from Jore3: could not parse export_id as short: {}.",
+                externalId,
+                input.exportId()
+            );
+            return null;
+        }
 
         return Jore4Line.of(
                 UUID.randomUUID(),
-                input.externalId().value(),
+                externalId,
+                exportId,
                 input.lineNumber(),
                 input.name(),
                 input.shortName(),
@@ -40,4 +54,12 @@ public Jore4Line process(final ImporterLine input) throws Exception {
                 input.legacyHslMunicipalityCode()
         );
     }
+
+    private Short tryParseExportId(String exportId) {
+         try {
+            return Short.parseShort(exportId);
+        } catch (final NumberFormatException ignored) {
+            return null;
+        }
+    }
 }

src/main/java/fi/hsl/jore/importer/feature/batch/line/LineExportWriter.java

@@ -2,6 +2,8 @@
 
 import fi.hsl.jore.importer.feature.batch.line.support.ILineImportRepository;
 import fi.hsl.jore.importer.feature.jore4.entity.Jore4Line;
+import fi.hsl.jore.importer.feature.jore4.entity.Jore4LineExternalId;
+import fi.hsl.jore.importer.feature.jore4.repository.IJore4LineExternalIdRepository;
 import fi.hsl.jore.importer.feature.jore4.repository.IJore4LineRepository;
 import fi.hsl.jore.importer.feature.network.line.dto.PersistableLineIdMapping;
 import org.springframework.batch.item.ItemWriter;
@@ -17,18 +19,40 @@
 public class LineExportWriter implements ItemWriter<Jore4Line> {
 
     private final ILineImportRepository importerRepository;
-    private final IJore4LineRepository jore4Repository;
+    private final IJore4LineRepository jore4LineRepository;
+    private final IJore4LineExternalIdRepository jore4LineExternalIdRepository;
 
     @Autowired
     public LineExportWriter(final ILineImportRepository importerRepository,
-                            final IJore4LineRepository jore4Repository) {
+                            final IJore4LineRepository jore4LineRepository,
+                            final IJore4LineExternalIdRepository jore4LineExternalIdRepository) {
         this.importerRepository = importerRepository;
-        this.jore4Repository = jore4Repository;
+        this.jore4LineRepository = jore4LineRepository;
+        this.jore4LineExternalIdRepository = jore4LineExternalIdRepository;
     }
 
     @Override
     public void write(final List<? extends Jore4Line> items) throws Exception {
-        jore4Repository.insert(items);
+        /**
+         * Insert related line_external_ids first so that lines use them and don't get new ids generated.
+         *
+         * Note on id terminology, on some relevant ids:
+         * - Importer database (imported from Jore 3): external_id (the primary key) and export_id (used eg. in exports)
+         * - Jore 4: line_external_id
+         * Importantly, the Jore 4 line_external_id is __NOT the same__ as Jore 3 external_id.
+         * Instead, the Jore 4 line_external_id is set based on Jore 3 export_id.
+         * The Jore 3 external_id is not exported to Jore 4 at all (we just need it here to write Jore 4 ids back to importer database).
+         * In short, Jore 3 export_id -> Jore 4 line_external_id
+         */
+        final io.vavr.collection.List<Jore4LineExternalId> jore4LineExternalIds = items.stream()
+                .map(item -> Jore4LineExternalId.of(
+                        item.label(),
+                        item.exportId() // Yes, this is correct.
+                ))
+                .collect(io.vavr.collection.List.collector());
+        jore4LineExternalIdRepository.insert(jore4LineExternalIds.toJavaList());
+
+        jore4LineRepository.insert(items);
 
         final io.vavr.collection.List<PersistableLineIdMapping> jore4IdMappings = items.stream()
                 .map(item -> PersistableLineIdMapping.of(

src/main/java/fi/hsl/jore/importer/feature/batch/line/LineProcessor.java

@@ -20,6 +20,7 @@ public PersistableLine process(final JrLine item) {
         final LegacyPublicTransportDestination lineLegacyPublicTransportDestination = item.lineId().destination();
 
         return PersistableLine.of(ExternalIdUtil.forLine(item),
+                                  item.id(),
                                   lineNumber,
                                   TransitTypeToNetworkTypeMapper.resolveNetworkType(item.transitType()),
                                   LineClassificationUtil.resolveTypeOfLine(item.transitType(),
@@ -29,4 +30,5 @@ public PersistableLine process(final JrLine item) {
                                   LegacyHslMunicipalityCode.of(lineLegacyPublicTransportDestination)
         );
     }
+
 }

src/main/java/fi/hsl/jore/importer/feature/batch/line/LineRowMapper.java

@@ -25,6 +25,7 @@ public class LineRowMapper implements RowMapper<JrLine> {
     public JrLine mapRow(final ResultSet rs,
                          final int rowNum) throws SQLException {
         return JrLine.of(LineId.from(getStringOrThrow(rs, "lintunnus")),
+                         getStringOrThrow(rs, "id"),
                          getOptionalString(rs, "linverkko")
                                  .flatMap(TransitType::of)
                                  .orElse(TransitType.UNKNOWN),

src/main/java/fi/hsl/jore/importer/feature/batch/line/support/LineImportRepository.java

@@ -45,13 +45,15 @@ public void clearStagingTable() {
     public void submitToStaging(final Iterable<? extends PersistableLine> lines) {
         final BatchBindStep batch = db.batch(db.insertInto(STAGING_TABLE,
                                                            STAGING_TABLE.NETWORK_LINE_EXT_ID,
+                                                           STAGING_TABLE.NETWORK_LINE_EXPORT_ID,
                                                            STAGING_TABLE.NETWORK_LINE_NUMBER,
                                                            STAGING_TABLE.INFRASTRUCTURE_NETWORK_TYPE,
                                                            STAGING_TABLE.NETWORK_LINE_TYPE_OF_LINE,
                                                            STAGING_TABLE.NETWORK_LINE_LEGACY_HSL_MUNICIPALITY_CODE)
-                                               .values((String) null, null, null, null, null));
+                                               .values((String) null, null, null, null, null, null));
 
         lines.forEach(line -> batch.bind(line.externalId().value(),
+                                         line.exportId(),
                                          line.lineNumber(),
                                          line.networkType().label(),
                                          line.typeOfLine().getValue(),
@@ -83,11 +85,13 @@ protected Set<LinePK> update() {
     protected Set<LinePK> insert() {
         return db.insertInto(TARGET_TABLE)
                  .columns(TARGET_TABLE.NETWORK_LINE_EXT_ID,
+                          TARGET_TABLE.NETWORK_LINE_EXPORT_ID,
                           TARGET_TABLE.NETWORK_LINE_NUMBER,
                           TARGET_TABLE.INFRASTRUCTURE_NETWORK_TYPE,
                           TARGET_TABLE.NETWORK_LINE_TYPE_OF_LINE,
                           TARGET_TABLE.NETWORK_LINE_LEGACY_HSL_MUNICIPALITY_CODE)
                  .select(db.select(STAGING_TABLE.NETWORK_LINE_EXT_ID,
+                                   STAGING_TABLE.NETWORK_LINE_EXPORT_ID,
                                    STAGING_TABLE.NETWORK_LINE_NUMBER,
                                    STAGING_TABLE.INFRASTRUCTURE_NETWORK_TYPE,
                                    STAGING_TABLE.NETWORK_LINE_TYPE_OF_LINE,

src/main/java/fi/hsl/jore/importer/feature/batch/scheduled_stop_point/ScheduledStopPointExportProcessor.java

@@ -65,6 +65,17 @@ public Jore4ScheduledStopPoint process(final ImporterScheduledStopPoint jore3Sto
         for (int index = 0; index < elyNumbers.size(); index++) {
             final ExternalId externalId = externalIds.get(index);
             final Long elyNumber = elyNumbers.get(index);
+            final Integer externalIdForExport = tryParseExternalId(externalId);
+            if (externalIdForExport == null) {
+                // Jore3 stop ids are always supposed to be numbers, even if they were stored as strings.
+                // The field is required, so abort processing of whole stop point if we can't handle it.
+                LOGGER.warn(
+                    "Skipping importing scheduled stop point {} from Jore3: could not parse external_id as integer: {}.",
+                    jore3Stop.shortId().get(),
+                    externalId.value()
+                );
+                continue;
+            }
 
             final Optional<DigiroadStop> digiroadStopContainer = digiroadStopService.findByNationalId(elyNumber);
             if (digiroadStopContainer.isPresent()) {
@@ -74,6 +85,7 @@ public Jore4ScheduledStopPoint process(final ImporterScheduledStopPoint jore3Sto
                 final Jore4ScheduledStopPoint jore4Stop = Jore4ScheduledStopPoint.of(
                         UUID.randomUUID(),
                         externalId.value(),
+                        externalIdForExport,
                         digiroadStop.digiroadLinkId(),
                         Jore4ScheduledStopPointDirection.valueOf(digiroadStop.directionOnInfraLink().name()),
                         jore3Stop.shortId().get(),
@@ -95,4 +107,12 @@ public Jore4ScheduledStopPoint process(final ImporterScheduledStopPoint jore3Sto
         );
         return null;
     }
+
+    private Integer tryParseExternalId(ExternalId externalId) {
+        try {
+            return Integer.parseInt(externalId.value());
+        } catch (final NumberFormatException ignored) {
+            return null;
+        }
+    }
 }

src/main/java/fi/hsl/jore/importer/feature/jore3/entity/JrLine.java

@@ -30,6 +30,12 @@ default JrLinePk pk() {
         return JrLinePk.of(lineId());
     }
 
+    // Note: references in Jore3 are still by primary key even if this is called "id".
+    @JoreColumn(name = "id",
+                nullable = false,
+                example = "1001")
+    String id();
+
     @JoreColumn(name = "lintilorg",
                 nullable = true,
                 example = "HEL")
@@ -50,13 +56,15 @@ default JrLinePk pk() {
     boolean isTrunkLine();
 
     static JrLine of(final LineId lineId,
+                     final String id,
                      final TransitType transitType,
                      final ClientOrganization clientOrganization,
                      final PublicTransportType publicTransportType,
                      final PublicTransportDestination publicTransportDestination,
                      final boolean isTrunkLine) {
         return ImmutableJrLine.builder()
                               .lineId(lineId)
+                              .id(id)
                               .transitType(transitType)
                               .clientOrganization(clientOrganization)
                               .publicTransportType(publicTransportType)

src/main/java/fi/hsl/jore/importer/feature/jore4/Jore4DataEraser.java

@@ -12,6 +12,7 @@
 import static fi.hsl.jore.jore4.jooq.route.Tables.LINE;
 import static fi.hsl.jore.jore4.jooq.route.Tables.ROUTE_;
 import static fi.hsl.jore.jore4.jooq.service_pattern.Tables.SCHEDULED_STOP_POINT;
+import static fi.hsl.jore.jore4.jooq.service_pattern.Tables.VEHICLE_MODE_ON_SCHEDULED_STOP_POINT;
 import static fi.hsl.jore.jore4.jooq.timing_pattern.Tables.TIMING_PLACE;
 
 @Component
@@ -42,6 +43,7 @@ public void deleteRoutesAndLines() {
     @Transactional
     @Override
     public void deleteScheduledStopPointsAndTimingPlaces() {
+        db.truncateTable(VEHICLE_MODE_ON_SCHEDULED_STOP_POINT).restartIdentity().execute();
         db.truncateTable(SCHEDULED_STOP_POINT).restartIdentity().cascade().execute();
         db.truncateTable(TIMING_PLACE).restartIdentity().cascade().execute();
     }

src/main/java/fi/hsl/jore/importer/feature/jore4/entity/Jore4Line.java

@@ -18,6 +18,8 @@ public interface Jore4Line {
 
     String externalLineId();
 
+    Short exportId();
+
     String label();
 
     MultilingualString name();
@@ -38,6 +40,7 @@ public interface Jore4Line {
 
     static ImmutableJore4Line of(final UUID lineId,
                                  final String externalLineId,
+                                 final Short exportId,
                                  final String label,
                                  final MultilingualString name,
                                  final MultilingualString shortName,
@@ -50,6 +53,7 @@ static ImmutableJore4Line of(final UUID lineId,
         return ImmutableJore4Line.builder()
                 .lineId(lineId)
                 .externalLineId(externalLineId)
+                .exportId(exportId)
                 .label(label)
                 .name(name)
                 .shortName(shortName)

src/main/java/fi/hsl/jore/importer/feature/jore4/entity/Jore4LineExternalId.java

@@ -0,0 +1,23 @@
+package fi.hsl.jore.importer.feature.jore4.entity;
+
+import org.immutables.value.Value;
+
+/**
+ * Contains the information of a line external id
+ * which can inserted into the Jore 4 database.
+ */
+@Value.Immutable
+public interface Jore4LineExternalId {
+
+    String label();
+
+    Short externalId();
+
+    static ImmutableJore4LineExternalId of(final String label,
+                                           final Short externalId) {
+        return ImmutableJore4LineExternalId.builder()
+                .label(label)
+                .externalId(externalId)
+                .build();
+    }
+}

src/main/java/fi/hsl/jore/importer/feature/jore4/entity/Jore4ScheduledStopPoint.java

@@ -18,8 +18,17 @@ public interface Jore4ScheduledStopPoint {
 
     String externalInfrastructureLinkId();
 
+    /**
+     * The unprocessed external id from importer database.
+     * Required because write Jore4 ids back to importer database and use this for matching.
+     */
     String externalScheduledStopPointId();
 
+    /**
+     * Tha parsed external id that will be exported to Jore4 database.
+     */
+    Integer externalIdForExport();
+
     Jore4ScheduledStopPointDirection directionOnInfraLink();
 
     String label();
@@ -36,6 +45,7 @@ public interface Jore4ScheduledStopPoint {
 
     static ImmutableJore4ScheduledStopPoint of(final UUID scheduledStopPointId,
                                                final String externalScheduledStopPointId,
+                                               final Integer externalIdForExport,
                                                final String externalInfrastructureLinkId,
                                                final Jore4ScheduledStopPointDirection directionOnInfraLink,
                                                final String label,
@@ -47,6 +57,7 @@ static ImmutableJore4ScheduledStopPoint of(final UUID scheduledStopPointId,
         return ImmutableJore4ScheduledStopPoint.builder()
                 .scheduledStopPointId(scheduledStopPointId)
                 .externalScheduledStopPointId(externalScheduledStopPointId)
+                .externalIdForExport(externalIdForExport)
                 .externalInfrastructureLinkId(externalInfrastructureLinkId)
                 .directionOnInfraLink(directionOnInfraLink)
                 .label(label)

src/main/java/fi/hsl/jore/importer/feature/jore4/repository/IJore4LineExternalIdRepository.java

@@ -0,0 +1,18 @@
+package fi.hsl.jore.importer.feature.jore4.repository;
+
+import fi.hsl.jore.importer.feature.jore4.entity.Jore4LineExternalId;
+
+import java.util.List;
+
+/**
+ * Declares CRUD operations for line external id table
+ * which are found from the Jore 4 database.
+ */
+public interface IJore4LineExternalIdRepository {
+
+    /**
+     * Inserts the line external ids into the database.
+     * @param lineExternalIds The information of the inserted line external ids.
+     */
+    void insert(List<? extends Jore4LineExternalId> lineExternalIds);
+}