Release 1.0.0

.github/workflows/build.yml

@@ -2,7 +2,10 @@ name: Build
 
 on:
   push:
-    branches: [ main, develop ]
+    branches:
+      - main
+      - develop
+      - 'release/*'
     tags:
       - '[0-9]+.[0-9]+.[0-9]+*'
   pull_request:

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/config/BusinessRulesConfig.java

@@ -69,8 +69,7 @@ private PipelineTask sanityCheckPipelineTask() {
 
     private PipelineTask preCleansingPipelineTask() {
         return new DefaultPipelineTask(List.of(
-                new DiscardZeroBalanceTxItemsTaskItem(),
-                new JournalAccountCreditEnrichmentTaskItem(organisationPublicApi)
+                new DiscardZeroBalanceTxItemsTaskItem()
         ));
     }
 
@@ -79,7 +78,8 @@ private PipelineTask preValidationPipelineTask() {
                 new AmountsFcyCheckTaskItem(),
                 new AmountsLcyCheckTaskItem(),
                 new AmountLcyBalanceZerosOutCheckTaskItem(),
-                new AmountFcyBalanceZerosOutCheckTaskItem()
+                new AmountFcyBalanceZerosOutCheckTaskItem(),
+                new JournalAccountCreditEnrichmentTaskItem(organisationPublicApi)
         ));
     }
 

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/job/TxStatusUpdaterJob.java

@@ -0,0 +1,71 @@
+package org.cardanofoundation.lob.app.accounting_reporting_core.job;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.TxStatusUpdate;
+import org.cardanofoundation.lob.app.accounting_reporting_core.domain.entity.TransactionEntity;
+import org.cardanofoundation.lob.app.accounting_reporting_core.service.internal.LedgerService;
+import org.cardanofoundation.lob.app.accounting_reporting_core.service.internal.TransactionBatchService;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+@ConditionalOnProperty(value = "lob.accounting_reporting_core.enabled", havingValue = "true", matchIfMissing = true)
+public class TxStatusUpdaterJob {
+
+    private final Map<String, TxStatusUpdate> txStatusUpdatesMap = new ConcurrentHashMap<>();
+    private final LedgerService ledgerService;
+    private final TransactionBatchService transactionBatchService;
+
+    @Value("${lob.blockchain.tx-status-updater.max-map-size:1000}")
+    private int maxMapSize;
+
+    // This Job collects all TxStatusUpdate events and updates the transactions in the database
+    @Scheduled(
+            fixedDelayString = "${ob.blockchain.tx-status-updater.fixed_delay:PT30S}",
+            initialDelayString = "${lob.blockchain.tx-status-updater.delay:PT30S}")
+    @Transactional
+    public void execute() {
+        Map<String, TxStatusUpdate> updates;
+        synchronized (txStatusUpdatesMap) {
+            updates = new HashMap<>(txStatusUpdatesMap);
+        }
+        if(updates.isEmpty()) {
+            log.debug("No TxStatusUpdate events to process");
+            return;
+        }
+        try {
+            log.info("Updating Status of {} transactions", updates.size());
+            List<TransactionEntity> transactionEntities = ledgerService.updateTransactionsWithNewStatuses(updates);
+            ledgerService.saveAllTransactionEntities(transactionEntities);
+
+            transactionBatchService.updateBatchesPerTransactions(updates);
+            updates.forEach(txStatusUpdatesMap::remove);
+        } catch (Exception e) {
+            log.error("Failed to process TxStatusUpdates - entries will be retained in the map", e);
+        }
+    }
+
+    public void addToStatusUpdateMap(Map<String, TxStatusUpdate> updateMap) {
+        synchronized (txStatusUpdatesMap) {
+            txStatusUpdatesMap.putAll(updateMap);
+        }
+        if(txStatusUpdatesMap.size() > maxMapSize) {
+            log.warn("TxStatusUpdate map size exceeded the limit of {}. Current size: {}", maxMapSize, txStatusUpdatesMap.size());
+        }
+    }
+
+
+}

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/resource/presentation_layer_service/AccountingCorePresentationViewService.java

@@ -36,6 +36,7 @@
 import org.cardanofoundation.lob.app.accounting_reporting_core.resource.views.*;
 import org.cardanofoundation.lob.app.accounting_reporting_core.service.internal.AccountingCoreService;
 import org.cardanofoundation.lob.app.accounting_reporting_core.service.internal.TransactionRepositoryGateway;
+import org.cardanofoundation.lob.app.organisation.OrganisationPublicApiIF;
 import org.cardanofoundation.lob.app.organisation.domain.entity.OrganisationCostCenter;
 import org.cardanofoundation.lob.app.organisation.domain.entity.OrganisationProject;
 import org.cardanofoundation.lob.app.organisation.repository.CostCenterRepository;
@@ -57,6 +58,7 @@ public class AccountingCorePresentationViewService {
     private final TransactionReconcilationRepository transactionReconcilationRepository;
     private final CostCenterRepository costCenterRepository;
     private final ProjectMappingRepository projectMappingRepository;
+    private final OrganisationPublicApiIF organisationPublicApiIF;
 
     /**
      * TODO: waiting for refactoring the layer to remove this
@@ -519,7 +521,8 @@ public BigDecimal getAmountLcyTotalForAllDebitItems(TransactionEntity tx) {
         Set<TransactionItemEntity> items = tx.getItems();
 
         if (tx.getTransactionType().equals(TransactionType.Journal)) {
-            items = tx.getItems().stream().filter(txItems -> txItems.getOperationType().equals(OperationType.DEBIT)).collect(toSet());
+            Optional<String> dummyAccount = organisationPublicApiIF.findByOrganisationId(tx.getOrganisation().getId()).orElse(new org.cardanofoundation.lob.app.organisation.domain.entity.Organisation()).getDummyAccount();
+            items = tx.getItems().stream().filter(txItems -> txItems.getAccountDebit().isPresent() && txItems.getAccountDebit().get().getCode().equals(dummyAccount.orElse(""))).collect(toSet());
         }
 
         if (tx.getTransactionType().equals(TransactionType.FxRevaluation)) {

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/resource/presentation_layer_service/ExtractionItemService.java

@@ -21,18 +21,24 @@
 import org.cardanofoundation.lob.app.accounting_reporting_core.repository.TransactionItemExtractionRepository;
 import org.cardanofoundation.lob.app.accounting_reporting_core.resource.views.ExtractionTransactionItemView;
 import org.cardanofoundation.lob.app.accounting_reporting_core.resource.views.ExtractionTransactionView;
+import org.cardanofoundation.lob.app.organisation.OrganisationPublicApi;
+import org.cardanofoundation.lob.app.organisation.domain.entity.OrganisationCostCenter;
+import org.cardanofoundation.lob.app.organisation.domain.entity.OrganisationProject;
 
 @Service
 @Slf4j
 @RequiredArgsConstructor
 @Transactional
 public class ExtractionItemService {
     private final TransactionItemExtractionRepository transactionItemRepositoryImpl;
+    private final OrganisationPublicApi organisationPublicApi;
 
     @Transactional(readOnly = true)
     public ExtractionTransactionView findTransactionItems(LocalDate dateFrom, LocalDate dateTo, List<String> accountCode, List<String> costCenter, List<String> project, List<String> accountType, List<String> accountSubType) {
 
-        List<ExtractionTransactionItemView> transactionItem = transactionItemRepositoryImpl.findByItemAccount(dateFrom, dateTo, accountCode, costCenter, project,accountType,accountSubType).stream().map(this::extractionTransactionItemViewBuilder).collect(Collectors.toList());
+        List<ExtractionTransactionItemView> transactionItem = transactionItemRepositoryImpl.findByItemAccount(dateFrom, dateTo, accountCode, costCenter, project, accountType, accountSubType).stream().map(item -> {
+            return extractionTransactionItemViewBuilder(item);
+        }).collect(Collectors.toList());
 
         return ExtractionTransactionView.createSuccess(transactionItem);
     }
@@ -79,7 +85,9 @@ private ExtractionTransactionItemView extractionTransactionItemViewBuilder(Trans
                 item.getDocument().flatMap(d -> d.getCounterparty().map(Counterparty::getCustomerCode)).orElse(null),
                 item.getDocument().flatMap(d -> d.getCounterparty().map(Counterparty::getType)).isPresent() ? item.getDocument().flatMap(d -> d.getCounterparty().map(Counterparty::getType)).map(Object::toString).orElse(null) : null,
                 item.getDocument().flatMap(document -> document.getCounterparty().flatMap(Counterparty::getName)).orElse(null),
-                item.getRejection().map(Rejection::getRejectionReason).orElse(null)
+                item.getRejection().map(Rejection::getRejectionReason).orElse(null),
+                organisationPublicApi.findCostCenter(item.getTransaction().getOrganisation().getId(),item.getCostCenter().map(CostCenter::getCustomerCode).orElse(null)).map(OrganisationCostCenter::getParentCustomerCode).orElse(null),
+                organisationPublicApi.findProject(item.getTransaction().getOrganisation().getId(),item.getProject().map(Project::getCustomerCode).orElse(null)).map(OrganisationProject::getParentCustomerCode).orElse(null)
         );
     }
 }

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/resource/views/ExtractionTransactionItemView.java

@@ -86,5 +86,8 @@ public class ExtractionTransactionItemView {
 
     private RejectionReason rejectionReason;
 
+    private String parentCostCenterCustomerCode;
+
+    private String parentProjectCustomerCode;
 
 }

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/service/business_rules/items/DocumentConversionTaskItem.java

@@ -1,6 +1,7 @@
 package org.cardanofoundation.lob.app.accounting_reporting_core.service.business_rules.items;
 
 import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.Source.ERP;
 import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.Source.LOB;
 import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.TransactionViolationCode.*;
 import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.Violation.Severity.ERROR;
@@ -76,6 +77,15 @@ private Optional<CoreCurrency> enrichCurrency(String organisationId,
         val customerCurrencyCode = document.getCurrency().getCustomerCode();
 
         if (isBlank(customerCurrencyCode)) {
+            tx.addViolation(TransactionViolation.builder()
+                    .txItemId(txItem.getId())
+                    .code(CURRENCY_DATA_NOT_FOUND)
+                    .severity(ERROR)
+                    .source(ERP)
+                    .processorModule(getClass().getSimpleName())
+                    .bag(Map.of("customerCode", customerCurrencyCode, "transactionNumber", tx.getTransactionInternalNumber()))
+                    .build());
+
             return Optional.empty();
         }
 

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/service/business_rules/items/JournalAccountCreditEnrichmentTaskItem.java

@@ -1,6 +1,7 @@
 package org.cardanofoundation.lob.app.accounting_reporting_core.service.business_rules.items;
 
 import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.OperationType.CREDIT;
+import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.OperationType.DEBIT;
 import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.Source.ERP;
 import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.Source.LOB;
 import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.TransactionType.Journal;
@@ -69,6 +70,8 @@ public void run(TransactionEntity tx) {
                 }
                 Account accountDebit = txItem.getAccountDebit().orElseThrow();
                 txItem.setAccountCredit(Optional.of(accountDebit));
+                // If we switch the account credit, we need to set the operation type to DEBIT
+                txItem.setOperationType(DEBIT);
 
                 txItem.clearAccountCodeDebit();
             }

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/service/internal/AccountingCoreEventHandler.java

@@ -1,6 +1,7 @@
 package org.cardanofoundation.lob.app.accounting_reporting_core.service.internal;
 
 import java.time.LocalDate;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -17,8 +18,8 @@
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.FatalError;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.ReportStatusUpdate;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.Transaction;
-import org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.TxStatusUpdate;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.entity.TransactionEntity;
+import org.cardanofoundation.lob.app.accounting_reporting_core.domain.entity.report.ReportEntity;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.event.extraction.TransactionBatchChunkEvent;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.event.extraction.TransactionBatchFailedEvent;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.event.extraction.TransactionBatchStartedEvent;
@@ -28,6 +29,7 @@
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.event.reconcilation.ReconcilationFailedEvent;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.event.reconcilation.ReconcilationFinalisationEvent;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.event.reconcilation.ReconcilationStartedEvent;
+import org.cardanofoundation.lob.app.accounting_reporting_core.job.TxStatusUpdaterJob;
 import org.cardanofoundation.lob.app.accounting_reporting_core.service.business_rules.ProcessorFlags;
 import org.cardanofoundation.lob.app.support.modulith.EventMetadata;
 
@@ -43,17 +45,15 @@ public class AccountingCoreEventHandler {
     private final TransactionBatchService transactionBatchService;
     private final TransactionReconcilationService transactionReconcilationService;
     private final ApplicationEventPublisher applicationEventPublisher;
+    private final TxStatusUpdaterJob txStatusUpdaterJob;
 
 
     @EventListener
     @Async
     public void handleLedgerUpdatedEvent(TxsLedgerUpdatedEvent event) {
         log.info("Received handleLedgerUpdatedEvent event, event: {}", event.getStatusUpdates());
 
-        Map<String, TxStatusUpdate> txStatusUpdatesMap = event.statusUpdatesMap();
-
-        ledgerService.updateTransactionsWithNewStatuses(txStatusUpdatesMap);
-        transactionBatchService.updateBatchesPerTransactions(txStatusUpdatesMap);
+        txStatusUpdaterJob.addToStatusUpdateMap(event.statusUpdatesMap());
 
         log.info("Finished processing handleLedgerUpdatedEvent event, event: {}", event.getStatusUpdates());
     }
@@ -65,7 +65,8 @@ public void handleReportsLedgerUpdated(ReportsLedgerUpdatedEvent event) {
 
         Map<String, ReportStatusUpdate> reportStatusUpdatesMap = event.statusUpdatesMap();
 
-        ledgerService.updateReportsWithNewStatuses(reportStatusUpdatesMap);
+        List<ReportEntity> reportEntities = ledgerService.updateReportsWithNewStatuses(reportStatusUpdatesMap);
+        ledgerService.saveAllReports(reportEntities);
 
         log.info("Finished processing handleReportsLedgerUpdated, event: {}", event);
     }

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/service/internal/AccountingCoreService.java

@@ -53,7 +53,7 @@ public class AccountingCoreService {
     @Value("${lob.max.transaction.numbers.per.batch:600}")
     private int maxTransactionNumbersPerBatch = 600;
 
-    @Transactional
+    @Transactional(readOnly = true)
     public Either<Problem, Void> scheduleIngestion(UserExtractionParameters userExtractionParameters) {
         log.info("scheduleIngestion, parameters: {}", userExtractionParameters);
 
@@ -69,7 +69,7 @@ public Either<Problem, Void> scheduleIngestion(UserExtractionParameters userExtr
         if (userExtractionParameters.getTransactionNumbers().size() > maxTransactionNumbersPerBatch) {
             return Either.left(Problem.builder()
                     .withTitle("TOO_MANY_TRANSACTIONS")
-                    .withDetail(STR."Too many transactions requested, maximum is \{maxTransactionNumbersPerBatch}")
+                    .withDetail("Too many transactions requested, maximum is %s".formatted(maxTransactionNumbersPerBatch))
                     .withStatus(BAD_REQUEST)
                     .build());
         }
@@ -84,7 +84,7 @@ public Either<Problem, Void> scheduleIngestion(UserExtractionParameters userExtr
         return Either.right(null); // all fine
     }
 
-    @Transactional
+    @Transactional(readOnly = true)
     public Either<Problem, Void> scheduleReconcilation(String organisationId,
                                                        LocalDate fromDate,
                                                        LocalDate toDate) {

accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/service/internal/LedgerService.java

@@ -1,7 +1,7 @@
 package org.cardanofoundation.lob.app.accounting_reporting_core.service.internal;
 
-import static org.springframework.transaction.annotation.Propagation.SUPPORTS;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -50,8 +50,8 @@ public class LedgerService {
     @Value("${ledger.dispatch.batch.size:100}")
     private int dispatchBatchSize;
 
-    @Transactional
-    public void updateTransactionsWithNewStatuses(Map<String, TxStatusUpdate> statuses) {
+    @Transactional(readOnly = true)
+    public List<TransactionEntity> updateTransactionsWithNewStatuses(Map<String, TxStatusUpdate> statuses) {
         log.info("Updating dispatch status for statusMapCount: {}", statuses.size());
 
         Set<String> txIds = statuses.keySet();
@@ -72,14 +72,17 @@ public void updateTransactionsWithNewStatuses(Map<String, TxStatusUpdate> status
                 tx.setLedgerDispatchReceipt(new LedgerDispatchReceipt(type, hash));
             }
         }
+        return transactionEntities;
 
-        accountingCoreTransactionRepository.saveAll(transactionEntities);
-
-        log.info("Updated dispatch status for statusMapCount: {} completed.", statuses.size());
     }
 
     @Transactional
-    public void updateReportsWithNewStatuses(Map<String, ReportStatusUpdate> reportStatusUpdateMap) {
+    public void saveAllTransactionEntities(Collection<TransactionEntity> transactionEntities) {
+        accountingCoreTransactionRepository.saveAll(transactionEntities);
+    }
+
+    @Transactional(readOnly = true)
+    public List<ReportEntity> updateReportsWithNewStatuses(Map<String, ReportStatusUpdate> reportStatusUpdateMap) {
         log.info("Updating dispatch status for statusMapCount: {}", reportStatusUpdateMap.size());
 
         Set<String> reportIds = reportStatusUpdateMap.keySet();
@@ -101,12 +104,15 @@ public void updateReportsWithNewStatuses(Map<String, ReportStatusUpdate> reportS
             }
         }
 
-        reportRepository.saveAll(reports);
-
-        log.info("Updated dispatch status for statusMapCount: {} completed.", reportStatusUpdateMap.size());
+        return reports;
     }
 
     @Transactional
+    public void saveAllReports(Collection<ReportEntity> reports) {
+        reportRepository.saveAll(reports);
+    }
+
+    @Transactional(readOnly = true)
     public void dispatchPending(int limit) {
         for (Organisation organisation : organisationPublicApi.listAll()) {
             Set<TransactionEntity> dispatchTransactions = accountingCoreTransactionRepository.findDispatchableTransactions(organisation.getId(), Limit.of(limit));
@@ -119,7 +125,7 @@ public void dispatchPending(int limit) {
         }
     }
 
-    @Transactional(propagation = SUPPORTS)
+    @Transactional(readOnly = true)
     public void dispatchPendingTransactions(String organisationId,
                                             Set<TransactionEntity> transactions) {
         log.info("dispatchTransactionToBlockchainPublisher, total tx count: {}", transactions.size());
@@ -145,7 +151,6 @@ public void dispatchPendingTransactions(String organisationId,
         }
     }
 
-    @Transactional(propagation = SUPPORTS)
     public void dispatchReports(String organisationId,
                                 Set<ReportEntity> reportEntities) {
         log.info("dispatchReports, total reports count: {}", reportEntities.size());