From 17746156769d862acd24875cfe86c35c012e11c8 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 2 Jan 2020 17:59:52 +0200 Subject: [PATCH 1/4] Only add tails as for solid entry points --- .../snapshot/impl/SnapshotServiceImpl.java | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java index 332c23546e..1887999a83 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java @@ -575,7 +575,8 @@ private boolean isOrphaned(Tangle tangle, TransactionViewModel transaction, * @param targetMilestone milestone that is used as an anchor for our checks * @return true if the transaction is a solid entry point and false otherwise */ - private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, MilestoneViewModel targetMilestone) { + private boolean shouldTailsBeSolidEntryPoint(Tangle tangle, Hash transactionHash, + MilestoneViewModel targetMilestone) { Set unconfirmedApprovers = new HashSet<>(); try { @@ -620,24 +621,33 @@ private boolean isSolidEntryPoint(Tangle tangle, Hash transactionHash, Milestone * @param solidEntryPoints map that is used to collect the solid entry points */ private void processOldSolidEntryPoints(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneViewModel targetMilestone, Map solidEntryPoints) { + MilestoneViewModel targetMilestone, Map solidEntryPoints) throws SnapshotException { ProgressLogger progressLogger = new IntervalProgressLogger( "Taking local snapshot [analyzing old solid entry points]", log) .start(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().size()); + try { + Snapshot initialSnapshot = snapshotProvider.getInitialSnapshot(); + Map orgSolidEntryPoints = initialSnapshot.getSolidEntryPoints(); + for (Map.Entry solidPoint : orgSolidEntryPoints.entrySet()) { + Hash hash = solidPoint.getKey(); + int milestoneIndex = solidPoint.getValue(); + if (!Hash.NULL_HASH.equals(hash) + && targetMilestone.index() - milestoneIndex <= SOLID_ENTRY_POINT_LIFETIME + && shouldTailsBeSolidEntryPoint(tangle, hash, targetMilestone)) { + TransactionViewModel tvm = TransactionViewModel.fromHash(tangle, hash); + addTailsToSolidEntryPoints(milestoneIndex, solidEntryPoints, tvm); + solidEntryPoints.put(hash, milestoneIndex); + } - Snapshot initialSnapshot = snapshotProvider.getInitialSnapshot(); - initialSnapshot.getSolidEntryPoints().forEach((hash, milestoneIndex) -> { - if (!Hash.NULL_HASH.equals(hash) && targetMilestone.index() - milestoneIndex <= SOLID_ENTRY_POINT_LIFETIME - && isSolidEntryPoint(tangle, hash, targetMilestone)) { - - solidEntryPoints.put(hash, milestoneIndex); + progressLogger.progress(); } - - progressLogger.progress(); - }); - - progressLogger.finish(); + } catch (Exception e) { + throw new SnapshotException( + "Couldn't process old solid entry point for target milestone " + targetMilestone.index(), e); + } finally { + progressLogger.finish(); + } } /** @@ -675,8 +685,9 @@ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot currentMilestone.getHash(), currentTransaction -> currentTransaction.snapshotIndex() >= currentMilestone.index(), currentTransaction -> { - if (isSolidEntryPoint(tangle, currentTransaction.getHash(), targetMilestone)) { - solidEntryPoints.put(currentTransaction.getHash(), targetMilestone.index()); + if (shouldTailsBeSolidEntryPoint(tangle, currentTransaction.getHash(), targetMilestone)) { + addTailsToSolidEntryPoints(targetMilestone.index(), solidEntryPoints, + currentTransaction); } } ); @@ -696,4 +707,15 @@ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot throw new SnapshotException("could not generate the solid entry points for " + targetMilestone, e); } } + + private void addTailsToSolidEntryPoints(int milestoneIndex, Map solidEntryPoints, + TransactionViewModel currentTransaction) throws TraversalException { + // if tail + if (currentTransaction.getCurrentIndex() == 0) { + solidEntryPoints.put(currentTransaction.getHash(), milestoneIndex); + } else { + Set tails = DAGHelper.get(tangle).findTails(currentTransaction); + tails.forEach(tail -> solidEntryPoints.put(tail, milestoneIndex)); + } + } } From 0b188c22bd71237a98bb9d1a447e3884b3f81115 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 2 Jan 2020 19:42:42 +0200 Subject: [PATCH 2/4] change docs --- .../snapshot/impl/SnapshotServiceImpl.java | 61 ++++++++++--------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java index 1887999a83..cbf91fc878 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java @@ -510,7 +510,7 @@ private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot ne /** *

- * This method determines if a transaction is orphaned. + * This method determines if a transaction is orphaned when none of its approvers is confirmed by a milestone. *

*

* Since there is no hard definition for when a transaction can be considered to be orphaned, we define orphaned in @@ -522,14 +522,14 @@ private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot ne * a relatively safe way to determine if a subtangle "above" a transaction got orphaned. *

* - * @param tangle Tangle object which acts as a database interface - * @param transaction transaction that shall be checked - * @param referenceTransaction transaction that acts as a judge to the other transaction + * @param tangle Tangle object which acts as a database interface + * @param transaction transaction that shall be checked + * @param referenceTransaction transaction that acts as a judge to the other transaction * @param processedTransactions transactions that were visited already while trying to determine the orphaned status * @return true if the transaction got orphaned and false otherwise * @throws SnapshotException if anything goes wrong while determining the orphaned status */ - private boolean isOrphaned(Tangle tangle, TransactionViewModel transaction, + private boolean isProbablyOrphaned(Tangle tangle, TransactionViewModel transaction, TransactionViewModel referenceTransaction, Set processedTransactions) throws SnapshotException { AtomicBoolean nonOrphanedTransactionFound = new AtomicBoolean(false); @@ -553,30 +553,27 @@ private boolean isOrphaned(Tangle tangle, TransactionViewModel transaction, /** *

- * This method checks if a transaction is a solid entry point for the targetMilestone. - *

- *

- * A transaction is considered a solid entry point if it has non-orphaned approvers. + * We determine whether future milestones will approve {@param transactionHash}. This should aid in determining + * solid entry points. *

*

* To check if the transaction has non-orphaned approvers we first check if any of its approvers got confirmed by a * future milestone, since this is very cheap. If none of them got confirmed by another milestone we do the more - * expensive check from {@link #isOrphaned(Tangle, TransactionViewModel, TransactionViewModel, Set)}. + * expensive check from {@link #isProbablyOrphaned(Tangle, TransactionViewModel, TransactionViewModel, Set)}. *

*

* Since solid entry points have a limited life time and to prevent potential problems due to temporary errors in - * the database, we assume that the checked transaction is a solid entry point if any error occurs while determining - * its status. This is a storage <=> reliability trade off, since the only bad effect of having too many solid entry - * points) is a bigger snapshot file. + * the database, we assume that the checked transaction is not orphaned if any error occurs while determining its + * status, thus adding solid entry points. This is a storage <=> reliability trade off, since the only bad effect of + * having too many solid entry points) is a bigger snapshot file. *

* - * @param tangle Tangle object which acts as a database interface + * @param tangle Tangle object which acts as a database interface * @param transactionHash hash of the transaction that shall be checked * @param targetMilestone milestone that is used as an anchor for our checks * @return true if the transaction is a solid entry point and false otherwise */ - private boolean shouldTailsBeSolidEntryPoint(Tangle tangle, Hash transactionHash, - MilestoneViewModel targetMilestone) { + private boolean isNotOrphaned(Tangle tangle, Hash transactionHash, MilestoneViewModel targetMilestone) { Set unconfirmedApprovers = new HashSet<>(); try { @@ -593,7 +590,7 @@ private boolean shouldTailsBeSolidEntryPoint(Tangle tangle, Hash transactionHash Set processedTransactions = new HashSet<>(); TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, targetMilestone.getHash()); for (TransactionViewModel unconfirmedApprover : unconfirmedApprovers) { - if (!isOrphaned(tangle, unconfirmedApprover, milestoneTransaction, processedTransactions)) { + if (!isProbablyOrphaned(tangle, unconfirmedApprover, milestoneTransaction, processedTransactions)) { return true; } } @@ -611,13 +608,14 @@ private boolean shouldTailsBeSolidEntryPoint(Tangle tangle, Hash transactionHash * This method analyzes the old solid entry points and determines if they are still not orphaned. *

*

- * It simply iterates through the old solid entry points and checks them one by one. If an old solid entry point - * is found to still be relevant it is added to the passed in map. + * It simply iterates through the old solid entry points and checks them one by one. If an old solid entry point is + * found to still be relevant it is added to the passed in map. *

- * - * @param tangle Tangle object which acts as a database interface + * + * @see #processNewSolidEntryPoints to understand the definition for solid entry points + * @param tangle Tangle object which acts as a database interface * @param snapshotProvider data provider for the {@link Snapshot}s that are relevant for the node - * @param targetMilestone milestone that is used to generate the solid entry points + * @param targetMilestone milestone that is used to generate the solid entry points * @param solidEntryPoints map that is used to collect the solid entry points */ private void processOldSolidEntryPoints(Tangle tangle, SnapshotProvider snapshotProvider, @@ -634,7 +632,7 @@ private void processOldSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot int milestoneIndex = solidPoint.getValue(); if (!Hash.NULL_HASH.equals(hash) && targetMilestone.index() - milestoneIndex <= SOLID_ENTRY_POINT_LIFETIME - && shouldTailsBeSolidEntryPoint(tangle, hash, targetMilestone)) { + && isNotOrphaned(tangle, hash, targetMilestone)) { TransactionViewModel tvm = TransactionViewModel.fromHash(tangle, hash); addTailsToSolidEntryPoints(milestoneIndex, solidEntryPoints, tvm); solidEntryPoints.put(hash, milestoneIndex); @@ -655,16 +653,21 @@ && shouldTailsBeSolidEntryPoint(tangle, hash, targetMilestone)) { * This method retrieves the new solid entry points of the snapshot reference given by the target milestone. *

*

+ * A transaction is considered a solid entry point if it is a bundle tail that can be traversed down from a + * non-orphaned transaction that was approved by a milestone that is above the last local snapshot. Or if it is a + * bundle tail of a non-orphaned transaction that was approved by a milestone that is above the last local snapshot. + * * It iterates over all unprocessed milestones and analyzes their directly and indirectly approved transactions. - * Every transaction is checked for being a solid entry point and added to the passed in map (if it was found to be - * one). + * Every transaction is checked for being not orphaned and the appropriate SEP is added to {@param SolidEntryPoints} *

- * - * @param tangle Tangle object which acts as a database interface + * + * + * @param tangle Tangle object which acts as a database interface * @param snapshotProvider data provider for the {@link Snapshot}s that are relevant for the node - * @param targetMilestone milestone that is used to generate the solid entry points + * @param targetMilestone milestone that is used to generate the solid entry points * @param solidEntryPoints map that is used to collect the solid entry points * @throws SnapshotException if anything goes wrong while determining the solid entry points + * @see #isNotOrphaned(Tangle, Hash, MilestoneViewModel) */ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneViewModel targetMilestone, Map solidEntryPoints) throws SnapshotException { @@ -685,7 +688,7 @@ private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot currentMilestone.getHash(), currentTransaction -> currentTransaction.snapshotIndex() >= currentMilestone.index(), currentTransaction -> { - if (shouldTailsBeSolidEntryPoint(tangle, currentTransaction.getHash(), targetMilestone)) { + if (isNotOrphaned(tangle, currentTransaction.getHash(), targetMilestone)) { addTailsToSolidEntryPoints(targetMilestone.index(), solidEntryPoints, currentTransaction); } From 07983a9ada5efb1be985b6f0d134e959ab758286 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 5 Jan 2020 13:35:30 +0200 Subject: [PATCH 3/4] up version to 1.8.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 78401e9f56..bd427b0bfc 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.iota iri - 1.8.3 + 1.8.4 IRI IOTA Reference Implementation From 344b7d6e72a341dce2a760a1241d7afb9843bf3f Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 5 Jan 2020 13:41:06 +0200 Subject: [PATCH 4/4] update changelog --- changelog.txt | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/changelog.txt b/changelog.txt index 032b9a0f75..02023040ed 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,34 @@ +1.8.4 + +Hotfix: Ensure proper creation of solid entrypoints (#1702) + +1.8.3 + +There is an edge case where IRI didn't account for a transaction that was shared between two distinct bundles. +Once it marked it as "counted" in one bundle, it was ignored for the next bundle. This lead to a corrupt ledger state. + +Hotfix: Gyros - take transaction reattachments into account (#1699) + +1.8.2 + +Change: The alpha value is now 0 by default, streamlining the tip selection process which will reduce memory and cpu load. This is done by skipping cumulative weight calculations. (Issue #1567) + +Fix: correct data setting into Hash object (#1589) +Fix: Persistables merge and load functionality +Documentation: Fix broken link of online documentation (#1623) +Documentation: Update README.md (#1617) +Fix: always set the domain field of a neighbor (#1604) +Change: Use mocked SnapshotProvider (#1531) +Fix: Fixes regression introduced through the bundle validator refactor (#1588) +Documentation: Document Persistable and Indexable (#1169) +Feature: Adds progress bar and estimated time until the node is synced (#1575) +Change: Curl improvement +Change: Bundle validator refactor +Change: Use maven assembly plugin to create one jar with all dependencies (#1573) +Documentation: Document Persistence Provider (#1157) +Documentation: Updated links to official documentation (#1560) + + 1.8.1 - Feature: Improved CW Calculation (#1451)