FeatureRNG [DO NOT MERGE]

Builds/CMake/RippledCore.cmake

@@ -461,6 +461,7 @@ target_sources (rippled PRIVATE
   src/ripple/app/tx/impl/CronSet.cpp
   src/ripple/app/tx/impl/DeleteAccount.cpp
   src/ripple/app/tx/impl/DepositPreauth.cpp
+  src/ripple/app/tx/impl/Entropy.cpp
   src/ripple/app/tx/impl/Escrow.cpp
   src/ripple/app/tx/impl/GenesisMint.cpp
   src/ripple/app/tx/impl/Import.cpp

src/ripple/app/consensus/RCLConsensus.cpp

@@ -35,6 +35,8 @@
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/app/misc/ValidatorKeys.h>
 #include <ripple/app/misc/ValidatorList.h>
+#include <ripple/app/tx/impl/Change.h>
+#include <ripple/app/tx/impl/Entropy.h>
 #include <ripple/basics/random.h>
 #include <ripple/beast/core/LexicalCast.h>
 #include <ripple/consensus/LedgerTiming.h>
@@ -225,6 +227,8 @@ RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal)
 
     prop.set_signature(sig.data(), sig.size());
 
+    injectShuffleTxn(app_, sig);
+
     auto const suppression = proposalUniqueId(
         proposal.position(),
         proposal.prevLedger(),
@@ -652,6 +656,12 @@ RCLConsensus::Adaptor::doAccept(
             tapNONE,
             "consensus",
             [&](OpenView& view, beast::Journal j) {
+                if (rules->enabled(featureRNG))
+                {
+                    auto tx = makeEntropyTxn(view, app_, j_);
+                    if (tx)
+                        app_.getOPs().submitTransaction(tx);
+                }
                 return app_.getTxQ().accept(app_, view);
             });
 

src/ripple/app/hook/Enum.h

@@ -350,7 +350,8 @@ enum hook_return_code : int64_t {
     MEM_OVERLAP = -43,   // one or more specified buffers are the same memory
     TOO_MANY_STATE_MODIFICATIONS = -44,  // more than 5000 modified state
                                          // entires in the combined hook chains
-    TOO_MANY_NAMESPACES = -45
+    TOO_MANY_NAMESPACES = -45,
+    TOO_LITTLE_ENTROPY = -46,
 };
 
 enum ExitType : uint8_t {
@@ -459,6 +460,8 @@ static const APIWhitelist import_whitelist{
     HOOK_API_DEFINITION(I64, otxn_slot, (I32)),
     HOOK_API_DEFINITION(I64, otxn_param, (I32, I32, I32, I32)),
     HOOK_API_DEFINITION(I64, meta_slot, (I32)),
+    HOOK_API_DEFINITION(I64, dice, (I32)),
+    HOOK_API_DEFINITION(I64, random, (I32, I32)),
     // clang-format on
 };
 

src/ripple/app/hook/applyHook.h

@@ -406,6 +406,9 @@ DECLARE_HOOK_FUNCTION(
     uint32_t slot_no_tx,
     uint32_t slot_no_meta);
 
+DECLARE_HOOK_FUNCTION(int64_t, dice, uint32_t sides);
+DECLARE_HOOK_FUNCTION(int64_t, random, uint32_t write_ptr, uint32_t write_len);
+
 /*
     DECLARE_HOOK_FUNCTION(int64_t,  str_find,           uint32_t hread_ptr,
    uint32_t hread_len, uint32_t nread_ptr, uint32_t nread_len, uint32_t mode,
@@ -513,6 +516,8 @@ struct HookResult
         false;  // hook_again allows strong pre-apply to nominate
                 // additional weak post-apply execution
     std::shared_ptr<STObject const> provisionalMeta;
+    uint64_t rngCallCounter{
+        0};  // used to ensure conseq. rng calls don't return same data
 };
 
 class HookExecutor;
@@ -877,6 +882,9 @@ class HookExecutor
         ADD_HOOK_FUNCTION(meta_slot, ctx);
         ADD_HOOK_FUNCTION(xpop_slot, ctx);
 
+        ADD_HOOK_FUNCTION(dice, ctx);
+        ADD_HOOK_FUNCTION(random, ctx);
+
         /*
         ADD_HOOK_FUNCTION(str_find, ctx);
         ADD_HOOK_FUNCTION(str_replace, ctx);

src/ripple/app/hook/impl/applyHook.cpp

@@ -6156,6 +6156,117 @@ DEFINE_HOOK_FUNCTION(
 
     HOOK_TEARDOWN();
 }
+
+// byteCount must be a multiple of 32
+inline std::vector<uint8_t>
+fairRng(ApplyContext& applyCtx, hook::HookResult& hr, uint32_t byteCount)
+{
+    if (byteCount > 512)
+        byteCount = 512;
+
+    // force the byte count to be a multiple of 32
+    byteCount &= ~0b11111;
+
+    if (byteCount == 0)
+        return {};
+
+    auto& view = applyCtx.view();
+
+    auto const sleRandom = view.peek(ripple::keylet::random());
+    auto const seq = view.info().seq;
+
+    if (!sleRandom || sleRandom->getFieldU32(sfLedgerSequence) != seq ||
+        sleRandom->getFieldU16(sfEntropyCount) < 5)
+        return {};
+
+    // we'll generate bytes in lots of 32
+
+    uint256 rndData = sha512Half(
+        view.info().seq,
+        applyCtx.tx.getTransactionID(),
+        hr.otxnAccount,
+        hr.hookHash,
+        hr.account,
+        hr.hookChainPosition,
+        hr.executeAgainAsWeak ? std::string("weak") : std::string("strong"),
+        sleRandom->getFieldH256(sfRandomData),
+        hr.rngCallCounter++);
+
+    std::vector<uint8_t> bytesOut;
+    bytesOut.resize(byteCount);
+
+    uint8_t* ptr = bytesOut.data();
+    while (1)
+    {
+        std::memcpy(ptr, rndData.data(), 32);
+        ptr += 32;
+
+        if (ptr - bytesOut.data() >= byteCount)
+            break;
+
+        rndData = sha512Half(rndData);
+    }
+
+    return bytesOut;
+}
+
+DEFINE_HOOK_FUNCTION(int64_t, dice, uint32_t sides)
+{
+    HOOK_SETUP();
+
+    auto vec = fairRng(applyCtx, hookCtx.result, 32);
+
+    if (vec.empty())
+        return TOO_LITTLE_ENTROPY;
+
+    if (vec.size() != 32)
+        return INTERNAL_ERROR;
+
+    uint32_t value;
+    std::memcpy(&value, vec.data(), sizeof(uint32_t));
+
+    return value % sides;
+
+    HOOK_TEARDOWN();
+}
+
+DEFINE_HOOK_FUNCTION(int64_t, random, uint32_t write_ptr, uint32_t write_len)
+{
+    HOOK_SETUP();
+
+    if (write_len == 0)
+        return TOO_SMALL;
+
+    if (write_len > 512)
+        return TOO_BIG;
+
+    uint32_t required = write_len;
+
+    if (required & ~0b11111 == required)
+    {
+        // already a multiple of 32 bytes
+    }
+    else
+    {
+        // round up
+        required &= ~0b11111;
+        required += 32;
+    }
+
+    if (NOT_IN_BOUNDS(write_ptr, write_len, memory_length))
+        return OUT_OF_BOUNDS;
+
+    auto vec = fairRng(applyCtx, hookCtx.result, required);
+
+    if (vec.empty())
+        return TOO_LITTLE_ENTROPY;
+
+    WRITE_WASM_MEMORY_AND_RETURN(
+        write_ptr, write_len, vec.data(), vec.size(), memory, memory_length);
+
+    HOOK_TEARDOWN();
+}
+
 /*
 
 DEFINE_HOOK_FUNCTION(

src/ripple/app/ledger/impl/BuildLedger.cpp

@@ -25,6 +25,7 @@
 #include <ripple/app/misc/CanonicalTXSet.h>
 #include <ripple/app/tx/apply.h>
 #include <ripple/protocol/Feature.h>
+#include <ripple/protocol/digest.h>
 
 namespace ripple {
 
@@ -103,6 +104,50 @@ applyTransactions(
     bool certainRetry = true;
     std::size_t count = 0;
 
+    // apply the ttSHUFFLE txns first in the ledger to
+    // ensure no one can predict the outcome
+    // then apply ttENTROPY transactions
+    if (view.rules().enabled(featureRNG))
+        for (auto tt : {ttSHUFFLE, ttENTROPY})
+        {
+            for (auto it = txns.begin(); it != txns.end();)
+            {
+                if (tt != it->second->getFieldU16(sfTransactionType))
+                {
+                    ++it;
+                    continue;
+                }
+
+                auto const txid = it->first.getTXID();
+                try
+                {
+                    switch (applyTransaction(
+                        app, view, *it->second, certainRetry, tapNONE, j))
+                    {
+                        case ApplyResult::Success:
+                            it = txns.erase(it);
+                            ++count;
+                            break;
+
+                        case ApplyResult::Fail:
+                            failed.insert(txid);
+                            it = txns.erase(it);
+                            break;
+
+                        case ApplyResult::Retry:
+                            ++it;
+                    }
+                }
+                catch (std::exception const& ex)
+                {
+                    JLOG(j.warn())
+                        << "Transaction " << txid << " throws: " << ex.what();
+                    failed.insert(txid);
+                    it = txns.erase(it);
+                }
+            }
+        }
+
     // Attempt to apply all of the retriable transactions
     for (int pass = 0; pass < LEDGER_TOTAL_PASSES; ++pass)
     {

src/ripple/app/main/Application.cpp

@@ -599,6 +599,12 @@ class ApplicationImp : public Application, public BasicApp
         return validatorKeys_.publicKey;
     }
 
+    SecretKey const&
+    getValidationSecretKey() const override
+    {
+        return validatorKeys_.secretKey;
+    }
+
     NetworkOPs&
     getOPs() override
     {

src/ripple/app/main/Application.h

@@ -241,6 +241,9 @@ class Application : public beast::PropertyStream::Source
     virtual PublicKey const&
     getValidationPublicKey() const = 0;
 
+    virtual SecretKey const&
+    getValidationSecretKey() const = 0;
+
     virtual Resource::Manager&
     getResourceManager() = 0;
     virtual PathRequests&

src/ripple/app/misc/NetworkOPs.cpp

@@ -228,7 +228,7 @@ class NetworkOPsImp final : public NetworkOPs
     doTransactionSync(
         std::shared_ptr<Transaction> transaction,
         bool bUnlimited,
-        FailHard failType);
+        FailHard failType) override;
 
     /**
      * For transactions not submitted by a locally connected client, fire and
@@ -1078,6 +1078,12 @@ NetworkOPsImp::submitTransaction(std::shared_ptr<STTx const> const& iTrans)
         return;
     }
 
+    if (view->rules().enabled(featureRNG) && iTrans->getTxnType() == ttSHUFFLE)
+    {
+        // as above
+        return;
+    }
+
     // this is an asynchronous interface
     auto const trans = sterilize(*iTrans);
 

src/ripple/app/misc/NetworkOPs.h

@@ -112,6 +112,13 @@ class NetworkOPs : public InfoSub::Source
         bool bLocal,
         FailHard failType) = 0;
 
+    // directly inject transaction, skipping checks
+    virtual void
+    doTransactionSync(
+        std::shared_ptr<Transaction> transaction,
+        bool bUnlimited,
+        FailHard failType) = 0;
+
     //--------------------------------------------------------------------------
     //
     // Owner functions

src/ripple/app/misc/impl/TxQ.cpp

@@ -25,6 +25,7 @@
 #include <ripple/app/tx/apply.h>
 #include <ripple/basics/mulDiv.h>
 #include <ripple/protocol/Feature.h>
+#include <ripple/protocol/digest.h>
 #include <ripple/protocol/jss.h>
 #include <ripple/protocol/st.h>
 #include <algorithm>
@@ -1930,13 +1931,15 @@ TxQ::tryDirectApply(
     const bool isFirstImport = !sleAccount &&
         view.rules().enabled(featureImport) && tx->getTxnType() == ttIMPORT;
 
+    const bool accRequired = !(isFirstImport || isUVTx(*tx));
+
     // Don't attempt to direct apply if the account is not in the ledger.
-    if (!sleAccount && !isFirstImport)
+    if (!sleAccount && accRequired)
         return {};
 
     std::optional<SeqProxy> txSeqProx;
 
-    if (!isFirstImport)
+    if (accRequired)
     {
         SeqProxy const acctSeqProx =
             SeqProxy::sequence((*sleAccount)[sfSequence]);
@@ -1949,7 +1952,7 @@ TxQ::tryDirectApply(
     }
 
     FeeLevel64 const requiredFeeLevel =
-        isFirstImport ? FeeLevel64{0} : [this, &view, flags]() {
+        !accRequired ? FeeLevel64{0} : [this, &view, flags]() {
             std::lock_guard lock(mutex_);
             return getRequiredFeeLevel(
                 view, flags, feeMetrics_.getSnapshot(), lock);

src/ripple/app/rdb/backend/detail/impl/Node.cpp

@@ -319,7 +319,7 @@ saveValidatedLedger(
                     *db << sql;
                 }
                 else if (auto const& sleTxn = acceptedLedgerTx->getTxn();
-                         !isPseudoTx(*sleTxn))
+                         !isPseudoTx(*sleTxn) && !isUVTx(*sleTxn))
                 {
                     // It's okay for pseudo transactions to not affect any
                     // accounts.  But otherwise...