Merge pull request #1543 from ton-blockchain/testnet

Merge developer branch

Author: EmelyanenkoK

.github/workflows/docker-ubuntu-branch-image.yml

@@ -20,10 +20,12 @@ jobs:
           submodules: 'recursive'
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v3.5.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/[email protected]
+        with:
+          driver-opts: image=moby/buildkit:v0.11.0
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
@@ -32,6 +34,17 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          load: true
+          context: ./
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test
+
+      - name: Test
+        run: |
+          docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test
+
       - name: Get tag as branch name
         id: tag
         run: |

.github/workflows/docker-ubuntu-image.yml

@@ -20,10 +20,10 @@ jobs:
           submodules: 'recursive'
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        
+        uses: docker/setup-qemu-action@v3.5.0
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v3.10.0
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3

Changelog.md

@@ -1,3 +1,17 @@
+## 2025.03 Update
+1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10)
+2. Optmization of validation process, in particular CellStorageStat.
+3. Flag for speeding up broadcasts in various overlays.
+4. Fixes for static builds for emulator and tonlibjson
+5. Improving getstats output: adds
+  * Liteserver queries count
+  * Collated/validated blocks count, number of active sessions
+  * Persistent state sizes
+  * Initial sync progress
+6. Fixes in logging, TON Storage, external message checking, persistent state downloading, UB in tonlib
+
+Besides the work of the core team, this update is based on the efforts of  @Sild from StonFi(UB in tonlib).
+
 ## 2025.02 Update
 1. Series of improvement/fixes for `Config8.version >= 9`, check [GlobalVersions.md](./doc/GlobalVersions.md)
 2. Fix for better discovery of updated nodes' (validators') IPs: retry dht queries

Dockerfile

@@ -1,6 +1,13 @@
 FROM ubuntu:22.04 AS builder
+ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
-        DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev lsb-release software-properties-common gnupg
+        rm /var/lib/dpkg/info/libc-bin.* && \
+        apt-get clean && \
+        apt-get update && \
+        apt install libc-bin && \
+        apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git \
+        ninja-build libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool \
+        libjemalloc-dev lsb-release software-properties-common gnupg
 
 RUN wget https://apt.llvm.org/llvm.sh && \
     chmod +x llvm.sh && \
@@ -25,6 +32,7 @@ RUN mkdir build && \
     blockchain-explorer emulator tonlibjson http-proxy adnl-proxy
 
 FROM ubuntu:22.04
+ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
     apt-get install -y wget curl libatomic1 openssl libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop \
     net-tools netcat iptraf-ng jq tcpdump pv plzip && \

catchain/catchain-receiver.cpp

@@ -526,10 +526,12 @@ void CatChainReceiverImpl::start_up() {
   for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
     root_keys.emplace(get_source(i)->get_hash(), OVERLAY_MAX_ALLOWED_PACKET_SIZE);
   }
-  td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay,
+  overlay::OverlayOptions overlay_options;
+  overlay_options.broadcast_speed_multiplier_ = opts_.broadcast_speed_multiplier;
+  td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay_ex,
                           get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids),
                           make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)},
-                          R"({ "type": "catchain" })");
+                          R"({ "type": "catchain" })", std::move(overlay_options));
 
   CHECK(root_block_);
 

common/global-version.h

@@ -19,6 +19,6 @@
 namespace ton {
 
 // See doc/GlobalVersions.md
-const int SUPPORTED_VERSION = 9;
+constexpr int SUPPORTED_VERSION = 10;
 
 }

crypto/block/block.cpp

@@ -360,7 +360,6 @@ MsgProcessedUptoCollection::MsgProcessedUptoCollection(ton::ShardIdFull _owner,
     z.shard = key.get_uint(64);
     z.mc_seqno = (unsigned)((key + 64).get_uint(32));
     z.last_inmsg_lt = value.write().fetch_ulong(64);
-    // std::cerr << "ProcessedUpto shard " << std::hex << z.shard << std::dec << std::endl;
     return value.write().fetch_bits_to(z.last_inmsg_hash) && z.shard && ton::shard_contains(owner.shard, z.shard);
   });
 }
@@ -862,8 +861,10 @@ td::Status ShardState::unpack_out_msg_queue_info(Ref<vm::Cell> out_msg_queue_inf
   out_msg_queue_ =
       std::make_unique<vm::AugmentedDictionary>(std::move(qinfo.out_queue), 352, block::tlb::aug_OutMsgQueue);
   if (verbosity >= 3 * 1) {
-    LOG(DEBUG) << "unpacking ProcessedUpto of our previous block " << id_.to_str();
-    block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info);
+    FLOG(DEBUG) {
+      sb << "unpacking ProcessedUpto of our previous block " << id_.to_str();
+      block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info);
+    };
   }
   if (!block::gen::t_ProcessedInfo.validate_csr(1024, qinfo.proc_info)) {
     return td::Status::Error(
@@ -1349,6 +1350,35 @@ bool CurrencyCollection::clamp(const CurrencyCollection& other) {
   return ok || invalidate();
 }
 
+bool CurrencyCollection::check_extra_currency_limit(td::uint32 max_currencies) const {
+  td::uint32 count = 0;
+  return vm::Dictionary{extra, 32}.check_for_each([&](td::Ref<vm::CellSlice>, td::ConstBitPtr, int) {
+    ++count;
+    return count <= max_currencies;
+  });
+}
+
+bool CurrencyCollection::remove_zero_extra_currencies(Ref<vm::Cell>& root, td::uint32 max_currencies) {
+  td::uint32 count = 0;
+  vm::Dictionary dict{root, 32};
+  int res = dict.filter([&](const vm::CellSlice& cs, td::ConstBitPtr, int) -> int {
+    ++count;
+    if (count > max_currencies) {
+      return -1;
+    }
+    td::RefInt256 val = tlb::t_VarUInteger_32.as_integer(cs);
+    if (val.is_null()) {
+      return -1;
+    }
+    return val->sgn() > 0;
+  });
+  if (res < 0) {
+    return false;
+  }
+  root = dict.get_root_cell();
+  return true;
+}
+
 bool CurrencyCollection::operator==(const CurrencyCollection& other) const {
   return is_valid() && other.is_valid() && !td::cmp(grams, other.grams) &&
          (extra.not_null() == other.extra.not_null()) &&

crypto/block/block.h

@@ -391,6 +391,8 @@ struct CurrencyCollection {
   CurrencyCollection operator-(CurrencyCollection&& other) const;
   CurrencyCollection operator-(td::RefInt256 other_grams) const;
   bool clamp(const CurrencyCollection& other);
+  bool check_extra_currency_limit(td::uint32 max_currencies) const;
+  static bool remove_zero_extra_currencies(Ref<vm::Cell>& root, td::uint32 max_currencies);
   bool store(vm::CellBuilder& cb) const;
   bool store_or_zero(vm::CellBuilder& cb) const;
   bool fetch(vm::CellSlice& cs);

crypto/block/block.tlb

@@ -801,7 +801,7 @@ size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells
   max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig;
 size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16
   max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32
-  max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 = SizeLimitsConfig;
+  max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 = SizeLimitsConfig;
 _ SizeLimitsConfig = ConfigParam 43;
 
 // key is [ wc:int32 addr:uint256 ]

crypto/block/mc-config.cpp

@@ -163,8 +163,11 @@ td::Status ConfigInfo::unpack() {
   }
   gen::McStateExtra::Record extra_info;
   if (!tlb::unpack_cell(state_extra_root_, extra_info)) {
-    vm::load_cell_slice(state_extra_root_).print_rec(std::cerr);
-    block::gen::t_McStateExtra.print_ref(std::cerr, state_extra_root_);
+    FLOG(WARNING) {
+      sb << "state extra information is invalid: ";
+      vm::load_cell_slice(state_extra_root_).print_rec(sb);
+      block::gen::t_McStateExtra.print_ref(sb, state_extra_root_);
+    };
     return td::Status::Error("state extra information is invalid");
   }
   gen::ValidatorInfo::Record validator_info;
@@ -1067,7 +1070,6 @@ Ref<McShardHash> ShardConfig::get_shard_hash(ton::ShardIdFull id, bool exact) co
   ton::ShardIdFull true_id;
   vm::CellSlice cs;
   if (get_shard_hash_raw(cs, id, true_id, exact)) {
-    // block::gen::t_ShardDescr.print(std::cerr, vm::CellSlice{cs});
     return McShardHash::unpack(cs, true_id);
   } else {
     return {};
@@ -1637,8 +1639,10 @@ bool ShardConfig::set_shard_info(ton::ShardIdFull shard, Ref<vm::Cell> value) {
   if (!gen::t_BinTree_ShardDescr.validate_ref(1024, value)) {
     LOG(ERROR) << "attempting to store an invalid (BinTree ShardDescr) at shard configuration position "
                << shard.to_str();
-    gen::t_BinTree_ShardDescr.print_ref(std::cerr, value);
-    vm::load_cell_slice(value).print_rec(std::cerr);
+    FLOG(WARNING) {
+      gen::t_BinTree_ShardDescr.print_ref(sb, value);
+      vm::load_cell_slice(value).print_rec(sb);
+    };
     return false;
   }
   auto root = shard_hashes_dict_->lookup_ref(td::BitArray<32>{shard.workchain});
@@ -1956,6 +1960,7 @@ td::Result<SizeLimitsConfig> Config::do_get_size_limits_config(td::Ref<vm::CellS
     limits.max_acc_state_cells = rec.max_acc_state_cells;
     limits.max_acc_public_libraries = rec.max_acc_public_libraries;
     limits.defer_out_queue_size_limit = rec.defer_out_queue_size_limit;
+    limits.max_msg_extra_currencies = rec.max_msg_extra_currencies;
   };
   gen::SizeLimitsConfig::Record_size_limits_config rec_v1;
   gen::SizeLimitsConfig::Record_size_limits_config_v2 rec_v2;

crypto/block/mc-config.h

@@ -397,6 +397,7 @@ struct SizeLimitsConfig {
   td::uint32 max_acc_state_bits = (1 << 16) * 1023;
   td::uint32 max_acc_public_libraries = 256;
   td::uint32 defer_out_queue_size_limit = 256;
+  td::uint32 max_msg_extra_currencies = 2;
 };
 
 struct CatchainValidatorsConfig {

crypto/block/output-queue-merger.cpp

@@ -138,7 +138,6 @@ bool OutputQueueMerger::add_root(int src, Ref<vm::Cell> outmsg_root) {
   if (outmsg_root.is_null()) {
     return true;
   }
-  //block::gen::HashmapAug{352, block::gen::t_EnqueuedMsg, block::gen::t_uint64}.print_ref(std::cerr, outmsg_root);
   auto kv = std::make_unique<MsgKeyValue>(src, std::move(outmsg_root));
   if (kv->replace_by_prefix(common_pfx.cbits(), common_pfx_len)) {
     heap.push_back(std::move(kv));