Merge pull request #550 from shaojunda/rc/v0.9.1

[ᚬmaster] Rc/v0.9.1

Author: shaojunda

CHANGELOG.md

@@ -1,3 +1,45 @@
+# [0.9.1](https://github.com/shaojunda/ckb-explorer/compare/v0.8.4...v0.9.0) (2020-01-13)
+
+
+### Features
+
+* add block list serializer ([8d729d7](https://github.com/shaojunda/ckb-explorer/commit/8d729d7))
+* add block statistic generator service ([db91614](https://github.com/shaojunda/ckb-explorer/commit/db91614))
+* add block timestamp to dao event ([a40453b](https://github.com/shaojunda/ckb-explorer/commit/a40453b))
+* add capacity_involved column to ckb transaction ([7bcd7a5](https://github.com/shaojunda/ckb-explorer/commit/7bcd7a5))
+* add chart forked event processor ([83927bf](https://github.com/shaojunda/ckb-explorer/commit/83927bf))
+* add consumed block timestamp to cell output ([0f4986f](https://github.com/shaojunda/ckb-explorer/commit/0f4986f))
+* add epoch statistic generator service ([a99926c](https://github.com/shaojunda/ckb-explorer/commit/a99926c))
+* add estimated_apc to dao contract ([cb31d1c](https://github.com/shaojunda/ckb-explorer/commit/cb31d1c))
+* add forked event model ([2cf8bff](https://github.com/shaojunda/ckb-explorer/commit/2cf8bff))
+* add hash rate to epoch statistic ([e0746df](https://github.com/shaojunda/ckb-explorer/commit/e0746df))
+* add index action to ckb transactions controller ([e71fef7](https://github.com/shaojunda/ckb-explorer/commit/e71fef7))
+* add live_cell_changes to block and ckb_transaction ([2f04c62](https://github.com/shaojunda/ckb-explorer/commit/2f04c62))
+* add live_cell_changes to forked blocks ([a2d42b2](https://github.com/shaojunda/ckb-explorer/commit/a2d42b2))
+* add miner reward to block ([efe97fa](https://github.com/shaojunda/ckb-explorer/commit/efe97fa))
+* add more field to daily statistics ([3912139](https://github.com/shaojunda/ckb-explorer/commit/3912139))
+* add pagination to ckb transactions controller ([2251ba2](https://github.com/shaojunda/ckb-explorer/commit/2251ba2))
+* add ratio scale ([3b68742](https://github.com/shaojunda/ckb-explorer/commit/3b68742))
+* calculate estimated apc ([4a4f04f](https://github.com/shaojunda/ckb-explorer/commit/4a4f04f))
+* create forked event when forked ([ce61a1f](https://github.com/shaojunda/ckb-explorer/commit/ce61a1f))
+* implement ckb transactions index action ([26e5a36](https://github.com/shaojunda/ckb-explorer/commit/26e5a36))
+* regenerate block statistic data when block forked ([c5a444e](https://github.com/shaojunda/ckb-explorer/commit/c5a444e))
+* return hash_rate ([354ebf1](https://github.com/shaojunda/ckb-explorer/commit/354ebf1))
+* save capacity involved to ckb_transaction ([61fc5ba](https://github.com/shaojunda/ckb-explorer/commit/61fc5ba))
+* save hash rate on epoch statistic worker ([48be8e4](https://github.com/shaojunda/ckb-explorer/commit/48be8e4))
+* save live_cell_changes to block ([76f3efc](https://github.com/shaojunda/ckb-explorer/commit/76f3efc))
+* save live_cell_changes to ckb_transaction ([0c5589f](https://github.com/shaojunda/ckb-explorer/commit/0c5589f))
+* show more attributes on dao contract ([6369a86](https://github.com/shaojunda/ckb-explorer/commit/6369a86))
+* use ckb transaction list serializer ([7d12caa](https://github.com/shaojunda/ckb-explorer/commit/7d12caa))
+
+
+### Performance Improvements
+
+* add index on block timestamp, status and event type ([bce28f4](https://github.com/shaojunda/ckb-explorer/commit/bce28f4))
+* use redis pipeline and use delete replace delete_matched ([da666e0](https://github.com/shaojunda/ckb-explorer/commit/da666e0))
+
+
+
 # [0.9.0](https://github.com/shaojunda/ckb-explorer/compare/v0.8.3...v0.9.0) (2020-01-02)
 
 

app/models/address.rb

@@ -9,6 +9,8 @@ class Address < ApplicationRecord
   validates :balance, :cell_consumed, :ckb_transactions_count, :interest, :dao_deposit, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
 
   scope :visible, -> { where(visible: true) }
+  scope :created_after, ->(block_timestamp) { where("block_timestamp >= ?", block_timestamp) }
+  scope :created_before, ->(block_timestamp) { where("block_timestamp <= ?", block_timestamp) }
 
   after_commit :flush_cache
 

app/models/cell_output.rb

@@ -16,6 +16,11 @@ class CellOutput < ApplicationRecord
 
   attribute :tx_hash, :ckb_hash
 
+  scope :consumed_before, -> (block_timestamp) { where("consumed_block_timestamp <= ?", block_timestamp) }
+  scope :unconsumed_at, -> (block_timestamp) { where("consumed_block_timestamp > ? or consumed_block_timestamp is null", block_timestamp) }
+  scope :generated_after, -> (block_timestamp) { where("block_timestamp >= ?", block_timestamp) }
+  scope :generated_before, -> (block_timestamp) { where("block_timestamp <= ?", block_timestamp) }
+
   after_commit :flush_cache
 
   def address_hash

app/models/ckb_transaction.rb

@@ -18,6 +18,8 @@ class CkbTransaction < ApplicationRecord
   scope :recent, -> { order(block_timestamp: :desc) }
   scope :cellbase, -> { where(is_cellbase: true) }
   scope :normal, -> { where(is_cellbase: false) }
+  scope :created_after, -> (block_timestamp) { where("block_timestamp >= ?", block_timestamp) }
+  scope :created_before, -> (block_timestamp) { where("block_timestamp <= ?", block_timestamp) }
 
   after_commit :flush_cache
   before_destroy :recover_dead_cell

app/models/dao_event.rb

@@ -6,6 +6,8 @@ class DaoEvent < ApplicationRecord
   belongs_to :block
   belongs_to :ckb_transaction
   belongs_to :address
+
+  scope :created_before, ->(block_timestamp) { where("block_timestamp <= ?", block_timestamp) }
 end
 
 # == Schema Information
@@ -26,5 +28,7 @@ class DaoEvent < ApplicationRecord
 #
 # Indexes
 #
-#  index_dao_events_on_block_id  (block_id)
+#  index_dao_events_on_block_id               (block_id)
+#  index_dao_events_on_block_timestamp        (block_timestamp)
+#  index_dao_events_on_status_and_event_type  (status,event_type)
 #

app/services/charts/daily_statistic_generator.rb

@@ -2,21 +2,21 @@ module Charts
   class DailyStatisticGenerator
     MILLISECONDS_IN_DAY = BigDecimal(24 * 60 * 60 * 1000)
 
-    def initialize(datetime = nil)
+    def initialize(datetime = nil, from_scratch = false)
       raise "datetime must be a Time" if datetime.present? && !datetime.is_a?(Time)
 
       @datetime = datetime
+      @from_scratch = from_scratch
     end
 
     def call
-      daily_ckb_transactions_count = CkbTransaction.where("block_timestamp >= ? and block_timestamp <= ?", started_at, ended_at).count
-      addresses_count = Address.where("block_timestamp <= ?", ended_at).count
+      daily_ckb_transactions_count = CkbTransaction.created_after(started_at).created_before(ended_at).count
       cell_outputs = CellOutput.where.not(cell_type: "normal")
-      current_tip_block = Block.where("timestamp <= ?", ended_at).recent.first
       mining_reward = Block.where("timestamp <= ?", ended_at).sum(:secondary_reward)
       deposit_compensation = unclaimed_compensation(cell_outputs, current_tip_block) + claimed_compensation(cell_outputs)
       estimated_apc = DaoContract.default_contract.estimated_apc(current_tip_block.fraction_epoch)
       block_timestamp = Block.created_after(started_at).created_before(ended_at).recent.pick(:timestamp)
+      addresses_count = processed_addresses_count
       daily_statistic = ::DailyStatistic.find_or_create_by!(created_at_unixtimestamp: to_be_counted_date.to_i)
       daily_statistic.update(block_timestamp: block_timestamp, transactions_count: daily_ckb_transactions_count,
                              addresses_count: addresses_count, total_dao_deposit: total_dao_deposit,
@@ -28,14 +28,35 @@ def call
 
     private
 
+    attr_reader :datetime, :from_scratch
+
+    def processed_addresses_count
+      if from_scratch
+        Address.created_before(ended_at).count
+      else
+        Address.created_after(started_at).created_before(ended_at).count + latest_daily_statistic.addresses_count.to_i
+      end
+    end
+
+    def current_tip_block
+      @current_tip_block ||=
+        begin
+          if from_scratch
+            Block.created_before(ended_at).recent.first
+          else
+            Block.created_after(started_at).created_before(ended_at).recent.first
+          end
+        end
+    end
+
     def total_dao_deposit
-      deposit_amount = DaoEvent.processed.where("block_timestamp <= ?", ended_at).where(event_type: "deposit_to_dao").sum(:value)
-      withdraw_amount = DaoEvent.processed.where("block_timestamp <= ?", ended_at).where(event_type: "withdraw_from_dao").sum(:value)
+      deposit_amount = DaoEvent.processed.deposit_to_dao.created_before(ended_at).sum(:value)
+      withdraw_amount = DaoEvent.processed.withdraw_from_dao.created_before(ended_at).sum(:value)
       deposit_amount - withdraw_amount
     end
 
     def dao_depositors_count
-      DaoEvent.processed.where(event_type: "new_dao_depositor").where("block_timestamp <= ?", ended_at).count - DaoEvent.processed.where(event_type: "take_away_all_deposit").where("block_timestamp <= ?", ended_at).count
+      DaoEvent.processed.new_dao_depositor.created_before(ended_at).count - DaoEvent.processed.take_away_all_deposit.created_before(ended_at).count
     end
 
     def to_be_counted_date
@@ -50,8 +71,6 @@ def ended_at
       @ended_at ||= time_in_milliseconds(to_be_counted_date.end_of_day)
     end
 
-    attr_reader :datetime
-
     def unclaimed_compensation(cell_outputs, current_tip_block)
       @unclaimed_compensation ||=
         begin
@@ -62,22 +81,22 @@ def unclaimed_compensation(cell_outputs, current_tip_block)
     def claimed_compensation(cell_outputs)
       @claimed_compensation ||=
         begin
-          cell_outputs.where("consumed_block_timestamp <= ?", ended_at).nervos_dao_withdrawing.dead.reduce(0) do |memo, nervos_dao_withdrawing_cell|
+          cell_outputs.nervos_dao_withdrawing.consumed_before(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell|
             memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell)
           end
         end
     end
 
     def phase1_dao_interests(cell_outputs)
-      cell_outputs.where("block_timestamp <= ?", ended_at).nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell|
+      cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell|
         memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell)
       end
     end
 
     def unmade_dao_interests(cell_outputs, current_tip_block)
       @unmade_dao_interests ||=
         begin
-          cell_outputs.where("block_timestamp <= ?", ended_at).nervos_dao_deposit.live.reduce(0) do |memo, cell_output|
+          cell_outputs.nervos_dao_deposit.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, cell_output|
             dao = cell_output.block.dao
             tip_dao = current_tip_block.dao
             parse_dao = CkbUtils.parse_dao(dao)
@@ -90,17 +109,16 @@ def unmade_dao_interests(cell_outputs, current_tip_block)
     def average_deposit_time(cell_outputs)
       interest_bearing_deposits = 0
       uninterest_bearing_deposits = 0
-      sum_interest_bearing = cell_outputs.where("block_timestamp <= ?", ended_at).nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell|
+      sum_interest_bearing = cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell|
         nervos_dao_withdrawing_cell_generated_tx = nervos_dao_withdrawing_cell.generated_by
         nervos_dao_deposit_cell = nervos_dao_withdrawing_cell_generated_tx.cell_inputs.order(:id)[nervos_dao_withdrawing_cell.cell_index].previous_cell_output
         interest_bearing_deposits += nervos_dao_deposit_cell.capacity
         memo + nervos_dao_deposit_cell.capacity * (nervos_dao_withdrawing_cell.block_timestamp - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY
       end
-      sum_uninterest_bearing = cell_outputs.where("block_timestamp <= ?", ended_at).nervos_dao_deposit.live.reduce(0) do |memo, nervos_dao_deposit_cell|
-        current_time = time_in_milliseconds(Time.current)
+      sum_uninterest_bearing = cell_outputs.nervos_dao_deposit.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_deposit_cell|
         uninterest_bearing_deposits += nervos_dao_deposit_cell.capacity
 
-        memo + nervos_dao_deposit_cell.capacity * (current_time - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY
+        memo + nervos_dao_deposit_cell.capacity * (ended_at - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY
       end
 
       (sum_interest_bearing + sum_uninterest_bearing) / (interest_bearing_deposits + uninterest_bearing_deposits)
@@ -114,5 +132,9 @@ def treasury_amount(cell_outputs, current_tip_block)
     def time_in_milliseconds(time)
       (time.to_f * 1000).floor
     end
+
+    def latest_daily_statistic
+      ::DailyStatistic.order(created_at_unixtimestamp: :desc).first || OpenStruct.new(addresses_count: 0, total_dao_deposit: 0, dao_depositors_count: 0, unclaimed_compensation: 0, claimed_compensation: 0, average_deposit_time: 0, mining_reward: 0, deposit_compensation: 0, treasury_amount: 0)
+    end
   end
 end

db/migrate/20200110123617_add_index_to_block_timestamp_on_dao_events.rb

@@ -0,0 +1,6 @@
+class AddIndexToBlockTimestampOnDaoEvents < ActiveRecord::Migration[6.0]
+  def change
+    add_index :dao_events, :block_timestamp
+    add_index :dao_events, [:status, :event_type]
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_01_03_051008) do
+ActiveRecord::Schema.define(version: 2020_01_10_123617) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -197,6 +197,8 @@
     t.datetime "updated_at", precision: 6, null: false
     t.decimal "block_timestamp", precision: 30
     t.index ["block_id"], name: "index_dao_events_on_block_id"
+    t.index ["block_timestamp"], name: "index_dao_events_on_block_timestamp"
+    t.index ["status", "event_type"], name: "index_dao_events_on_status_and_event_type"
   end
 
   create_table "epoch_statistics", force: :cascade do |t|

test/services/charts/daily_statistic_generator_test.rb

@@ -3,9 +3,9 @@
 module Charts
   class DailyStatisticGeneratorTest < ActiveSupport::TestCase
     test "should create daily statistic record" do
-      block = create(:block, dao: "0xaff1568bbe49672f8a02516252ab2300df8c9e15dad428000035a1d671700007")
+      block = create(:block, dao: "0xaff1568bbe49672f8a02516252ab2300df8c9e15dad428000035a1d671700007", timestamp: (Time.current - 1.day).end_of_day.to_i * 1000)
       tx = create(:ckb_transaction, block: block)
-      create(:cell_output, cell_type: "nervos_dao_deposit", generated_by: tx, ckb_transaction: tx, block: block, capacity: 10**8 * 1000, block_timestamp: (Time.current - 1.day).end_of_day.strftime("%Q"))
+      create(:cell_output, cell_type: "nervos_dao_deposit", generated_by: tx, ckb_transaction: tx, block: block, capacity: 10**8 * 1000, block_timestamp: (Time.current - 1.day).end_of_day.to_i * 1000)
       assert_difference -> { ::DailyStatistic.count }, 1 do
         Charts::DailyStatisticGenerator.new.call
       end