Koios v1.3.2 (#328)

- `/address_outputs` - Basic transaction output info for given addresses
- `/pool_calidus_keys` - List of valid calidus keys for all pools
- `/pool_groups` - List of all registered pool and their groups across sources from [pool_groups](https://github.com/cardano-community/pool_groups) repository. This is only relevant for mainnet
- `/pool_owner_history` - Return information about pool owner's historical stake and their promised pledge to their pools
- `/account_stake_history` - Get history for dreps voting power distribution
- `/account_reward_history` - Get the full rewards history (including MIR) for given stake addresses
- `/account_update_history` - Get historical updates (registration, deregistration, delegation and withdrawals) for given stake addresses
- `/drep_voting_power_history` - Renamed from drep_history (left drep_history for consistency)
- `/pool_voting_power_history` - New endpoint to give voting_power_history by pool
- `/vote_list` - List of all votes posted on-chain

- Input (non-breaking) - `/voter_proposal_list` - Make _voter_id optional
- Output - `/tip` - Deprecate column `block_no` and use `block_height` to be consistent across all endpoints
- Output - `/pool_list` - Add column `pool_group` to output
- Output - `/pool_voting_summary` - Update fields with data type lovelace/integer to string
- Output - `/account_info` - Add column proposal_refund to output

- `/account_history` - Part of flattening the JSON heirarchy (so that horizontal filtering is optimal), use `/account_stake_history` instead
- `/account_rewards` - Part of flattening the JSON heirarchy (so that horizontal filtering is optimal), use `/account_rewards_history` instead
- `/account_updates` - Part of flattening the JSON heirarchy (so that horizontal filtering is optimal), use `/account_update_history` instead
- `/drep_history` - Renamed to `/drep_voting_power_history` to be more precise
- `/drep_votes` - Should be available via `/vote_list` (can use horizontal filtering)
- `/pool_votes` - Should be available via `/vote_list` (can use horizontal filtering)

- Replace [pg_bech32](https://github.com/cardano-community/pg_bech32) library with [pg_cardano](https://github.com/cardano-community/pg_cardano)
- Improve grest.epoch_info_cache performance
- `/proposal_voting_summary` - 	Do lookup for gov action id beforehand, Added handling of inactive DRep stake in voting as abstain, unless they voted in epoch of interest
- `/account_info` - Add limit filters to ensure there is a single hit per account
- Add indexes for voting_procedure

- None

Author: rdlrt

files/grest/.sqlfluff

@@ -1,6 +1,6 @@
 [sqlfluff]
 dialect = postgres
-exclude_rules = structure.column_order, references.keywords
+#exclude_rules = structure.column_order, references.keywords
 max_line_length=260
 capitalisation_policy = upper
 extended_capitalisation_policy = upper

files/grest/cron/jobs/pool-groups-update.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+DB_NAME=cexplorer
+POOL_GROUP_URL=https://raw.githubusercontent.com/cardano-community/pool_groups/refs/heads/main/spos.json
+
+dttime=$(date +%F_%H:%M:%S)
+
+network=$(psql ${DB_NAME} -qbt -c "SELECT network_name from public.meta;" | awk '{print $1}')
+[[ "${network}" != "mainnet" ]] && echo "pool_groups endpoint is not applicable for networks other than mainnet!" && exit 0
+
+echo "${dttime} - START - Import pool groups"
+
+curl -sfL "${POOL_GROUP_URL}" -o .poolgroups.json
+
+[[ -f '.poolgroups.csv' ]] && rm -f .poolgroups.csv
+jq -er '.[] | [ .pool_id_bech32,.group,.ticker,.adastat_group,.balanceanalytics_group ] | @csv' .poolgroups.json > .poolgroups.csv
+
+cat << EOF > .poolgroups.sql
+CREATE TEMP TABLE tmppoolgrps (like grest.pool_groups);
+\COPY tmppoolgrps FROM '.poolgroups.csv' DELIMITER ',' CSV;
+INSERT INTO grest.pool_groups SELECT * FROM tmppoolgrps ON CONFLICT(pool_id_bech32) DO UPDATE SET pool_id_bech32=excluded.pool_id_bech32, pool_group=excluded.pool_group, ticker=excluded.ticker, adastat_group=excluded.adastat_group, balanceanalytics_group=excluded.balanceanalytics_group;
+EOF
+
+psql ${DB_NAME} -qb -f .poolgroups.sql >/dev/null && rm -f .poolgroups.sql
+psql ${DB_NAME} -qb -c "INSERT INTO grest.control_table (key, last_value) VALUES ('pool_groups_check','${dttime}') ON CONFLICT(key) DO UPDATE SET last_value='${dttime}'"
+echo "$(date +%F_%H:%M:%S) - END - Pool Groups Update finished."

files/grest/rpc/000_utilities/cip129.sql

@@ -27,9 +27,9 @@ LANGUAGE plpgsql STABLE
 AS $$
 BEGIN
   IF LENGTH(_cc_hot) = 60 THEN
-    RETURN SUBSTRING(b32_decode(_cc_hot) from 3);
+    RETURN SUBSTRING(ENCODE(cardano.bech32_decode_data(_cc_hot),'hex') from 3);
   ELSE
-    RETURN b32_decode(_cc_hot);
+    RETURN ENCODE(cardano.bech32_decode_data(_cc_hot),'hex');
   END IF;
 END;
 $$;
@@ -40,7 +40,7 @@ LANGUAGE plpgsql STABLE
 AS $$
 BEGIN
   IF LENGTH(_cc_hot) = 60 THEN
-    RETURN SUBSTRING(b32_decode(_cc_hot) from 2 for 1) = '3';
+    RETURN SUBSTRING(ENCODE(cardano.bech32_decode_data(_cc_hot),'hex') from 2 for 1) = '3';
   ELSE
     RETURN STARTS_WITH(_cc_hot, 'cc_hot_script');
   END IF;
@@ -54,9 +54,9 @@ AS $$
 BEGIN
   IF _raw IS NULL THEN RETURN NULL; END IF;
   IF _is_script THEN
-    RETURN b32_encode('cc_hot', ('\x03'::bytea || _raw)::text);
+    RETURN cardano.bech32_encode('cc_hot', ('\x03'::bytea || _raw));
   ELSE
-    RETURN b32_encode('cc_hot', ('\x02'::bytea || _raw)::text);
+    RETURN cardano.bech32_encode('cc_hot', ('\x02'::bytea || _raw));
   END IF;
 END;
 $$;
@@ -67,9 +67,9 @@ LANGUAGE plpgsql STABLE
 AS $$
 BEGIN
   IF LENGTH(_cc_cold) = 61 THEN
-    RETURN SUBSTRING(b32_decode(_cc_cold) from 3);
+    RETURN SUBSTRING(ENCODE(cardano.bech32_decode_data(_cc_cold),'hex') from 3);
   ELSE
-    RETURN b32_decode(_cc_cold);
+    RETURN ENCODE(cardano.bech32_decode_data(_cc_cold),'hex');
   END IF;
 END;
 $$;
@@ -80,7 +80,7 @@ LANGUAGE plpgsql STABLE
 AS $$
 BEGIN
   IF LENGTH(_cc_cold) = 61 THEN
-    RETURN SUBSTRING(b32_decode(_cc_cold) from 2 for 1) = '3';
+    RETURN SUBSTRING(ENCODE(cardano.bech32_decode_data(_cc_cold),'hex') from 2 for 1) = '3';
   ELSE
     RETURN STARTS_WITH(_cc_cold, 'cc_cold_script');
   END IF;
@@ -94,9 +94,9 @@ AS $$
 BEGIN
   IF _raw IS NULL THEN RETURN NULL; END IF;
   IF _is_script THEN
-    RETURN b32_encode('cc_cold', ('\x13'::bytea || _raw)::text);
+    RETURN cardano.bech32_encode('cc_cold', ('\x13'::bytea || _raw));
   ELSE
-    RETURN b32_encode('cc_cold', ('\x12'::bytea || _raw)::text);
+    RETURN cardano.bech32_encode('cc_cold', ('\x12'::bytea || _raw));
   END IF;
 END;
 $$;
@@ -107,9 +107,9 @@ LANGUAGE plpgsql STABLE
 AS $$
 BEGIN
   IF LENGTH(_drep_id) = 58 THEN
-    RETURN SUBSTRING(b32_decode(_drep_id) from 3);
+    RETURN SUBSTRING(ENCODE(cardano.bech32_decode_data(_drep_id),'hex') from 3);
   ELSE
-    RETURN b32_decode(_drep_id);
+    RETURN ENCODE(cardano.bech32_decode_data(_drep_id),'hex');
   END IF;
 END;
 $$;
@@ -120,7 +120,7 @@ LANGUAGE plpgsql STABLE
 AS $$
 BEGIN
   IF LENGTH(_drep_id) = 58 THEN
-    RETURN SUBSTRING(b32_decode(_drep_id) from 2 for 1) = '3';
+    RETURN SUBSTRING(ENCODE(cardano.bech32_decode_data(_drep_id),'hex') from 2 for 1) = '3';
   ELSE
     RETURN STARTS_WITH(_drep_id, 'drep_script');
   END IF;
@@ -134,9 +134,9 @@ AS $$
 BEGIN
   IF _raw IS NULL THEN RETURN NULL; END IF;
   IF _is_script THEN
-    RETURN b32_encode('drep', ('\x23'::bytea || _raw)::text);
+    RETURN cardano.bech32_encode('drep', ('\x23'::bytea || _raw));
   ELSE
-    RETURN b32_encode('drep', ('\x22'::bytea || _raw)::text);
+    RETURN cardano.bech32_encode('drep', ('\x22'::bytea || _raw));
   END IF;
 END;
 $$;
@@ -148,7 +148,7 @@ AS $$
 DECLARE
   proposal_id_hex text;
 BEGIN
-  SELECT INTO proposal_id_hex b32_decode(_proposal_id);
+  SELECT INTO proposal_id_hex ENCODE(cardano.bech32_decode_data(_proposal_id),'hex');
   RETURN ARRAY[LEFT(proposal_id_hex, 64), ('x' || RIGHT(proposal_id_hex, -64))::bit(8)::int::text];
 END;
 $$;
@@ -158,7 +158,7 @@ RETURNS text
 LANGUAGE plpgsql STABLE
 AS $$
 BEGIN
-  RETURN b32_encode('gov_action', (_tx_hash || DECODE(LPAD(TO_HEX(_index), 2, '0'), 'hex'))::text);
+  RETURN cardano.bech32_encode('gov_action', (_tx_hash || DECODE(LPAD(TO_HEX(_index), 2, '0'), 'hex')));
 END;
 $$;
 

files/grest/rpc/000_utilities/cip5.sql

@@ -9,11 +9,14 @@ AS $$
 BEGIN
   IF _raw IS NULL THEN
    RETURN NULL;
-  END IF;
-  IF SUBSTRING(ENCODE(_raw, 'hex') from 2 for 1) = '0' THEN
-    RETURN b32_encode('stake_test', _raw::text);
   ELSE
-    RETURN b32_encode('stake', _raw::text);
+    RETURN cardano.tools_shelley_address_build(
+      ''::bytea,
+      FALSE,
+      SUBSTRING(_raw FROM 2),
+      FALSE,
+      SUBSTRING(ENCODE(_raw, 'hex') from 2 for 1)::integer
+      )::text;
   END IF;
 END;
 $$;

files/grest/rpc/00_blockchain/tip.sql

@@ -4,6 +4,7 @@ RETURNS TABLE (
   epoch_no word31type,
   abs_slot word63type,
   epoch_slot word31type,
+  block_height word31type,
   block_no word31type,
   block_time integer
 )
@@ -14,7 +15,8 @@ AS $$
     b.epoch_no AS epoch_no,
     b.slot_no AS abs_slot,
     b.epoch_slot_no AS epoch_slot,
-    b.block_no,
+    b.block_no AS block_height,
+    b.block_no AS block_no,
     EXTRACT(EPOCH FROM b.time)::integer
   FROM block AS b
   ORDER BY b.id DESC

files/grest/rpc/01_cached_tables/epoch_info_cache.sql

@@ -24,6 +24,7 @@ AS $$
 DECLARE
   _curr_epoch bigint;
   _latest_epoch_no_in_cache bigint;
+  _cutoff_epoch bigint;
 BEGIN
   -- Check previous cache update completed before running
   IF (
@@ -83,12 +84,25 @@ BEGIN
     tx_id bigint
   );
 
+  -- cutoff epoch is the last epoch with at least one tx referenced in cache table
+  SELECT COALESCE((
+      SELECT MIN(eic.epoch_no) 
+      FROM grest.epoch_info_cache eic
+      WHERE eic.i_last_tx_id = 
+        (SELECT MAX(i_last_tx_id) 
+        FROM grest.epoch_info_cache eic2 
+        WHERE i_last_tx_id is not null
+        )
+      ), 0)
+  INTO _cutoff_epoch;
+
   INSERT INTO last_tx_id_subset
     SELECT b.epoch_no, MAX(tx.id)
     FROM block AS b
     INNER JOIN tx ON tx.block_id = b.id
     WHERE b.block_no IS NOT NULL
       AND b.tx_count != 0
+      AND b.epoch_no >= _cutoff_epoch
     GROUP BY b.epoch_no;
 
   INSERT INTO grest.epoch_info_cache
@@ -148,16 +162,33 @@ CREATE OR REPLACE FUNCTION grest.update_latest_epoch_info_cache(_curr_epoch bigi
 RETURNS void
 LANGUAGE plpgsql
 AS $$
+DECLARE
+  _cutoff_epoch bigint;
 BEGIN
+
+  -- cutoff epoch is the last epoch with at least one tx referenced in cache table
+  SELECT COALESCE((
+      SELECT MIN(eic.epoch_no) 
+      FROM grest.epoch_info_cache eic
+      WHERE eic.i_last_tx_id = 
+        (SELECT MAX(i_last_tx_id) 
+        FROM grest.epoch_info_cache eic2 
+        WHERE i_last_tx_id is not null
+        )
+      ), 0)
+  INTO _cutoff_epoch;
+
   -- only update last tx id in case of new epoch
   IF _curr_epoch <> _epoch_no_to_update THEN
+
     UPDATE grest.epoch_info_cache
     SET i_last_tx_id = last_tx.tx_id
     FROM (
       SELECT MAX(tx.id) AS tx_id
       FROM block AS b
       INNER JOIN tx ON tx.block_id = b.id
       WHERE b.epoch_no <= _epoch_no_to_update
+        AND b.epoch_no >= _cutoff_epoch
         AND b.block_no IS NOT NULL
         AND b.tx_count != 0
     ) AS last_tx

files/grest/rpc/01_cached_tables/pool_history_cache.sql

@@ -40,7 +40,7 @@ DECLARE
   _pool_ids bigint [];
 BEGIN
   _pool_ids := (SELECT ARRAY_AGG(id) from pool_hash ph where ph.hash_raw = ANY(
-    SELECT DECODE(b32_decode(pool),'hex')
+    SELECT cardano.bech32_decode_data(pool)
     FROM UNNEST(_pool_bech32) AS pool)
   );