Stream 1201/livetable sproc ht

Makefile

@@ -21,10 +21,41 @@ rm_logs:
 	fi
 
 # deploy live table udtfs
-deploy_live_table_udtfs: rm_logs
+deploy_livetable_udtfs: rm_logs
 	dbt run \
-	-s near_models.deploy.near.live__table \
-	--vars '{UPDATE_UDFS_AND_SPS: true, ENABLE_LIVE_TABLE: true, LIVE_TABLE_MATERIALIZATION: ephemeral}' \
+	-s near_models.deploy.livetable \
+	--vars '{UPDATE_UDFS_AND_SPS: true, ENABLE_LIVE_TABLE: true, LIVE_TABLE_MATERIALIZATION: ephemeral, UDTF_TARGET: gold_core}' \
 	--profiles-dir ~/.dbt \
 	--profile near \
 	--target dev
+
+deploy_tx_sproc: rm_logs
+	dbt run-operation create_sp_refresh_fact_transactions_live \
+	--profiles-dir ~/.dbt \
+	--profile near \
+	--target dev
+
+deploy_tx_task: rm_logs
+	dbt run-operation create_fact_tx_sproc_task \
+	--profiles-dir ~/.dbt \
+	--profile near \
+	--target dev
+
+compile_sp_macro: rm_logs
+	dbt compile --select _compile_sp_macro \
+	--profiles-dir ~/.dbt \
+	--profile near \
+	--target dev
+
+compile_task: rm_logs
+	dbt compile --select _compile_task \
+	--profiles-dir ~/.dbt \
+	--profile near \
+	--target dev
+
+test_deploy_sf_tasks: rm_logs
+	dbt run --select _dummy_model \
+	--profiles-dir ~/.dbt \
+	--profile near \
+	--target dev
+

dbt_project.yml

@@ -28,7 +28,8 @@ clean-targets: # directories to be removed by `dbt clean`
 on-run-start:
   - "{{ create_udfs() }}"
   - "{{ create_sps() }}"
-
+  - "{{ deploy_sf_tasks() }}"
+
 on-run-end:
   - "{{ apply_meta_as_tags(results) }}"
 

macros/create_sps.sql

@@ -1,6 +1,7 @@
 {% macro create_sps() %}
     {% if target.database == 'NEAR' %}
         CREATE schema IF NOT EXISTS _internal;
-{{ sp_create_prod_clone('_internal') }};
+        {{ sp_create_prod_clone('_internal') }};
     {% endif %}
+    {{ create_sp_refresh_fact_transactions_live() }}
 {% endmacro %}

macros/livetable/assets/overview.md

@@ -0,0 +1,53 @@
+## Near Livetable SPROC
+
+### Overview
+```
++-----------------+      +---------------------------------------------+      +---------------------------+
+| Snowflake Task  |      | Stored Procedure (SP)                       |      |       _live.udf_api       |
+| (Sched: 5 min)  |----->| SP_REFRESH_FACT_TRANSACTIONS_LIVE()         |      |---------------------------|
+| QUERY_TAG: {...}|      |---------------------------------------------|      |                           |
++-----------------+      | 1. Check Schema/Table Exists (Idempotent)   |      |   [RPC call to Near Node] |
+                         |    - CREATE SCHEMA IF NOT EXISTS ...        |      |   Quicknode Endpoint      |
+                         |    - CREATE HYBRID TABLE IF NOT EXISTS ...  |      |                           |
+                         |      (Incl. PRIMARY KEY)                    |      +---------------------------+
+                         |                                             |                   ^
+                         | 2. Get Chain Head                           |                   |
+                         |    CALL _live.udf_api()---------------------+------------------>+
+                         |          |                                  |                   |
+                         |          |                                  |                   |
+                         |          v                                  |                   |
+                         |    [Near RPC API] <-------------------------|-------------------+
+                         |          ^                                  |                   
+                         |          | Returns Height                   |                   
+                         |    height <-- _live.udf_api()               |                   
+                         |                                             |                   
+                         | 3. Calculate Start Block                    |      .-------------------------.
+                         |    start_block = height - buffer            |      | Hybrid Table            |
+                         |                                             |      | CORE_LIVE.              |
+                         | 4. Call UDTF to Fetch Transactions          |      | FACT_TRANSACTIONS       |
+                         |    rows <--                                 |      |-------------------------|
+                         |       TABLE(livetable.tf_fact_tx(...)) ----+|      | - tx_hash (PK)          |
+                         |             |                              ||      | - block_id              |
+                         |             v UDTF Execution               ||      | - block_timestamp (*)   |
+                         |        (Requires Owner Permissions)        ||      | - tx_signer             |
+                         |        (Calls other UDFs for.              ||      | - tx_receiver           |
+                         |         block/chunk/tx details             ||      | - ...                   |
+                         |         internally)                        ||      | - _hybrid_updated_at    |
+                         |             | mimic's gold fact_tx         ||      `-------------------------'
+                         |             +------------------------------||                ^       |
+                         |                                             |                |       | Step 6: Prune
+                         | 5. MERGE Data into Hybrid Table             |--------------->|       | DELETE Rows
+                         |    rows_merged = SQLROWCOUNT                |                |       | Older than 60m
+                         |                                             |                |       | (based on block_ts)
+                         | 7. Return Status String                     |                |       v
+                         |    RETURN 'Fetched...Merged...Deleted...'   |                |  ............
+                         +--------------------|------------------------+                |  : Old Data :
+                                              |                                         |  :..........:
+                                              | Status String                           |
+                                              v                                         |
++------------------------+     +------------------------+                               |
+| Snowflake Task History |<----| Task Record            |<------------------------------'
+| (Logs Status, Return,  |     | (Captures SP Return)   |
+|  Duration)             |     +------------------------+
++------------------------+
+```

macros/livetable/near.yaml.sql

@@ -3,13 +3,13 @@
     This macro is used to generate the high level abstractions for the Near
     blockchain.
  #}
-{% set schema = blockchain ~ "_" ~ network %}
+{% set schema = blockchain %}
 
 - name: {{ schema }}.udf_get_latest_block_height
   signature: []
   return_type: INTEGER
   sql: |
-    {{ near_live_table_latest_block_height() | indent(4) -}}
+    {{ near_livetable_latest_block_height() | indent(4) -}}
 
 - name: {{ schema }}.lt_tx_udf_api
   signature:
@@ -24,7 +24,7 @@
   api_integration: '{{ var("API_INTEGRATION") }}'
   options: |
     NOT NULL
-    MAX_BATCH_ROWS = 25
+    MAX_BATCH_ROWS = 150
   sql: udf_api
 
 - name: {{ schema }}.lt_blocks_udf_api
@@ -40,7 +40,7 @@
   api_integration: '{{ var("API_INTEGRATION") }}'
   options: |
     NOT NULL
-    MAX_BATCH_ROWS = 25
+    MAX_BATCH_ROWS = 120
   sql: udf_api
 
 - name: {{ schema }}.lt_chunks_udf_api
@@ -71,21 +71,35 @@
     VOLATILE
     COMMENT = $$Returns the block data for a given block height.Fetches blocks for the specified number of blocks $$
   sql: |
-    {{ near_live_table_fact_blocks(schema, blockchain, network) | indent(4) -}}
+    {{ near_livetable_fact_blocks(schema, blockchain, network) | indent(4) -}}
 
   - name: {{ schema -}}.tf_fact_transactions
   signature:
     - [_block_height, INTEGER, The start block height to get the transactions from]
     - [row_count, INTEGER, The number of rows to fetch]
   return_type:
-    - "TABLE(tx_hash STRING, block_id NUMBER, block_timestamp TIMESTAMP_NTZ, nonce INT, signature STRING, tx_receiver STRING, tx_signer STRING, tx VARIANT, gas_used NUMBER, transaction_fee NUMBER, attached_gas NUMBER, tx_succeeded BOOLEAN, fact_transactions_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)"
+    - "TABLE(tx_hash STRING, block_id NUMBER, block_timestamp TIMESTAMP_NTZ, nonce INT, signature STRING, tx_receiver STRING, tx_signer STRING, tx VARIANT, gas_used NUMBER, transaction_fee NUMBER, attached_gas NUMBER, tx_succeeded BOOLEAN, fact_transactions_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ, data VARIANT, value OBJECT, partition_key NUMBER)"
   options: |
     NOT NULL
     RETURNS NULL ON NULL INPUT
     VOLATILE
     COMMENT = $$Returns transaction details for blocks starting from a given height.Fetches txs for the specified number of blocks.$$
   sql: |
-    {{ near_live_table_fact_transactions(schema, blockchain, network) | indent(4) -}}
+    {{ near_livetable_fact_transactions(schema, blockchain, network) | indent(4) -}}
+
+  - name: {{ schema -}}.tf_bronze_transactions
+  signature:
+    - [_block_height, INTEGER, The start block height to get the transactions from]
+    - [row_count, INTEGER, The number of rows to fetch]
+  return_type:
+    - "TABLE(data VARIANT, value OBJECT, partition_key NUMBER(38,0), _inserted_timestamp TIMESTAMP_LTZ(9))"
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+    VOLATILE
+    COMMENT = $$Returns transaction details for blocks starting from a given height.Fetches txs for the specified number of blocks.$$
+  sql: |
+    {{ near_livetable_bronze_transactions(schema, blockchain, network) | indent(4) -}}
 
   - name: {{ schema -}}.tf_fact_receipts
   signature:
@@ -99,7 +113,7 @@
     VOLATILE
     COMMENT = $$Returns receipt details for blocks starting from a given height. Fetches receipts for the specified number of blocks.$$
   sql: |
-    {{ near_live_table_fact_receipts(schema, blockchain, network) | indent(4) -}}
+    {{ near_livetable_fact_receipts(schema, blockchain, network) | indent(4) -}}
 
   - name: {{ schema -}}.tf_ez_actions
   signature:
@@ -113,7 +127,27 @@
     VOLATILE
     COMMENT = $$Returns decoded action details for blocks starting from a given height. Fetches actions for the specified number of blocks.$$
   sql: |
-    {{ near_live_table_ez_actions(schema, blockchain, network) | indent(4) -}}
+    {{ near_livetable_ez_actions(schema, blockchain, network) | indent(4) -}}
 
 {%- endmacro -%}
 
+{% macro lt_ephemeral_deploy(configs) %}
+{#
+    This macro is used to deploy livetable udf/udtfs using ephemeral models.
+ #}
+
+    {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %}
+        {% set sql %}
+            {% for config in configs %}
+                {{- livequery_models.crud_udfs_by_chain(config, this.schema, network, var("DROP_UDFS_AND_SPS")) -}}
+            {%- endfor -%}
+        {%- endset -%}
+        {%- if var("DROP_UDFS_AND_SPS") -%}
+            {%- do log("Drop partner udfs: " ~ this.database ~ "." ~ schema, true) -%}
+        {%- else -%}
+            {%- do log("Deploy partner udfs: " ~ this.database ~ "." ~ schema, true) -%}
+        {%- endif -%}
+        {%- do run_query(sql ~ livequery_models.apply_grants_by_schema(schema)) -%}
+    {%- endif -%}
+    SELECT '{{ model.schema }}' as schema_
+{%- endmacro -%}

macros/livetable/near_live_table_abstractions.sql

@@ -1,6 +1,6 @@
 -- Get Near Chain Head
 
-{% macro near_live_table_latest_block_height() %}
+{% macro near_livetable_latest_block_height() %}
 WITH rpc_call AS (
     SELECT
         DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS request_timestamp,
@@ -15,7 +15,7 @@ WITH rpc_call AS (
                     'params' : {'finality' : 'final'}
                 },
                 _utils.UDF_WHOAMI(),
-                'Vault/prod/near/quicknode/mainnet'
+                'Vault/prod/near/quicknode/livetable/mainnet'
         ):data::object AS rpc_result
 )
 SELECT
@@ -24,7 +24,7 @@ FROM
     rpc_call
 {% endmacro %}
 
-{% macro near_live_table_min_max_block_height(start_block, block_count) %}
+{% macro near_livetable_min_max_block_height(start_block, block_count) %}
 SELECT
     {{ start_block }} AS min_height,
     min_height + {{ block_count }} AS max_height, 
@@ -33,14 +33,14 @@ FROM
 {% endmacro %}
 
 -- Get Near Block Data
-{% macro near_live_table_target_blocks(start_block, block_count) %}
+{% macro near_livetable_target_blocks(start_block, block_count) %}
 
     WITH heights AS (
         SELECT
             min_height,
             max_height,
         FROM (
-             {{- near_live_table_min_max_block_height(start_block=start_block, block_count=block_count) | indent(13) -}}
+             {{- near_livetable_min_max_block_height(start_block=start_block, block_count=block_count) | indent(13) -}}
         )
     ),
     block_spine AS (
@@ -59,7 +59,7 @@ FROM
     FROM block_spine
 {% endmacro %}
 
-{% macro near_live_table_get_spine(table_name) %}
+{% macro near_livetable_get_spine(table_name) %}
 SELECT
     block_height,
     ROW_NUMBER() OVER (ORDER BY block_height) - 1 as partition_num
@@ -81,11 +81,11 @@ FROM
     )
 {% endmacro %}
 
-{% macro near_live_table_get_raw_block_data(spine) %} 
+{% macro near_livetable_get_raw_block_data(spine) %} 
 SELECT
     block_height,
     DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS request_timestamp,
-    live_table.lt_blocks_udf_api(
+    livetable.lt_blocks_udf_api(
         'POST',
         '{Service}',
         {'Content-Type' : 'application/json'},
@@ -96,14 +96,14 @@ SELECT
             'params':{'block_id': block_height}
         },
         _utils.UDF_WHOAMI(),
-        'Vault/prod/near/quicknode/mainnet'
+        'Vault/prod/near/quicknode/livetable/mainnet'
     ):data.result AS rpc_data_result
 from
     {{spine}}
 
 {% endmacro %}
 
-{% macro near_live_table_extract_raw_block_data(raw_blocks) %}
+{% macro near_livetable_extract_raw_block_data(raw_blocks) %}
 SELECT 
     block_data:header:height::string as block_id,
     TO_TIMESTAMP_NTZ(
@@ -146,22 +146,27 @@ FROM {{raw_blocks}}
 
 {% endmacro %}
 
-{% macro near_live_table_fact_blocks(schema, blockchain, network) %}
-    {%- set near_live_table_fact_blocks = livequery_models.get_rendered_model('near_models', 'livetable_fact_blocks', schema, blockchain, network) -%}
-    {{ near_live_table_fact_blocks }}
+{% macro near_livetable_fact_blocks(schema, blockchain, network) %}
+    {%- set near_livetable_fact_blocks = livequery_models.get_rendered_model('near_models', 'livetable_fact_blocks', schema, blockchain, network) -%}
+    {{ near_livetable_fact_blocks }}
 {% endmacro %}
 
-{% macro near_live_table_fact_transactions(schema, blockchain, network) %}
-    {%- set near_live_table_fact_transactions = livequery_models.get_rendered_model('near_models', 'livetable_fact_transactions', schema, blockchain, network) -%}
-    {{ near_live_table_fact_transactions }}
+{% macro near_livetable_fact_transactions(schema, blockchain, network) %}
+    {%- set near_livetable_fact_transactions = livequery_models.get_rendered_model('near_models', 'livetable_fact_transactions', schema, blockchain, network) -%}
+    {{ near_livetable_fact_transactions }}
 {% endmacro %}
 
-{% macro near_live_table_fact_receipts(schema, blockchain, network) %}
-    {%- set near_live_table_fact_receipts = livequery_models.get_rendered_model('near_models', 'livetable_fact_receipts', schema, blockchain, network) -%}
-    {{ near_live_table_fact_receipts }}
+{% macro near_livetable_bronze_transactions(schema, blockchain, network) %}
+    {%- set near_livetable_bronze_transactions = livequery_models.get_rendered_model('near_models', 'livetable_bronze_transactions', schema, blockchain, network) -%}
+    {{ near_livetable_bronze_transactions }}
 {% endmacro %}
 
-{% macro near_live_table_ez_actions(schema, blockchain, network) %}
-    {%- set near_live_table_ez_actions = livequery_models.get_rendered_model('near_models', 'livetable_ez_actions', schema, blockchain, network) -%}
-    {{ near_live_table_ez_actions }}
+{% macro near_livetable_fact_receipts(schema, blockchain, network) %}
+    {%- set near_livetable_fact_receipts = livequery_models.get_rendered_model('near_models', 'livetable_fact_receipts', schema, blockchain, network) -%}
+    {{ near_livetable_fact_receipts }}
+{% endmacro %}
+
+{% macro near_livetable_ez_actions(schema, blockchain, network) %}
+    {%- set near_livetable_ez_actions = livequery_models.get_rendered_model('near_models', 'livetable_ez_actions', schema, blockchain, network) -%}
+    {{ near_livetable_ez_actions }}
 {% endmacro %}

macros/livetable/permissions/udtf_permissions.sql

@@ -0,0 +1,14 @@
+{% macro lt_udtf_permissions() %}
+
+
+    {% set udtfs = [
+        '{{ target.database }}.livetable.lt_blocks_udf_api',
+        '{{ target.database }}.livetable.lt_tx_udf_api',
+        '{{ target.database }}.livetable.lt_chunks_udf_api',
+    ] %}
+
+    {% for udtf in udtfs %}
+        GRANT USAGE ON FUNCTION {{ udtf }} TO ROLE {{ target.role }};
+    {% endfor %}
+
+{% endmacro %}