update feeds | add base | create ocr macro

duneanalytics · Jul 15, 2024 · 6042590 · 6042590
1 parent 84b8eb4
commit 6042590
Show file tree

Hide file tree

Showing 120 changed files with 2,265 additions and 3,728 deletions.
diff --git a/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_fulfilled_transactions.sql b/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_fulfilled_transactions.sql
@@ -7,59 +7,22 @@
     file_format='delta',
     incremental_strategy='merge',
     unique_key=['tx_hash', 'tx_index', 'node_address'],
-    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')],
-    post_hook='{{ expose_spells(\'["arbitrum"]\',
-                                "project",
-                                "chainlink",
-                                \'["linkpool_ryan"]\') }}'
+    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
   )
 }}
 
-
 WITH
-  arbitrum_usd AS (
-    SELECT
-      minute as block_time,
-      price as usd_amount
-    FROM
-      {{ source('prices', 'usd') }} price
-    WHERE
-      symbol = 'ETH'
-      {% if is_incremental() %}
-        AND {{ incremental_predicate('minute') }}
-      {% endif %}
-  ),
-  ocr_fulfilled_transactions AS (
-    SELECT
-      tx.hash as tx_hash,
-      tx.index as tx_index,
-      MAX(tx.block_time) as block_time,
-      cast(date_trunc('month', MAX(tx.block_time)) as date) as date_month,
-      tx."from" as "node_address",
-      MAX(
-        (cast((gas_used) as double) / 1e18) * effective_gas_price
-      ) as token_amount,
-      MAX(arbitrum_usd.usd_amount) as usd_amount
-    FROM
-      {{ source('arbitrum', 'transactions') }} tx
-      RIGHT JOIN {{ ref('chainlink_arbitrum_ocr_gas_transmission_logs') }} ocr_gas_transmission_logs ON ocr_gas_transmission_logs.tx_hash = tx.hash
-      LEFT JOIN arbitrum_usd ON date_trunc('minute', tx.block_time) = arbitrum_usd.block_time
-    {% if is_incremental() %}
-      WHERE {{ incremental_predicate('tx.block_time') }}
-    {% endif %}
-    GROUP BY
-      tx.hash,
-      tx.index,
-      tx."from"
-  )
+ocr_fulfilled_transactions AS (
+    {{
+        chainlink_ocr_fulfilled_transactions(
+            blockchain = 'arbitrum',
+            gas_token_symbol = 'ETH',
+            gas_price_column = 'effective_gas_price',
+        )
+    }}
+)
+
 SELECT
- 'arbitrum' as blockchain,
-  block_time,
-  date_month,
-  node_address,
-  token_amount,
-  usd_amount,
-  tx_hash,
-  tx_index
+    *
 FROM
-  ocr_fulfilled_transactions
+    ocr_fulfilled_transactions
diff --git a/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_gas_daily.sql b/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_gas_daily.sql
@@ -1,107 +1,23 @@
 {{
   config(
-
     alias='ocr_gas_daily',
     partition_by=['date_month'],
     materialized='incremental',
     file_format='delta',
     incremental_strategy='merge',
-    unique_key=['date_start', 'node_address'],
-    post_hook='{{ expose_spells(\'["arbitrum"]\',
-                                "project",
-                                "chainlink",
-                                \'["linkpool_ryan"]\') }}'
+    unique_key=['date_start', 'node_address']
   )
 }}
 
-{% set truncate_by = 'day' %}
-
 WITH
-  ocr_gas_fulfilled_daily AS (
-    SELECT
-      cast(date_trunc('{{truncate_by}}', fulfilled.block_time) as date) as date_start,
-      fulfilled.node_address,
-      SUM(fulfilled.token_amount) as token_amount,
-      SUM(fulfilled.token_amount * fulfilled.usd_amount) as usd_amount
-    FROM
-      {{ ref('chainlink_arbitrum_ocr_fulfilled_transactions') }} fulfilled
-    {% if is_incremental() %}
-      WHERE
-        {{ incremental_predicate('fulfilled.block_time') }}
-    {% endif %}
-    GROUP BY
-      1, 2
-    ORDER BY
-      1, 2
-  ),
-  ocr_gas_reverted_daily AS (
-    SELECT
-      cast(date_trunc('{{truncate_by}}', reverted.block_time) as date) as date_start,
-      reverted.node_address,
-      SUM(reverted.token_amount) as token_amount,
-      SUM(reverted.token_amount * reverted.usd_amount) as usd_amount
-    FROM
-      {{ ref('chainlink_arbitrum_ocr_reverted_transactions') }} reverted
-    {% if is_incremental() %}
-      WHERE
-        {{ incremental_predicate('reverted.block_time') }}
-    {% endif %}
-    GROUP BY
-      1, 2
-    ORDER BY
-      1, 2
-  ),
-  ocr_gas_daily_meta AS (
-    SELECT
-      COALESCE(
-        fulfilled.date_start,
-        reverted.date_start
-      ) AS "date_start",
-      COALESCE(
-        fulfilled.node_address,
-        reverted.node_address
-      ) AS "node_address",
-      COALESCE(fulfilled.token_amount, 0) as fulfilled_token_amount,
-      COALESCE(reverted.token_amount, 0) as reverted_token_amount,
-      COALESCE(fulfilled.usd_amount, 0) as fulfilled_usd_amount,
-      COALESCE(reverted.usd_amount, 0) as reverted_usd_amount
-    FROM
-      ocr_gas_fulfilled_daily fulfilled
-      FULL OUTER JOIN ocr_gas_reverted_daily reverted ON
-        reverted.date_start = fulfilled.date_start AND
-        reverted.node_address = fulfilled.node_address
-    ORDER BY
-      1, 2
-  ),
   ocr_gas_daily AS (
-    SELECT
-      'arbitrum' as blockchain,
-      date_start,
-      cast(date_trunc('month', date_start) as date) as date_month,
-      ocr_gas_daily_meta.node_address as node_address,
-      operator_name,
-      fulfilled_token_amount,
-      fulfilled_usd_amount,
-      reverted_token_amount,
-      reverted_usd_amount,
-      fulfilled_token_amount + reverted_token_amount as total_token_amount,
-      fulfilled_usd_amount + reverted_usd_amount as total_usd_amount
-    FROM ocr_gas_daily_meta
-    LEFT JOIN {{ ref('chainlink_arbitrum_ocr_operator_node_meta') }} ocr_operator_node_meta ON ocr_operator_node_meta.node_address = ocr_gas_daily_meta.node_address
+    {{
+        chainlink_ocr_gas_daily(
+            blockchain = 'arbitrum'
+        )
+    }}
   )
 SELECT
-  blockchain,
-  date_start,
-  date_month,
-  node_address,
-  operator_name,
-  fulfilled_token_amount,
-  fulfilled_usd_amount,
-  reverted_token_amount,
-  reverted_usd_amount,
-  total_token_amount,
-  total_usd_amount
+    *
 FROM
-  ocr_gas_daily
-ORDER BY
-  "date_start"
+    ocr_gas_daily
diff --git a/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_gas_transmission_logs.sql b/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_gas_transmission_logs.sql
@@ -6,25 +6,20 @@
     post_hook='{{ expose_spells(\'["arbitrum"]\',
                                 "project",
                                 "chainlink",
-                                \'["linkpool_ryan"]\') }}'
+                                \'["linkpool_ryan","linkpool_jon"]\') }}'
   )
 }}
 
+WITH
+ocr_gas_transmission_logs AS (
+    {{
+        chainlink_ocr_gas_transmission_logs(
+            blockchain = 'arbitrum'
+        )
+    }}
+)
+
 SELECT
-  'arbitrum' as blockchain,
-  block_hash,
-  contract_address,
-  data,
-  topic0,
-  topic1,
-  topic2,
-  topic3,
-  tx_hash,
-  block_number,
-  block_time,
-  index,
-  tx_index
+    *
 FROM
-  {{ source('arbitrum', 'logs') }} logs
-WHERE
-  topic0 = 0xf6a97944f31ea060dfde0566e4167c1a1082551e64b60ecb14d599a9d023d451
+    ocr_gas_transmission_logs
diff --git a/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_operator_admin_meta.sql b/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_operator_admin_meta.sql
@@ -1,6 +1,5 @@
 {{
   config(
-
     alias='ocr_operator_admin_meta',
     materialized = 'view',
     post_hook='{{ expose_spells(\'["arbitrum"]\',
@@ -31,10 +30,11 @@ SELECT admin_address, operator_name FROM (VALUES
   (0x9efa0A617C0552F1558c95993aA8b8A68b3e709C, '{{dextrac}}'),
   (0x15918ff7f6C44592C81d999B442956B07D26CC44, '{{fiews}}'),
   (0x6eF38c3d1D85B710A9e160aD41B912Cb8CAc2589, '{{frameworkventures}}'),
+  (0xdD831352762e9de7ad5a264990e1bB9F87A6Fc21, '{{inotel}}'),
   (0xB8C6E43f37E04A2411562a13c1C48B3ad5975cf4, '{{inotel}}'),
   (0x4564A9c6061f6f1F2Eadb954B1b3C241D2DC984e, '{{linkforest}}'),
-  (0x797de2909991C66C66D8e730C8385bbab8D18eA6, '{{linkpool}}'),
   (0xDF812B91D8bf6DF698BFD1D8047839479Ba63420, '{{linkpool}}'),
+  (0x797de2909991C66C66D8e730C8385bbab8D18eA6, '{{linkpool}}'),
   (0x14f94049397C3F1807c45B6f854Cb5F36bC4393B, '{{linkriver}}'),
   (0x4dc81f63CB356c1420D4620414f366794072A3a8, '{{matrixedlink}}'),
   (0x0921E157b690c4F89F7C2a210cFd8bF3964F6776, '{{northwestnodes}}'),

diff --git a/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_operator_node_meta.sql b/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_operator_node_meta.sql
@@ -1,6 +1,5 @@
 {{
   config(
-
     alias='ocr_operator_node_meta',
     materialized = 'view',
     post_hook='{{ expose_spells(\'["arbitrum"]\',

diff --git a/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_request_daily.sql b/daily_spellbook/models/chainlink/arbitrum/chainlink_arbitrum_ocr_request_daily.sql
@@ -1,73 +1,23 @@
 {{
   config(
-
     alias='ocr_request_daily',
     partition_by=['date_month'],
     materialized='incremental',
     file_format='delta',
     incremental_strategy='merge',
-    unique_key=['date_start', 'node_address'],
-    post_hook='{{ expose_spells(\'["arbitrum"]\',
-                                "project",
-                                "chainlink",
-                                \'["linkpool_ryan"]\') }}'
+    unique_key=['date_start', 'node_address']
   )
 }}
 
-{% set truncate_by = 'day' %}
-
 WITH
-  ocr_request_daily_meta AS (
-    SELECT
-      COALESCE(
-        cast(date_trunc('{{truncate_by}}', fulfilled.block_time) as date),
-        cast(date_trunc('{{truncate_by}}', reverted.block_time) as date)
-      ) AS "date_start",
-      COALESCE(
-        fulfilled.node_address,
-        reverted.node_address
-      ) AS "node_address",
-      COALESCE(COUNT(fulfilled.token_amount), 0) as fulfilled_requests,
-      COALESCE(COUNT(reverted.token_amount), 0) as reverted_requests,
-      COALESCE(COUNT(fulfilled.token_amount), 0) + COALESCE(COUNT(reverted.token_amount), 0) as total_requests
-    FROM
-      {{ ref('chainlink_arbitrum_ocr_fulfilled_transactions') }} fulfilled
-      FULL OUTER JOIN {{ ref('chainlink_arbitrum_ocr_reverted_transactions') }} reverted ON
-        reverted.block_time = fulfilled.block_time AND
-        reverted.node_address = fulfilled.node_address
-    {% if is_incremental() %}
-      WHERE
-        {{ incremental_predicate('fulfilled.block_time') }}
-        OR {{ incremental_predicate('reverted.block_time') }}
-    {% endif %}
-    GROUP BY
-      1, 2
-    ORDER BY
-      1, 2
-  ),
   ocr_request_daily AS (
-    SELECT
-      'arbitrum' as blockchain,
-      date_start,
-      cast(date_trunc('month', date_start) as date) as date_month,
-      ocr_request_daily_meta.node_address as node_address,
-      operator_name,
-      fulfilled_requests,
-      reverted_requests,
-      total_requests
-    FROM ocr_request_daily_meta
-    LEFT JOIN {{ ref('chainlink_arbitrum_ocr_operator_node_meta') }} ocr_operator_node_meta ON ocr_operator_node_meta.node_address = ocr_request_daily_meta.node_address
+    {{
+        chainlink_ocr_request_daily(
+            blockchain = 'arbitrum'
+        )
+    }}
   )
 SELECT
-  blockchain,
-  date_start,
-  date_month,
-  node_address,
-  operator_name,
-  fulfilled_requests,
-  reverted_requests,
-  total_requests
+    *
 FROM
-  ocr_request_daily
-ORDER BY
-  "date_start"
+    ocr_request_daily