Merge pull request #28 from spicehq/phillip/option-to-skip-geth-traces

Add option to skip using geth traces for contract creations

Author: phillipleblanc

ethereumetl/cli/__init__.py

@@ -46,7 +46,7 @@
 
 
 @click.group()
-@click.version_option(version='1.11.0-spicehq/release/v1.0.18')
+@click.version_option(version='1.11.0-spicehq/release/v1.0.19')
 @click.pass_context
 def cli(ctx):
     pass

ethereumetl/cli/export_all.py

@@ -119,10 +119,11 @@ def get_partitions(start, end, partition_batch_size, provider_uri):
 @click.option('-P', '--postgres-connection-string', default='', show_default=False, type=str, help='Postgres connection string.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
 @click.option('-B', '--export-batch-size', default=100, show_default=True, type=int, help='The number of requests in JSON RPC batches.')
+@click.option('--skip-geth-traces', default=False, show_default=True, type=bool, help='Whether to skip using geth traces to get contracts')
 @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.')
 def export_all(start, end, partition_batch_size, provider_uri, output_dir, postgres_connection_string, max_workers, export_batch_size,
-               chain='ethereum'):
+               chain='ethereum', skip_geth_traces=False):
     """Exports all data for a range of blocks."""
     provider_uri = check_classic_provider_uri(chain, provider_uri)
     export_all_common(get_partitions(start, end, partition_batch_size, provider_uri),
-                      output_dir, postgres_connection_string, provider_uri, max_workers, export_batch_size)
+                      output_dir, postgres_connection_string, provider_uri, max_workers, export_batch_size, skip_geth_traces)

ethereumetl/jobs/export_all_common.py

@@ -89,7 +89,7 @@ def get_multi_item_exporter(item_exporters: list):
     return MultiItemExporter(valid_item_exporters)
 
 
-def export_all_common(partitions, output_dir, postgres_connection_string, provider_uri, max_workers, batch_size):
+def export_all_common(partitions, output_dir, postgres_connection_string, provider_uri, max_workers, batch_size, skip_geth_traces):
 
     for batch_start_block, batch_end_block, partition_dir in partitions:
         # # # start # # #
@@ -291,26 +291,28 @@ def export_all_common(partitions, output_dir, postgres_connection_string, provid
 
         # # # geth traces # # #
 
-        logger.info('Exporting geth traces from blocks {block_range}'.format(
-            block_range=block_range
-        ))
+        geth_traces_available = False
+        if not skip_geth_traces:
+            logger.info('Exporting geth traces from blocks {block_range}'.format(
+                block_range=block_range
+            ))
 
-        geth_traces_available = True
-        job = ExportGethTracesJob(
-            start_block=batch_start_block,
-            end_block=batch_end_block,
-            batch_size=batch_size,
-            batch_web3_provider=ThreadLocalProxy(
-                    lambda: get_provider_from_uri(provider_uri, batch=True)),
-            max_workers=max_workers,
-            item_exporter=inmemory_exporter
-        )
-        try:
-            job.run()
-        except HistoricalStateUnavailableError:
-            geth_traces_available = False
-        except HTTPError:
-            geth_traces_available = False
+            geth_traces_available = True
+            job = ExportGethTracesJob(
+                start_block=batch_start_block,
+                end_block=batch_end_block,
+                batch_size=batch_size,
+                batch_web3_provider=ThreadLocalProxy(
+                        lambda: get_provider_from_uri(provider_uri, batch=True)),
+                max_workers=max_workers,
+                item_exporter=inmemory_exporter
+            )
+            try:
+                job.run()
+            except HistoricalStateUnavailableError:
+                geth_traces_available = False
+            except HTTPError:
+                geth_traces_available = False
 
         contracts_output_dir = '{output_dir}/contracts{partition_dir}'.format(
             output_dir=output_dir,