neynar_parquet_importer/env.example at main · neynarxyz/neynar_parquet_importer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
# =============================================================================
# AWS S3 Configuration (Required for downloading parquet files)
# =============================================================================
# See boto3 documentation for more AWS environment variables:
# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#using-environment-variables

AWS_ACCESS_KEY_ID=your_actual_access_key_here
AWS_SECRET_ACCESS_KEY=your_actual_secret_key_here

# S3 Bucket Settings (usually don't need to change)
PARQUET_S3_BUCKET=tf-premium-parquet
PARQUET_S3_DATABASE=public-postgres

# =============================================================================
# Database Backend Selection
# =============================================================================
DATABASE_BACKEND=neo4j  # postgresql or neo4j

# =============================================================================
# Neo4j Configuration (when DATABASE_BACKEND=neo4j)
# =============================================================================
NEO4J_URI=neo4j://localhost:7687
NEO4J_USER=neo4j
NEO4J_PASSWORD=your_neo4j_password_here
NEO4J_DATABASE=neo4j
NEO4J_MAX_CONNECTIONS=10
BATCH_SIZE_NEO4J=5000
TRANSFORM_CHUNK_SIZE=100

# =============================================================================
# PostgreSQL Configuration (when DATABASE_BACKEND=postgresql)
# =============================================================================
POSTGRES_DSN=postgresql+psycopg://postgres:postgres@postgres:5432/neynar_parquet_importer
POSTGRES_POOL_SIZE=135

# =============================================================================
# Parquet Data Source Configuration
# =============================================================================
# Choose ONE of these configurations:

# OPTION 1: Nindexer v3 data (recommended for social graph - 1 minute increments)
NPE_VERSION=v3
PARQUET_S3_SCHEMA=nindexer
NPE_DURATION=1
# Available tables: follows,fids,verifications,casts,reactions,channels,channel_follows,etc.
TABLES=follows,fids,verifications

# OPTION 2: Farcaster v2 data (5 minute increments)
# NPE_VERSION=v2
# PARQUET_S3_SCHEMA=farcaster
# NPE_DURATION=300
# Available tables: casts,fids,reactions,user_data,signers,etc.
# TABLES=casts,fids,reactions

# Views (optional - computed tables)
VIEWS=

# =============================================================================
# Neynar config
# =============================================================================
#NEYNAR_API_KEY=
#APP_UUID=
#NEYNAR_API_URL=http://localhost:3000

# =============================================================================
# Logging and Monitoring
# =============================================================================
LOG_FORMAT=rich  # rich or json
LOG_LEVEL=DEBUG  # DEBUG, INFO, WARN, ERROR

# Datadog monitoring (optional)
DATADOG_ENABLED=false

# Filters (optional - see filters.example.json)
# FILTERS_JSON=./filters.json

# =============================================================================
# Development Settings
# =============================================================================
# INTERACTIVE_DEBUG=false  # Open shell on exceptions (development only)