values.yaml.example

#################### CHART VARIABLES ####################

local: true
hostIp: "" # on local dev deploy the value from `$ minikube ssh 'grep host.minikube.internal /etc/hosts | cut -f1'` can be used
registryHost: "" # hostIp:5001
nodeSelector: ""

#################### CLOUD VARIABLES ####################

######## AWS ########

AWS_ACCESS_KEY_ID: "" # values needed from an AWS account
AWS_SECRET_ACCESS_KEY: ""
AWS_DEFAULT_REGION: ""

awsRegistry: ""
imagePullSecrets: ""

######## GCP ########

GOOGLE_APPLICATION_CREDENTIALS: ""
GOOGLE_APPLICATION_LOCATION: ""

#################### ESTELA VARIABLES ####################

############ GLOBAL ############

# Deployment stage: DEVELOPMENT, STAGING, PRODUCTION
STAGE: "DEVELOPMENT"

# Database
SPIDERDATA_DB_ENGINE: mongodb
SPIDERDATA_DB_CONNECTION: "" # [MongoDB Atlas](https://www.mongodb.com/cloud/atlas) can be used to set a database
SPIDERDATA_DB_CERTIFICATE_PATH: config/ca-certificate.crt

# Queue Platform
QUEUE_PARAMETERS:
  QUEUE_PLATFORM: kafka
  QUEUE_PLATFORM_LISTENERS: "" # kafka
  QUEUE_PLATFORM_PORT: "" # 9092
  # Fill in any additional variables needed for the selected queue platform, visit: https://estela.bitmaker.la/estela/queueing.html#estela-queue-adapter

# Redis scrapy stats
REDIS_URL: redis://estela-redis-service
REDIS_STATS_INTERVAL: "" # 5.0

############ API ############

# Database
DB_HOST: "" # database
DB_PORT: "" # 3306
DB_NAME: "" # estela
DB_USER: "" # django-api
DB_PASSWORD: "" # estela12345

# Registry
REGISTRY_HOST: "" # same value as registryHost
REGISTRY_ID: ""
REPOSITORY_NAME: "" # estela-projects
BUCKET_NAME_PROJECTS: "" # estela-api-projects

# Settings
SECRET_KEY: "" # it can generate one on https://djecrety.ir/ To avoid reading conflicts, enclose the value in quotes.
DJANGO_SETTINGS_MODULE: "" # config.settings.local
ENGINE: "" # kubernetes
CREDENTIALS: "" # local
CORS_ORIGIN_WHITELIST: ""
DJANGO_API_HOST: dummy
DJANGO_ALLOWED_HOSTS: ""
DJANGO_EXTERNAL_APPS: "" # "app_1,app_2,..."
EXTERNAL_APP_KEYS: "" # "key_1,key_2,..."
EXTERNAL_MIDDLEWARES: "" # "app1.middlware,app2.middlware"
DEDICATED_SPIDER_NODES: "" # "True"
BUILD: "" # "default"

# Celery
CELERY_BROKER_URL: redis://estela-redis-service
CELERY_RESULT_BACKEND: redis://estela-redis-service:6379/0
CELERY_EXTERNAL_IMPORTS: "" # "app1,app2"

# Mailing
EMAIL_HOST: "" # http://smtp.gmail.com
EMAIL_PORT: "" # 587
EMAIL_HOST_USER: ""
EMAIL_HOST_PASSWORD: ""
EMAILS_TO_ALERT: ""
VERIFICATION_EMAIL: ""
REGISTER: "" # "True"

# Data Downloads
MAX_CLI_DOWNLOAD_CHUNK_MB: "2"
MAX_WEB_DOWNLOAD_SIZE_MB: "1024"

# Proxies
PROXY_PROVIDERS_TO_TRACK: ""

############ QUEUEING ############

CONSUMER_PRODUCTION: "" # "False"

WORKER_POOL: ""
HEARTBEAT_TICK: ""
QUEUE_BASE_TIMEOUT: ""
QUEUE_MAX_TIMEOUT: ""

SIZE_THRESHOLD: ""
INSERT_TIME_THRESHOLD: ""
ACTIVITY_TIME_THRESHOLD: ""

############ SPIDER JOB RESOURCES #########
SPIDER_NODE_ROLE: "bitmaker-worker"
WORKERS_CAPACITY_THRESHOLD: "0.95"
DISPATCH_RETRY_DELAY: "30"
RUN_JOBS_PER_LOT: "100"

############ RESOURCES LIMITS #############
API_CPU_LIMIT: "" # "250m"
API_MEM_LIMIT: "" # "1Gi"
CELERY_WORKER_CPU_LIMIT: "" # "100m"
CELERY_WORKER_MEM_LIMIT: "" # "500Mi"
CELERY_BEAT_CPU_LIMIT: "" # "100m"
CELERY_BEAT_MEM_LIMIT: "" # "250Mi"
CONSUMERS_CPU_LIMIT: "" # "250m"
CONSUMERS_MEM_LIMIT: "" # "750Mi"