Fix and automate local environment setup

docs/estela/installation/helm-variables.md

@@ -64,8 +64,8 @@ These variables define the estela behavior.
   by the spiders is stored. Currently, estela supports the _mongodb_ engine.
 
   {: .note }
-  >  For dev a free [MongoDB Atlas](https://www.mongodb.com/cloud/atlas) deploy can be used to set a database, as mentioned on [Estela Resources Guide](./resources.html).
-  >  Or a mongodb can be setup on a local cluster on a docker image.
+  >  For local development, MongoDB is included in the docker-compose setup and can be started with `make resources`. Use `mongodb://<hostIp>:27017/estela` as the connection string.
+  >  Alternatively, a free [MongoDB Atlas](https://www.mongodb.com/cloud/atlas) deploy can also be used, as mentioned on the [Estela Resources Guide](./resources.html).
 
 * _<SPIDERDATA\_DB\_CONNECTION>_ (Required): The connection URL to your database instance.
 

docs/estela/installation/installation.md

@@ -179,7 +179,7 @@ of the cloned [estela repository](https://github.com/bitmakerla/estela){:target=
 
 ## Resources
 
-All the named resources (except the _Document Oriented Database_ and the _SMTP Email Server_)
+All the named resources (except the _SMTP Email Server_)
 can be started locally by running this command in the  _installation_ folder:
 
 ```bash
@@ -258,6 +258,9 @@ following:
 $ make images
 ```
 
+{: .note }
+> In local mode (`CREDENTIALS: local`), project images are built and pushed directly as the production image. The `_candidate` suffix used in non-local deployments is not applied.
+
 The Helm deployment requires a release name and a namespace to identify the current version
 of the installed application. By default, the values of these variables are `base` and 
 `default`, respectively.
@@ -289,7 +292,13 @@ To proceed with the installation, perform the following steps:
 * If you are using the local resources, specifically MinIO, you will need to create a 
   public bucket with the name specified in the 
   [_BUCKET\_NAME\_PROJECTS_]({% link estela/installation/helm-variables.md %}#registry){:target="_blank"}
-  variable.
+  variable. You can do this automatically by running:
+
+  ```bash
+  $ make setup-minio-bucket
+  ```
+
+  Or manually via the web dashboard:
   * Go to the [web dashboard](http://localhost:9001){:target="_blank"} and log in using 
     the default credentials: `minioadmin : minioadmin`.
   * Then, [create a bucket](http://localhost:9001/buckets/add-bucket){:target="_blank"} 

estela-api/core/models.py

@@ -95,10 +95,18 @@ def container_image(self):
     def get_candidate_image(self):
         """Get the candidate container image for builds."""
         parsed_container_host = urlparse(settings.REGISTRY_HOST)
-        return "{}/{}:estela_{}_candidate".format(
+
+        # In local mode, don't use _candidate suffix (build directly as production)
+        if settings.CREDENTIALS == 'local':
+            suffix = ""
+        else:
+            suffix = "_candidate"
+
+        return "{}/{}:estela_{}{}".format(
             parsed_container_host.netloc or settings.REGISTRY_HOST,
             settings.REPOSITORY_NAME,
             self.pid,
+            suffix,
         )
 
     def get_production_image(self):

installation/Makefile

@@ -4,8 +4,8 @@ WEB_DIR = ../estela-web
 API_POD = $$(kubectl get pod -l app=estela-django-api -o jsonpath="{.items[0].metadata.name}")
 API_IP = $$(kubectl get services -n $${NAMESPACE} estela-django-api-service --output jsonpath='{.spec.clusterIP}')
 LOCAL_API_IP = $$(kubectl get services -n $${NAMESPACE} estela-django-api-service --output jsonpath='{.status.loadBalancer.ingress[0].ip}')
-RESOURCES = db registry minio zookeeper kafka
-SERVICES ?= django-api celery-worker celery-beat redis build-project
+RESOURCES = db registry minio zookeeper kafka mongodb
+SERVICES ?= django-api celery-worker celery-beat redis build-project project-downloader
 PLATFORM ?= linux/$(shell uname -m)
 
 
@@ -59,6 +59,18 @@ uninstall:
 	-. ./local/.env && cd helm-chart && helm uninstall $${RELEASE_NAME} -n $${NAMESPACE}
 
 
+MINIO_NETWORK = $$(docker inspect estela_storage --format '{{range $$k,$$v := .NetworkSettings.Networks}}{{$$k}}{{end}}')
+MINIO_IP = $$(docker inspect estela_storage --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
+BUCKET_NAME = $$(grep "^BUCKET_NAME_PROJECTS:" helm-chart/values.yaml | awk '{print $$2}' | tr -d '"' | tr -d "'")
+
+.PHONY: setup-minio-bucket
+setup-minio-bucket:
+	-docker run --rm --network $(MINIO_NETWORK) --entrypoint sh minio/mc -c \
+		"mc alias set myminio http://$(MINIO_IP):9000 minioadmin minioadmin && \
+		 mc mb myminio/$(BUCKET_NAME) --ignore-existing && \
+		 mc anonymous set download myminio/$(BUCKET_NAME)"
+
+
 .PHONY: update-api-ip
 update-api-ip:
 	-. ./local/.env && cd helm-chart && \

installation/helm-chart/templates/API/api-secrets.yaml

@@ -11,7 +11,7 @@ data:
   EMAIL_HOST_USER: {{ .Values.EMAIL_HOST_USER | b64enc }}
   EMAIL_HOST_PASSWORD: {{ .Values.EMAIL_HOST_PASSWORD | b64enc }}
   SECRET_KEY: {{ .Values.SECRET_KEY | b64enc }}
-  EXTERNAL_APP_KEYS: {{ .Values.EXTERNAL_APP_KEYS | b64enc }}
+  EXTERNAL_APP_KEYS: {{ .Values.EXTERNAL_APP_KEYS | default "none" | b64enc }}
   {{- if .Values.AWS_ACCESS_KEY_ID }}
   AWS_ACCESS_KEY_ID: {{ .Values.AWS_ACCESS_KEY_ID | b64enc }}
   {{- end }}

installation/helm-chart/values.yaml.example

@@ -30,7 +30,7 @@ STAGE: "DEVELOPMENT"
 
 # Database
 SPIDERDATA_DB_ENGINE: mongodb
-SPIDERDATA_DB_CONNECTION: "" # [MongoDB Atlas](https://www.mongodb.com/cloud/atlas) can be used to set a database
+SPIDERDATA_DB_CONNECTION: "" # Local docker-compose: mongodb://<hostIp>:27017/estela — or use MongoDB Atlas: https://www.mongodb.com/cloud/atlas
 SPIDERDATA_DB_CERTIFICATE_PATH: config/ca-certificate.crt
 
 # Queue Platform

installation/local/docker-compose.yaml

@@ -75,7 +75,19 @@ services:
     volumes:
     - /var/run/docker.sock:/var/run/docker.sock
 
+  mongodb:
+    container_name: estela_mongodb
+    image: mongo:6.0
+    ports:
+      - "27017:27017"
+    environment:
+      MONGO_INITDB_DATABASE: estela
+    volumes:
+      - mongodb_data:/data/db
+    restart: unless-stopped
+
 volumes:
   estela_dbdata:
   registry_data:
   datastore:
+  mongodb_data: