Merging cbrelease-4.8.21 changes to cbrelease-4.8.22

.circleci/config.yml

@@ -2,7 +2,8 @@ version: 2.1
 jobs:
   datapipeline-build:
     working_directory: ~/dp/data-pipeline-flink
-    machine: true
+    machine:
+      image: ubuntu-2004:202008-01
     steps:
       - checkout:
           path: ~/dp
@@ -22,4 +23,4 @@ workflows:
   build_and_test:
     jobs:
       - datapipeline-build
-#testing quality gate
+#testing  quality gate

.gitignore

@@ -12,6 +12,7 @@ database.yml
 .idea
 tmp/
 kibana-auth/node_modules/
+kubernetes/helm_charts/*/values.yaml
 
 # DevOps
 *.backup
@@ -28,3 +29,4 @@ target/
 *.iml
 **/.settings
 dependency-reduced-pom.xml
+/pom.xml

ansible/analyticsapi_logstash.yml

@@ -9,6 +9,6 @@
       apt: update_cache=yes cache_valid_time={{ apt_cache_valid_time }}
       become: yes
   roles:
-    - jdk-1.8.0_121
+    - openjdk
     - { role: ep_logstash, logstash_conf: ".indexer.device_api_metrics" }
   become: yes

ansible/artifacts-download.yml

@@ -3,8 +3,44 @@
   become: yes
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}"
-    AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}"
-  roles:
-    - artifacts-download-azure
+  tasks:
+    - name: Set GCP environment variables
+      set_fact:
+        gcp_environment:
+          GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcp_service_account.json"
+      tags: gcp
+
+    - name: Create temporary directory for GCP service account file
+      file:
+        path: /tmp/gcp_service_account.json
+        state: touch
+      tags: gcp
+
+    - name: Write GCP service account file
+      copy:
+        content: "{{ gcp_artifcats_upload_service_account }}"
+        dest: /tmp/gcp_service_account.json
+        mode: "0600"
+      tags: gcp
+
+    - name: Authenticate gcloud login
+      shell: gcloud auth activate-service-account --key-file /tmp/gcp_service_account.json
+      tags: gcp
+
+    - name: Ensure Google Cloud Storage bucket exists
+      shell: gsutil mb gs://{{ artifacts_container }} || true
+      environment: "{{ gcp_environment }}"
+      tags: gcp
+
+    - name: Download from Google Cloud Storage
+      shell: gsutil cp gs://{{ artifacts_container }}/{{ artifact }} {{ artifact_path }}
+      async: 3600
+      poll: 10
+      tags: gcp
+
+    - name: Remove temporary GCP service account file
+      file:
+        path: /tmp/gcp_service_account.json
+        state: absent
+      tags: gcp
+

ansible/artifacts-upload.yml

@@ -3,8 +3,13 @@
   become: yes
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}"
-    AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}"
   roles:
-    - artifacts-upload-azure
+    - role: artifacts-upload-gcp
+      vars:
+        service_account: "{{ gcp_artifcats_upload_service_account }}"
+      tags: gcp
+    - role: artifacts-upload-azure
+      environment:
+        AZURE_STORAGE_ACCOUNT: "{{ sunbird_artifact_storage_account_name }}"
+        AZURE_STORAGE_SAS_TOKEN: "{{ sunbird_artifact_storage_account_sas }}"
+      tags: azure

ansible/azure-hdinsight-spark.provision.yml

@@ -0,0 +1,18 @@
+- hosts: local
+  become: yes
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  roles:
+    - azure-hdinsight-spark-cluster
+  tags:
+    - copy-script
+
+- hosts: spark-hdinsight-cluster
+  become: yes
+  gather_facts: no
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  roles:
+    - provision-azure-spark-cluster
+  tags:
+    - spark-provision

ansible/cassandra_provision.yml

@@ -3,5 +3,5 @@
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
   roles:
-   - jre-1.8.0_144
+   - openjdk
    - cassandra-new

ansible/dp_kafka_indexer.yml

@@ -10,7 +10,7 @@
       become: yes
       tags: kafka-indexer
   roles:
-    - jdk-1.8.0_121
+    - openjdk
     - ep_logstash
   tags:
     - kafka-indexer

ansible/dp_kafka_provision.yml

@@ -1,22 +1,8 @@
 ---
 - hosts: "{{remote}}"
-  pre_tasks:
-    - service: name=monit state=stopped
-      become: yes
-      ignore_errors: true
-    - name: Uncompressing and copying to system path
-      unarchive:
-        src: https://sunbirdpublic.blob.core.windows.net/installation/jre-8u144-linux-x64.tar.gz
-        dest: /opt/
-        remote_src: yes
-        keep_newer: yes
-      become: yes
-      register: tar
-    - name: Updating java jre to system path
-      become: yes
-      shell: update-alternatives --install /usr/bin/java java /opt/jre1.8.0_144/bin/java 9999
   become: yes
   roles:
+    - openjdk
     - ansible-kafka-upgrade
   post_tasks:
    - service: name=monit state=started

ansible/dp_kafka_setup.yml

@@ -4,7 +4,7 @@
   pre_tasks:
     - name: Registering kafka_id
       set_fact:
-        kafka_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
+        kafka_id: "{% for servername in ansible_play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
     - name: Print kafka_id
       debug:
         var: kafka_id

ansible/dp_zookeeper_provision.yml

@@ -1,14 +1,14 @@
 ---
 - hosts: "{{remote}}"
-  pre_tasks:
-    - service: name=monit state=stopped
-      become: yes
-      ignore_errors: true
+  # pre_tasks:
+  #   - service: name=monit state=stopped
+  #     become: yes
+  #     ignore_errors: true
   become: yes
   roles:
-    - jdk-1.8.0_121
+    - openjdk
     - zookeeper-upgrade
-  post_tasks:
-    - service: name=monit state=started
-      become: yes
-      ignore_errors: true
+  # post_tasks:
+  #   - service: name=monit state=started
+  #     become: yes
+  #     ignore_errors: true

ansible/druid-ingestion.yml

@@ -1,6 +1,54 @@
 ---
+- hosts: localhost
+  tasks:
+   - name: set_fact
+     set_fact:
+       single_cluster: true
+     tags: populate_var
 
-- hosts: overlord
+   - name: check for hosts
+     set_fact:
+       single_cluster: false
+     when: "'rollup-overlord' in groups"
+     tags: populate_var
+
+- hosts: rollup-overlord
+  become: yes
+  roles:
+    - role: druid-ingestion
+      vars:
+        ingestion_task_names: "{{ rollup_ingestion_task_names }}"
+        overlord_host: "{{ rollup_overlord_host }}"
+      when: not hostvars['localhost']['single_cluster']
+
+- hosts: raw-overlord
+  become: yes
+  roles:
+    - role: druid-ingestion
+      vars:
+        ingestion_task_names: "{{ raw_ingestion_task_names }}"
+        overlord_host: "{{ raw_overlord_host }}"
+      when: not hostvars['localhost']['single_cluster']
+    - role: druid-ingestion
+      vars:
+        ingestion_task_names: "{{ raw_ingestion_task_names }}"
+        overlord_host: "{{ raw_overlord_host }}"
+      when: hostvars['localhost']['single_cluster']
+  tags:
+     - druid
+
+- hosts: ml-raw-overlord
   become: yes
   roles:
-     - druid-ingestion
+    - role: ml-druid-ingestion
+      vars:
+        ingestion_task_names: "{{ raw_ingestion_task_names }}"
+        overlord_host: "{{ ml_raw_overlord_host }}"
+      when: not hostvars['localhost']['single_cluster']
+    - role: ml-druid-ingestion
+      vars:
+        ingestion_task_names: "{{ raw_ingestion_task_names }}"
+        overlord_host: "{{ ml_raw_overlord_host }}"
+      when: hostvars['localhost']['single_cluster']
+  tags:
+     - ml-druid

ansible/druid-provision.yml

@@ -1,56 +1,32 @@
 -
   name: Install Java
-  hosts: druid-raw
+  hosts: "druid-{{remote}}"
   become: yes
-  roles: 
-    - jdk-1.8.0_121
+  roles:
+    - {role: openjdk, when: "'java' in deploy"}
 
 -
   name: Install java and zookeeper cluster for druid
-  hosts: raw-zookeeper
+  hosts: "{{ remote }}-zookeeper"
   become: yes
   roles:
-    - {role: zookeeper-upgrade}
+    - {role: zookeeper-upgrade, when: "'zookeeper' in deploy"}
   vars:
-    zookeeper_group: "{{ groups['raw-zookeeper'] }}"
-
-#-
-#  name: Install postgres on druid cluster
-#  hosts: druid-postgres
-#  vars_files:
-#    - "{{inventory_dir}}/secrets.yml"
-#  become: yes
-#  roles:
-#    - {role: postgres-provision }
-#  vars:
-#    postgres_address_space: "0.0.0.0/0"
-#    postgresql_databases:
-#      - name: druid
-#      - name: graphite
-#    postgresql_users:
-#      - name: druid
-#        password: "{{ dp_vault_druid_postgress_pass }}"
-#      - name: graphite 
-#        password: "{{ dp_vault_druid_postgress_graphite_pass }}"
-
+    zookeeper_group: "{{ groups[remote+'-zookeeper'] }}"
 
 # Create postgres databases and users
 
-- hosts: local
+- hosts: "{{ groups['postgres'][0] }}"
   gather_facts: no
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
   vars:
     postgresql_databases:
       - name: druid
-      - name: graphite
     postgresql_users:
       - name: druid
         db: druid
         password: "{{ dp_vault_druid_postgress_pass }}"
-      - name: graphite
-        db: graphite
-        password: "{{ dp_vault_druid_postgress_graphite_pass }}"     
   tasks:
     - name: Ensure database is created
       postgresql_db: name="{{ item.name }}" \
@@ -75,24 +51,15 @@
                   db="{{ item.db }}"
       with_items: "{{ postgresql_users }}"
 
--
-  name : Install graphite on druid
-  hosts: raw-graphite
-  become: yes
-  roles:
-    - {role: druid_graphite}
-
 -
   name: Install role druid
-  hosts: druid-raw
+  hosts: "druid-{{remote}}"
   become: yes
   roles:
-    - {role: analytics-druid, when: "'raw-coordinator' in group_names", druid_role: 'coordinator', service: 'coordinator' }
-    - {role: analytics-druid, when: "'raw-overlord' in group_names", druid_role: 'overlord', service: 'overlord'}
-    - {role: analytics-druid, when: "'raw-broker' in group_names", druid_role: 'broker', service: 'broker'}
-    - {role: analytics-druid, when: "'raw-historical' in group_names", druid_role: 'historical', service: 'historical'}
-    - {role: analytics-druid, when: "'raw-middlemanager' in group_names", druid_role: 'middlemanager', service: 'middleManager' }
-  vars:
-    druid_postgres_db: "druid"
-    druid_zookeeper_host: "{{ groups['raw-zookeeper']|join(':2181,')}}:2181"
-    druid_graphite_host: "{{ groups['raw-graphite'][0] }}"
+    - {role: analytics-druid, when: "'{{ remote }}-coordinator' in group_names and 'coordinator' in deploy", druid_role: 'coordinator', service: 'coordinator' }
+    - {role: analytics-druid, when: "'{{ remote }}-overlord' in group_names and 'overlord' in deploy", druid_role: 'overlord', service: 'overlord'}
+    - {role: analytics-druid, when: "'{{ remote }}-broker' in group_names and 'broker' in deploy", druid_role: 'broker', service: 'broker'}
+    - {role: analytics-druid, when: "'{{ remote }}-historical' in group_names and 'historical' in deploy", druid_role: 'historical', service: 'historical'}
+    - {role: analytics-druid, when: "'{{ remote }}-middlemanager' in group_names and 'mm' in deploy", druid_role: 'middlemanager', service: 'middleManager' }
+    - {role: analytics-druid, when: "'{{ remote }}-router' in group_names and 'router' in deploy", druid_role: 'router', service: 'router' }
+