diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f37b3b1e9..8964490e3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,8 @@ version: 2.1 jobs: datapipeline-build: working_directory: ~/dp/data-pipeline-flink - machine: true + machine: + image: ubuntu-2004:202008-01 steps: - checkout: path: ~/dp @@ -22,4 +23,4 @@ workflows: build_and_test: jobs: - datapipeline-build -#testing quality gate \ No newline at end of file +#testing quality gate diff --git a/.gitignore b/.gitignore index d222fe92b9..ad85201ce2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ database.yml .idea tmp/ kibana-auth/node_modules/ +kubernetes/helm_charts/*/values.yaml # DevOps *.backup @@ -28,3 +29,4 @@ target/ *.iml **/.settings dependency-reduced-pom.xml +/pom.xml \ No newline at end of file diff --git a/DataPipeline_HighLevel_Diagram.png b/DataPipeline_HighLevel_Diagram.png deleted file mode 100644 index 264a323825..0000000000 Binary files a/DataPipeline_HighLevel_Diagram.png and /dev/null differ diff --git a/ansible/analyticsapi_logstash.yml b/ansible/analyticsapi_logstash.yml index 669fa91368..eed2fa7226 100644 --- a/ansible/analyticsapi_logstash.yml +++ b/ansible/analyticsapi_logstash.yml @@ -9,6 +9,6 @@ apt: update_cache=yes cache_valid_time={{ apt_cache_valid_time }} become: yes roles: - - jdk-1.8.0_121 + - openjdk - { role: ep_logstash, logstash_conf: ".indexer.device_api_metrics" } become: yes diff --git a/ansible/artifacts-download.yml b/ansible/artifacts-download.yml index e75c4a941a..cd6c52307e 100644 --- a/ansible/artifacts-download.yml +++ b/ansible/artifacts-download.yml @@ -3,8 +3,44 @@ become: yes vars_files: - "{{inventory_dir}}/secrets.yml" - environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}" - AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}" - roles: - - artifacts-download-azure + tasks: + - name: Set GCP environment variables + set_fact: + gcp_environment: + GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcp_service_account.json" + tags: gcp + + - name: Create temporary directory for GCP service account file + file: + path: /tmp/gcp_service_account.json + state: touch + tags: gcp + + - name: Write GCP service account file + copy: + content: "{{ gcp_artifcats_upload_service_account }}" + dest: /tmp/gcp_service_account.json + mode: "0600" + tags: gcp + + - name: Authenticate gcloud login + shell: gcloud auth activate-service-account --key-file /tmp/gcp_service_account.json + tags: gcp + + - name: Ensure Google Cloud Storage bucket exists + shell: gsutil mb gs://{{ artifacts_container }} || true + environment: "{{ gcp_environment }}" + tags: gcp + + - name: Download from Google Cloud Storage + shell: gsutil cp gs://{{ artifacts_container }}/{{ artifact }} {{ artifact_path }} + async: 3600 + poll: 10 + tags: gcp + + - name: Remove temporary GCP service account file + file: + path: /tmp/gcp_service_account.json + state: absent + tags: gcp + diff --git a/ansible/artifacts-upload.yml b/ansible/artifacts-upload.yml index 41ef0387ef..36592e5f57 100644 --- a/ansible/artifacts-upload.yml +++ b/ansible/artifacts-upload.yml @@ -3,8 +3,13 @@ become: yes vars_files: - "{{inventory_dir}}/secrets.yml" - environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}" - AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}" roles: - - artifacts-upload-azure + - role: artifacts-upload-gcp + vars: + service_account: "{{ gcp_artifcats_upload_service_account }}" + tags: gcp + - role: artifacts-upload-azure + environment: + AZURE_STORAGE_ACCOUNT: "{{ sunbird_artifact_storage_account_name }}" + AZURE_STORAGE_SAS_TOKEN: "{{ sunbird_artifact_storage_account_sas }}" + tags: azure \ No newline at end of file diff --git a/ansible/azure-hdinsight-spark.provision.yml b/ansible/azure-hdinsight-spark.provision.yml new file mode 100644 index 0000000000..3cc166daf3 --- /dev/null +++ b/ansible/azure-hdinsight-spark.provision.yml @@ -0,0 +1,18 @@ +- hosts: local + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + roles: + - azure-hdinsight-spark-cluster + tags: + - copy-script + +- hosts: spark-hdinsight-cluster + become: yes + gather_facts: no + vars_files: + - "{{inventory_dir}}/secrets.yml" + roles: + - provision-azure-spark-cluster + tags: + - spark-provision diff --git a/ansible/cassandra_provision.yml b/ansible/cassandra_provision.yml index c24524db01..b08bf18377 100644 --- a/ansible/cassandra_provision.yml +++ b/ansible/cassandra_provision.yml @@ -3,5 +3,5 @@ vars_files: - "{{inventory_dir}}/secrets.yml" roles: - - jre-1.8.0_144 + - openjdk - cassandra-new \ No newline at end of file diff --git a/ansible/dp_kafka_indexer.yml b/ansible/dp_kafka_indexer.yml index edb29d415b..58f56bbebe 100644 --- a/ansible/dp_kafka_indexer.yml +++ b/ansible/dp_kafka_indexer.yml @@ -10,7 +10,7 @@ become: yes tags: kafka-indexer roles: - - jdk-1.8.0_121 + - openjdk - ep_logstash tags: - kafka-indexer diff --git a/ansible/dp_kafka_provision.yml b/ansible/dp_kafka_provision.yml index 1f4ee06a44..1da918cac5 100644 --- a/ansible/dp_kafka_provision.yml +++ b/ansible/dp_kafka_provision.yml @@ -1,22 +1,8 @@ --- - hosts: "{{remote}}" - pre_tasks: - - service: name=monit state=stopped - become: yes - ignore_errors: true - - name: Uncompressing and copying to system path - unarchive: - src: https://sunbirdpublic.blob.core.windows.net/installation/jre-8u144-linux-x64.tar.gz - dest: /opt/ - remote_src: yes - keep_newer: yes - become: yes - register: tar - - name: Updating java jre to system path - become: yes - shell: update-alternatives --install /usr/bin/java java /opt/jre1.8.0_144/bin/java 9999 become: yes roles: + - openjdk - ansible-kafka-upgrade post_tasks: - service: name=monit state=started diff --git a/ansible/dp_kafka_setup.yml b/ansible/dp_kafka_setup.yml index c48a5c4645..a5ea2ccc0f 100644 --- a/ansible/dp_kafka_setup.yml +++ b/ansible/dp_kafka_setup.yml @@ -4,7 +4,7 @@ pre_tasks: - name: Registering kafka_id set_fact: - kafka_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}" + kafka_id: "{% for servername in ansible_play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}" - name: Print kafka_id debug: var: kafka_id diff --git a/ansible/dp_yarn_provision.yml b/ansible/dp_yarn_provision.yml deleted file mode 100644 index 28225e9c8a..0000000000 --- a/ansible/dp_yarn_provision.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- hosts: yarn - gather_facts: false - become: true - tasks: - - name: Installaing python - raw: bash -c "test -e /usr/bin/python" || (apt update && apt install python -y) - -- hosts: yarn - become: yes - vars_files: - - "{{inventory_dir}}/secrets.yml" - tasks: - - name: Create group - group: - name: hadoop - state: present - - name: Create user - user: - name: hduser - comment: "hduser" - group: hadoop - groups: sudo - shell: /bin/bash - -- name: Install samza job server - hosts: "yarn-master" - become: yes - vars_files: - - "{{inventory_dir}}/secrets.yml" - roles: - - jdk-1.8.0_121 - - yarn - - samza-job-server - -- name: Install java on all yarn slaves - hosts: "yarn-slave" - become: yes - vars_files: - - "{{inventory_dir}}/secrets.yml" - remote_user: hduser - roles: - - jdk-1.8.0_121 diff --git a/ansible/dp_zookeeper_provision.yml b/ansible/dp_zookeeper_provision.yml index 7d5af4f0d5..68e163b9b2 100644 --- a/ansible/dp_zookeeper_provision.yml +++ b/ansible/dp_zookeeper_provision.yml @@ -1,14 +1,14 @@ --- - hosts: "{{remote}}" - pre_tasks: - - service: name=monit state=stopped - become: yes - ignore_errors: true + # pre_tasks: + # - service: name=monit state=stopped + # become: yes + # ignore_errors: true become: yes roles: - - jdk-1.8.0_121 + - openjdk - zookeeper-upgrade - post_tasks: - - service: name=monit state=started - become: yes - ignore_errors: true + # post_tasks: + # - service: name=monit state=started + # become: yes + # ignore_errors: true diff --git a/ansible/druid-ingestion.yml b/ansible/druid-ingestion.yml index bff64e86a4..bc1eba7a34 100644 --- a/ansible/druid-ingestion.yml +++ b/ansible/druid-ingestion.yml @@ -1,6 +1,54 @@ --- +- hosts: localhost + tasks: + - name: set_fact + set_fact: + single_cluster: true + tags: populate_var -- hosts: overlord + - name: check for hosts + set_fact: + single_cluster: false + when: "'rollup-overlord' in groups" + tags: populate_var + +- hosts: rollup-overlord + become: yes + roles: + - role: druid-ingestion + vars: + ingestion_task_names: "{{ rollup_ingestion_task_names }}" + overlord_host: "{{ rollup_overlord_host }}" + when: not hostvars['localhost']['single_cluster'] + +- hosts: raw-overlord + become: yes + roles: + - role: druid-ingestion + vars: + ingestion_task_names: "{{ raw_ingestion_task_names }}" + overlord_host: "{{ raw_overlord_host }}" + when: not hostvars['localhost']['single_cluster'] + - role: druid-ingestion + vars: + ingestion_task_names: "{{ raw_ingestion_task_names }}" + overlord_host: "{{ raw_overlord_host }}" + when: hostvars['localhost']['single_cluster'] + tags: + - druid + +- hosts: ml-raw-overlord become: yes roles: - - druid-ingestion + - role: ml-druid-ingestion + vars: + ingestion_task_names: "{{ raw_ingestion_task_names }}" + overlord_host: "{{ ml_raw_overlord_host }}" + when: not hostvars['localhost']['single_cluster'] + - role: ml-druid-ingestion + vars: + ingestion_task_names: "{{ raw_ingestion_task_names }}" + overlord_host: "{{ ml_raw_overlord_host }}" + when: hostvars['localhost']['single_cluster'] + tags: + - ml-druid \ No newline at end of file diff --git a/ansible/druid-provision.yml b/ansible/druid-provision.yml index 031f1d8fe5..74cab6b7c3 100644 --- a/ansible/druid-provision.yml +++ b/ansible/druid-provision.yml @@ -1,56 +1,32 @@ - name: Install Java - hosts: druid-raw + hosts: "druid-{{remote}}" become: yes - roles: - - jdk-1.8.0_121 + roles: + - {role: openjdk, when: "'java' in deploy"} - name: Install java and zookeeper cluster for druid - hosts: raw-zookeeper + hosts: "{{ remote }}-zookeeper" become: yes roles: - - {role: zookeeper-upgrade} + - {role: zookeeper-upgrade, when: "'zookeeper' in deploy"} vars: - zookeeper_group: "{{ groups['raw-zookeeper'] }}" - -#- -# name: Install postgres on druid cluster -# hosts: druid-postgres -# vars_files: -# - "{{inventory_dir}}/secrets.yml" -# become: yes -# roles: -# - {role: postgres-provision } -# vars: -# postgres_address_space: "0.0.0.0/0" -# postgresql_databases: -# - name: druid -# - name: graphite -# postgresql_users: -# - name: druid -# password: "{{ dp_vault_druid_postgress_pass }}" -# - name: graphite -# password: "{{ dp_vault_druid_postgress_graphite_pass }}" - + zookeeper_group: "{{ groups[remote+'-zookeeper'] }}" # Create postgres databases and users -- hosts: local +- hosts: "{{ groups['postgres'][0] }}" gather_facts: no vars_files: - "{{inventory_dir}}/secrets.yml" vars: postgresql_databases: - name: druid - - name: graphite postgresql_users: - name: druid db: druid password: "{{ dp_vault_druid_postgress_pass }}" - - name: graphite - db: graphite - password: "{{ dp_vault_druid_postgress_graphite_pass }}" tasks: - name: Ensure database is created postgresql_db: name="{{ item.name }}" \ @@ -75,24 +51,15 @@ db="{{ item.db }}" with_items: "{{ postgresql_users }}" -- - name : Install graphite on druid - hosts: raw-graphite - become: yes - roles: - - {role: druid_graphite} - - name: Install role druid - hosts: druid-raw + hosts: "druid-{{remote}}" become: yes roles: - - {role: analytics-druid, when: "'raw-coordinator' in group_names", druid_role: 'coordinator', service: 'coordinator' } - - {role: analytics-druid, when: "'raw-overlord' in group_names", druid_role: 'overlord', service: 'overlord'} - - {role: analytics-druid, when: "'raw-broker' in group_names", druid_role: 'broker', service: 'broker'} - - {role: analytics-druid, when: "'raw-historical' in group_names", druid_role: 'historical', service: 'historical'} - - {role: analytics-druid, when: "'raw-middlemanager' in group_names", druid_role: 'middlemanager', service: 'middleManager' } - vars: - druid_postgres_db: "druid" - druid_zookeeper_host: "{{ groups['raw-zookeeper']|join(':2181,')}}:2181" - druid_graphite_host: "{{ groups['raw-graphite'][0] }}" + - {role: analytics-druid, when: "'{{ remote }}-coordinator' in group_names and 'coordinator' in deploy", druid_role: 'coordinator', service: 'coordinator' } + - {role: analytics-druid, when: "'{{ remote }}-overlord' in group_names and 'overlord' in deploy", druid_role: 'overlord', service: 'overlord'} + - {role: analytics-druid, when: "'{{ remote }}-broker' in group_names and 'broker' in deploy", druid_role: 'broker', service: 'broker'} + - {role: analytics-druid, when: "'{{ remote }}-historical' in group_names and 'historical' in deploy", druid_role: 'historical', service: 'historical'} + - {role: analytics-druid, when: "'{{ remote }}-middlemanager' in group_names and 'mm' in deploy", druid_role: 'middlemanager', service: 'middleManager' } + - {role: analytics-druid, when: "'{{ remote }}-router' in group_names and 'router' in deploy", druid_role: 'router', service: 'router' } + diff --git a/ansible/druid-rollup-provision.yml b/ansible/druid-rollup-provision.yml index 18ef3e4b88..55e41a3bde 100644 --- a/ansible/druid-rollup-provision.yml +++ b/ansible/druid-rollup-provision.yml @@ -2,8 +2,8 @@ name: Install Java hosts: druid-rollup become: yes - roles: - - jdk-1.8.0_121 + roles: + - openjdk - name: Install java and zookeeper cluster for druid @@ -30,7 +30,7 @@ # postgresql_users: # - name: druid # password: "{{ dp_vault_druid_postgress_pass }}" -# - name: graphite +# - name: graphite # password: "{{ dp_vault_druid_postgress_graphite_pass }}" @@ -50,7 +50,7 @@ password: "{{ dp_vault_druid_postgress_pass }}" - name: graphite db: graphite-rollup - password: "{{ dp_vault_druid_postgress_graphite_pass }}" + password: "{{ dp_vault_druid_postgress_graphite_pass }}" tasks: - name: Ensure database is created postgresql_db: name="{{ item.name }}" \ @@ -75,7 +75,7 @@ db="{{ item.db }}" with_items: "{{ postgresql_users }}" -- +- name : Install graphite on druid hosts: druid-rollup-graphite become: yes @@ -91,5 +91,6 @@ - {role: analytics-druid, when: "'druid-rollup-broker' in group_names", druid_role: 'broker', service: 'broker'} - {role: analytics-druid, when: "'druid-rollup-historical' in group_names", druid_role: 'historical', service: 'historical'} - {role: analytics-druid, when: "'druid-rollup-middlemanager' in group_names", druid_role: 'middlemanager', service: 'middleManager' } + - {role: analytics-druid, when: "'druid-rollup-router' in group_names", druid_role: 'router', service: 'router' } vars: druid_postgres_db: "druid-rollup" diff --git a/ansible/druid-run-report.yml b/ansible/druid-run-report.yml index 9e1bf9a51d..a774d51a52 100644 --- a/ansible/druid-run-report.yml +++ b/ansible/druid-run-report.yml @@ -1,6 +1,8 @@ - name: Run Druid Job Report hosts: spark + vars_files: + - "{{inventory_dir}}/secrets.yml" become: yes roles: - analytics-run-report diff --git a/ansible/druid-summary-monthly-ingestion.yml b/ansible/druid-summary-monthly-ingestion.yml new file mode 100644 index 0000000000..0674884cc0 --- /dev/null +++ b/ansible/druid-summary-monthly-ingestion.yml @@ -0,0 +1,5 @@ +--- +- hosts: rollup-overlord + become: yes + roles: + - druid-summary-monthly-ingestion diff --git a/ansible/druid_anomaly_detection.yml b/ansible/druid_anomaly_detection.yml new file mode 100644 index 0000000000..db10993b06 --- /dev/null +++ b/ansible/druid_anomaly_detection.yml @@ -0,0 +1,4 @@ +- hosts: spark + become: yes + roles: + - druid-anomaly-detection \ No newline at end of file diff --git a/ansible/es_telemetry_cluster_setup.yml b/ansible/es_telemetry_cluster_setup.yml deleted file mode 100644 index bdc2998826..0000000000 --- a/ansible/es_telemetry_cluster_setup.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- hosts: telemetry-search-cluster - name: Setup telemetry ES cluster - remote_user: ecosystem - become: yes - pre_tasks: - - name: Uncompressing and copying to system path - unarchive: - src: https://sunbirdpublic.blob.core.windows.net/installation/jre-8u144-linux-x64.tar.gz - dest: /opt/ - remote_src: yes - keep_newer: yes - become: yes - register: tar - - name: Updating java jre to system path - become: yes - shell: update-alternatives --install /usr/bin/java java /opt/jre1.8.0_144/bin/java 9999 - - name: Registering node name - set_fact: - es_instance_name: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}es-node-{{ loop.index }}{% endif %}{% endfor %}" - - roles: - - { role: es6, - es_config: { - cluster.name: "elasticsearch-telemetry-search", - discovery.zen.ping.unicast.hosts: "{{ groups['telemetry-search-cluster'] }}", - http.port: 9200, - transport.tcp.port: 9300, - node.data: "{{ es_etc_node_data | default('true') }}", - node.master: "{{ es_etc_node_master | default('true') }}", - bootstrap.memory_lock: true, - }, - es_heap_size: "2g", - es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['telemetry-search-cluster'] }}", - es_etc_cluster_name: "elasticsearch_telemetry_search" - } diff --git a/ansible/graphite-exporter.yml b/ansible/graphite-exporter.yml new file mode 100644 index 0000000000..7c4d99edad --- /dev/null +++ b/ansible/graphite-exporter.yml @@ -0,0 +1,6 @@ +- hosts: druid-raw + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + roles: + - graphite-exporter \ No newline at end of file diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index dffa6d3877..951c18580e 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -1,4 +1,3 @@ -###################### DP ############################ # Not used metrics_search_params_bucket: "dev-data-store" log4j_appender_kafka_topic: "{{env}}.telemetry.backend" @@ -8,46 +7,27 @@ log4j_appender_kafka_topic: "{{env}}.telemetry.backend" # What's the implication # Is it azure blob or s3 bucket?? channel_data_exhaust_bucket: dev-data-store - - -#azure_account_name: "{{dp_azure_account_name}}" secrets_path: '{{inventory_dir}}/secrets.yml' artifacts_container: "{{dp_vault_artifacts_container}}" -#azure_storage_secret: "{{sunbird_private_storage_account_key}}" - -report_azure_account_name: "{{sunbird_private_storage_account_name}}" -report_azure_storage_secret: "{{sunbird_private_storage_account_key}}" -#azure_storage_secret: "{{dp_vault_azure_account_key}}" -#monasca_log_level: ERROR -resourcemanager: "{{ groups['yarn-master'][0] }}" -yarn_slaves: "{{ groups['yarn-slave']}}" -monitor_yarn_url: "http://{{ resourcemanager }}:8088/ws/v1/cluster/apps?states=RUNNING" - -# Artifact upload and download vars -#artifact_azure_account_name: "{{dp_azure_account_name}}" -#artifact_azure_account_key: "{{dp_vault_azure_account_key}}" -# Artifact upload and download vars -#artifact_azure_account_name: "{{dp_azure_account_name}}" -#artifact_azure_account_key: "{{dp_vault_azure_account_key}}" +report_azure_account_name: "{{sunbird_private_storage_account_name | default('')}}" +report_azure_storage_secret: "{{sunbird_private_storage_account_key | default('')}}" redis_host: "{{ groups['redis'][0] }}" metadata_redis_host: "{{ groups['redis'][0] }}" -searchServiceAuthorizationToken: "{{ dp_search_service_authorization_token }}" - +metadata2_redis_host: "{{ groups['redis'][0] }}" +searchServiceAuthorizationToken: "{{ sunbird_api_auth_token }}" # Learning Service organization and location search endpoints channelSearchServiceEndpoint: "{{ proto}}://{{domain_name}}/api/org/v1/search" locationSearchServiceEndpoint: "{{ proto}}://{{domain_name}}/api/data/v1/location/search" -__yarn_host__: "{{ groups['yarn-master'][0] }}" -search_service_endpoint: "{{proto}}://{{domain_name}}/api/search/v2/search" -yarn_es_hosts: "{{ groups['telemetry-search-cluster'] | join(',') }}" env_name: "{{env}}" kafka_brokers: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" zookeepers: "{{groups['zookeeper']|join(':2181,')}}:2181" # Ingestion cluster -ingestion_kafka_brokers: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" +ingestion_kafka_brokers: "{{groups['ingestion-cluster-kafka']|join(':9092,')}}:9092" +secor_ingestion_kafka_brokers: "{{groups['processing-cluster-kafka']|join(',')}}" ingestion_zookeepers: "{{groups['zookeeper']|join(':2181,')}}:2181" telemetry_schema_directory: /etc/{{env}}/telemetry @@ -56,29 +36,23 @@ schema_repo_url: https://github.com/project-sunbird/sunbird-data-pipeline.git # Create learningall group with LP ips cassandra_host: "{{ groups['cassandra'][0] }}" core_cassandra_host: "{{ groups['core-cassandra'][0] }}" -content_to_vec_url: http://{{ groups['analytics-api'][0] }}:9000/content-to-vec - -kibana_base_path : "/pipeline-dashboard" +lp_cassandra_host: "{{ groups['lp-cassandra'][0] }}" -################ LPA ########################### +report_cassandra_host: "{{ groups['report-cassandra'][0] }}" analytics_user: analytics analytics_group: analytics -#azure_storage_account: "{{dp_azure_account_name}}" -#azure_storage_key: "{{dp_vault_azure_account_key}}" - - analytics_user_home: /home/{{ analytics_user }} sbin_path: "{{ analytics_user_home }}/sbin" - # Secor vars secor: - properties: ['secor.azure', 'secor.common', 'secor', 'secor.partition', 'log4j'] + properties: ['secor.common', 'secor', 'secor.partition', 'log4j'] artifact_dir: /mount/secor + artifact_ver: "0.25" azure: - account_name: "{{sunbird_private_storage_account_name}}" - account_key: "{{sunbird_private_storage_account_key}}" + account_name: "{{sunbird_private_storage_account_name | default('')}}" + account_key: "{{sunbird_private_storage_account_key | default('')}}" container_name: "{{channel_data_exhaust_bucket}}" paths: ['/mount/secor', '/mount/secor/reports', '/mount/secor/logs', '/home/analytics/sbin', '/mount/data/analytics'] channel: "{{secor_alerts_slack_channel}}" @@ -89,10 +63,12 @@ secor: postgresql_databases: - name: analytics owner: analytics - + postgresql_users: - name: analytics password: "{{dp_vault_pgdb_password}}" + - name: druid + password: "{{dp_vault_druid_postgress_pass}}" postgres: db_url: "{{ groups['postgres'][0] }}" @@ -104,11 +80,9 @@ postgres: db_admin_user: analytics db_admin_password: "{{dp_vault_pgdb_admin_password}}" -postgres_address_space: 0.0.0.0/0 # Postgres trust address space - lp_composite_search: "http://{{ groups['composite-search-cluster'][0] }}:9200" # Composite Cluster ip of LP lp_composite_search_host: "{{ groups['composite-search-cluster'][0] }}" -lp_search: "http://{{ groups['search'][0] }}:9000" +lp_search: "http://{{private_ingressgateway_ip}}/search" lp_url: http://{{ groups['learning'][0] }}:8080/learning-service service: search: @@ -119,29 +93,23 @@ cassandra_hierarchy_store_prefix: "{{env}}_" data_exhaust_token: "{{dp_vault_data_exhaust_token}}" default_channel: "{{default_org_hash_id}}" -#### backup storage secret -#backup_azure_storage_account_name: "{{dp_azure_account_name}}" -#backup_azure_storage_access_key: "{{dp_vault_azure_account_key}}" - -## es backup -es_snapshot_host: "{{ groups['telemetry-search-cluster'][0] }}" -snapshot_base_path: telemetrysearch - +telemetry_default_org_id: "{{ default_org_id }}" +telemetry_frac_org_id: "{{ frac_org_id }}" # sunbird learner service sunbird_es_host: "{{groups['core-es']| join(',')}}" +single_node_es_host: "{{ groups['core-es'][0] }}" ### Druid related vars -overlord_host: "{{ groups['overlord'][0] }}" +raw_overlord_host: "{{ groups['raw-overlord'][0] }}" +ml_raw_overlord_host: "{{ groups['ml-raw-overlord'][0] }}" +rollup_overlord_host: "{{ groups['rollup-overlord'][0] }}" ### Datapipeline Custom monitoring related vars ### influxdb: "{{ groups['influxdb'][0] }}" CONTAINER_NAME_SAMZA: samza-logs script_path: /usr/local/hadoop -#### samza jobs deployment related vars #### -samza_tar_files_localpath: roles/samza-jobs/defaults - job_names: DeDuplication_1: job_file_name: 'de-duplication' @@ -172,7 +140,7 @@ job_names: ContentCacheUpdater_1: job_file_name: 'content-cache-updater' ShareEventsFlattener_1: - job_file_name: 'share-events-flattener' + job_file_name: 'share-events-flattener' druid_ingestion_specs: telemetry-events: @@ -186,7 +154,48 @@ druid_ingestion_specs: #Druid Proxy APi service sunbird_druid_broker_host: "http://{{ groups['raw-broker'][0] }}" -sunbird_learner_service_url: "{{swarm_manager_lb_ip}}:9000" +sunbird_learner_service_url: "http://{{private_ingressgateway_ip}}/learner" location_search_url: "{{ domain_name }}/api/data/" -location_search_token: "Bearer {{ dp_search_service_authorization_token }}" +location_search_token: "Bearer {{ sunbird_api_auth_token }}" + +druid_report_url_endpoint : "{{ proto}}://{{domain_name}}/api/data/v1/report/jobs" +druid_report_url : "{{ proto}}://{{domain_name}}/api/data/v1/" +druid_report_token : "Bearer {{ sunbird_api_auth_token }}" + +#redis multiprocess config +content_port: 6379 +device_port: 6380 +user_port: 6381 +dialcode_port: 6382 + +### kafka and zookeeper ip vars and being used in kafka topiccreation role. +ingestion_zookeeper_ip: "{{ groups['ingestion-cluster-zookeepers'][0] }}" +processing_zookeeper_ip: "{{ groups['processing-cluster-zookeepers'][0] }}" +### docker image pull secrets name and this will be created in kubernetes +imagepullsecrets: "{{env}}-registry-secret" +dialcode_host: "http://{{private_ingressgateway_ip}}/dial" +dialcode_endpoint: "dialcode/v3/read/" + +# s3 storage config +report_storage_type: "gs" +s3_storage_key: "{{ sunbird_private_s3_storage_key | default('') }}" +s3_storage_secret: "{{ sunbird_private_s3_storage_secret | default('') }}" +s3_storage_endpoint: "" +s3_request_signature_version: AWS4-HMAC-SHA256 +s3_path_style_access: false +s3_https_only: false +s3_default_bucket_location: "" +s3_storage_container: "" +s3_dp_storage_container: "" + + +postgres_address_space: "{{groups['postgres'][0].split(\".\")[0]}}.0.0.0/8" +postgres_exporter_user: postgres_exporter +postgresql_hba_entries: + - { type: local, database: all, user: postgres, auth_method: peer } + - { type: local, database: all, user: all, auth_method: peer } + - { type: host, database: all, user: all, address: '127.0.0.1/32', auth_method: md5 } + - { type: host, database: all, user: all, address: '::1/128', auth_method: md5 } + - { type: host, database: all, user: "{{ postgres_exporter_user }}", address: '{{ postgres_address_space }}', auth_method: md5 } + - { type: host, database: all, user: all, address: '{{ postgres_address_space }}', auth_method: md5 } diff --git a/ansible/kibana-proxy-deploy.yml b/ansible/kibana-proxy-deploy.yml deleted file mode 100644 index 7263492e85..0000000000 --- a/ansible/kibana-proxy-deploy.yml +++ /dev/null @@ -1,7 +0,0 @@ -- hosts: telemetry-search-cluster-kibana-node - become: yes - remote_user: ecosystem - roles: - - kibana-oauth-proxy - vars_files: - - "{{inventory_dir}}/secrets.yml" diff --git a/ansible/kibana_provision.yml b/ansible/kibana_provision.yml deleted file mode 100644 index 8c9dba46f9..0000000000 --- a/ansible/kibana_provision.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- hosts: telemetry-search-cluster-kibana-node - name: Setup Kibana - remote_user: ecosystem - become: yes - roles: - - kibana diff --git a/ansible/lpa_api_provision.yml b/ansible/lpa_api_provision.yml deleted file mode 100644 index f9377dc886..0000000000 --- a/ansible/lpa_api_provision.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- hosts: analytics-api - vars_files: - - "{{inventory_dir}}/secrets.yml" - become: yes - gather_facts: false - roles: - - analytics-bootstrap-always - - analytics-api-provision diff --git a/ansible/lpa_cassandra_provision.yml b/ansible/lpa_cassandra_provision.yml deleted file mode 100644 index 95df6f1c3d..0000000000 --- a/ansible/lpa_cassandra_provision.yml +++ /dev/null @@ -1,6 +0,0 @@ -- hosts: cassandra - # Install cassandra - become: yes - roles: - - analytics-bootstrap-always - - cassandra-3.7.0 diff --git a/ansible/manual-scripts/dsr-mailer-update/Playbook-redis-test.sh b/ansible/manual-scripts/dsr-mailer-update/Playbook-redis-test.sh new file mode 100644 index 0000000000..24fa3f43e2 --- /dev/null +++ b/ansible/manual-scripts/dsr-mailer-update/Playbook-redis-test.sh @@ -0,0 +1,10 @@ +rm -rf /var/lib/jenkins/ansible_play/aman-inventory/PMU_Report_Playbook/automate_data/Dsr-data/script-logs.txt +rm -rf /var/lib/jenkins/ansible_play/aman-inventory/PMU_Report_Playbook/automate_data/Dsr-data/pmu.html + +cd /var/lib/jenkins/ansible_play/aman-inventory/PMU_Report_Playbook/automate_data/Dsr-data + +/usr/bin/ansible-playbook -i inventory main-redis.yml + +#/usr/bin/ansible-playbook mail.yml +#bash mail-redis-dsr.sh +python3 mail-redis-dsr.py diff --git a/ansible/manual-scripts/dsr-mailer-update/dsr-redisscript.yml b/ansible/manual-scripts/dsr-mailer-update/dsr-redisscript.yml new file mode 100644 index 0000000000..349bac4cb9 --- /dev/null +++ b/ansible/manual-scripts/dsr-mailer-update/dsr-redisscript.yml @@ -0,0 +1,162 @@ +--- +- name: Central Ministries + set_fact: + central_ministries: 89 +- name: State Ut + set_fact: + state_ut: 36 +- name: course_publishers + command: "redis-cli -n 12 GET lp_es_live_course_publisher_count" + register: course_publishers + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for lp_es_live_course_publisher_count + set_fact: + course_publishers: "{{ course_publishers.stdout }}" + when: course_publishers is succeeded +- debug: + var: course_publishers +- name: CoursesPublished + command: "redis-cli -n 12 GET lp_es_live_course_count" + register: courses_published + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for lp_es_live_course_count + set_fact: + courses_published: "{{ courses_published.stdout }}" + when: courses_published is succeeded +- debug: + var: courses_published +- name: CourseDuration + command: "redis-cli -n 12 GET lp_es_agg_course_duration" + register: course_duration + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for lp_es_agg_course_duration + set_fact: + course_duration: "{{ course_duration.stdout }}" + when: course_duration is succeeded +- debug: + var: course_duration +- name: CourseEnrolments + command: "redis-cli -n 12 GET dashboard_enrolment_count" + register: course_enrolments + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for dashboard_enrolment_count + set_fact: + course_enrolments: "{{ course_enrolments.stdout }}" + when: course_enrolments is succeeded +- debug: + var: course_enrolments +- name: CourseCompletions + command: "redis-cli -n 12 GET dashboard_completed_count" + register: course_completions + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for dashboard_completed_count + set_fact: + course_completions: "{{ course_completions.stdout }}" + when: course_completions is succeeded +- debug: + var: course_completions +- name: newUserRegistrationsYesterday + command: "redis-cli -n 12 GET dashboard_new_users_registered_yesterday" + register: newuser_registrations_yesterday + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for newuser_registrations_yesterday + set_fact: + newuser_registrations_yesterday: "{{ newuser_registrations_yesterday.stdout }}" + when: newuser_registrations_yesterday is succeeded +- debug: + var: newuser_registrations_yesterday +- name: TotalUserCount + command: "redis-cli -n 12 GET mdo_total_registered_officer_count" + register: user_count + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for lp_es_user_count + set_fact: + user_count: "{{ user_count.stdout }}" + when: user_count is succeeded +- debug: + var: user_count + +- name: usersEnrolledinAtleastOneCourse + command: "redis-cli -n 12 GET dashboard_unique_users_enrolled_count" + register: users_enrolledin_atleast_one_course + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for users_enrolledin_atleast_one_course + set_fact: + users_enrolledin_atleast_one_course: "{{ users_enrolledin_atleast_one_course.stdout }}" + when: users_enrolledin_atleast_one_course is succeeded +- debug: + var: users_enrolledin_atleast_one_course +- name: DepartmentOrganisationsOnboarded + command: "redis-cli -n 12 GET lp_es_org_count" + register: department_organisations_onboarded + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for department_organisations_onboarded + # var: Central_Ministries_Onboarded + #var: States_UTs_Onboarded_with_registered_users + set_fact: + #Central_Ministries_Onboarded: 79 + #States_UTs_Onboarded_with_registered_users: 36 + department_organisations_onboarded: "{{ department_organisations_onboarded.stdout | int - central_ministries - state_ut }}" + when: department_organisations_onboarded is succeeded +- debug: + var: department_organisations_onboarded +- name: UserLoggedinYesterday + uri: + url: "{{ druid_sql_url }}" + method: POST + headers: + Content-Type: application/json + body_format: json + body: + resultFormat: object + header: false + context: + sqlOuterLimit: 100 + query: "SELECT COUNT(DISTINCT(actor_id)) AS UserLoggedinYesterday FROM \"telemetry-events-syncts\" WHERE eid='IMPRESSION' AND actor_type='User' AND __time >= TIME_FLOOR(CURRENT_TIMESTAMP + INTERVAL '5:30' HOUR TO MINUTE - INTERVAL '24' HOUR, 'P1D') AND __time < TIME_FLOOR(CURRENT_TIMESTAMP + INTERVAL '5:30' HOUR TO MINUTE, 'P1D')" + + register: user_loggedin_yesterday + until: user_loggedin_yesterday is not failed + retries: 10 + delay: 10 + +- debug: + var: user_loggedin_yesterday.json + +- set_fact: + user_loggedin_yesterday: "{{ user_loggedin_yesterday.json[0].UserLoggedinYesterday }}" + +- debug: + var: user_loggedin_yesterday + +- name: TotalEventsEnrolmentCount + command: "redis-cli -n 12 GET dashboard_events_enrolment_count" + register: dashboard_events_enrolment_count + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for dashboard_events_enrolment_count + set_fact: + dashboard_events_enrolment_count: "{{ dashboard_events_enrolment_count.stdout }}" + when: dashboard_events_enrolment_count is succeeded +- debug: + var: dashboard_events_enrolment_count + +- name: TotalEventsPublishedCount + command: "redis-cli -n 12 GET dashboard_events_published_count" + register: dashboard_events_published_count + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for dashboard_events_published_count + set_fact: + dashboard_events_published_count: "{{ dashboard_events_published_count.stdout }}" + when: dashboard_events_published_count is succeeded +- debug: + var: dashboard_events_published_count + +- name: TotalEventsCompletedCount + command: "redis-cli -n 12 GET dashboard_events_completed_count" + register: dashboard_events_completed_count + ignore_errors: yes # Ignore errors if the key doesn't exist +- name: Set fact for dashboard_events_completed_count + set_fact: + dashboard_events_completed_count: "{{ dashboard_events_completed_count.stdout }}" + when: dashboard_events_completed_count is succeeded +- debug: + var: dashboard_events_completed_count diff --git a/ansible/manual-scripts/dsr-mailer-update/final-report.html.j2 b/ansible/manual-scripts/dsr-mailer-update/final-report.html.j2 new file mode 100644 index 0000000000..760366834f --- /dev/null +++ b/ansible/manual-scripts/dsr-mailer-update/final-report.html.j2 @@ -0,0 +1,100 @@ + + + + + +

Dear Sir/Mam, +

Please find enclosed here with IGOT Daily Status Report Dated hrs.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ diff --git a/ansible/manual-scripts/dsr-mailer-update/mail-redis-dsr.py b/ansible/manual-scripts/dsr-mailer-update/mail-redis-dsr.py new file mode 100644 index 0000000000..4d2a80fbf0 --- /dev/null +++ b/ansible/manual-scripts/dsr-mailer-update/mail-redis-dsr.py @@ -0,0 +1,94 @@ +#!/usr/bin/python3 +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from datetime import datetime + +# Email configurations +TO = """ + S.Ramadorai@tcs.com, + secy_mop@nic.in, + secretary@meity.gov.in, + ceo.karmayogi@gov.in, + chairperson-ncvet@gov.in, + govindiyer09@gmail.com, + debjani@nasscom.in, + santrupt.misra@adityabirla.com, + pankaj@peoplestrong.com, + som.sharma@gov.in, + shyama.roy@nic.in, + nalini@gov.in, + Zachariahthomas.edu@nic.in, + pawan.joshi@nic.in, + jitesh.gupta@nic.in, + rakesh.verma@karmayogi.in, + shubho.gupta@karmayogi.in, + monojeet.chakravorty@karmayogi.in, + ap.singh@karmayogi.in, + priyanka.agarwal@karmayogi.in, + bm@keenable.in, + memberhr-cbc@gov.in, + memberadmn.cbc-dopt@gov.in, + chairperson-cbc@gov.in, + rajul.bhatt@gov.in, + anshu.aggarwal28@karmayogi.in, + nalini.nautiyal@gmail.com, + singhr11@ias.nic.in, + cfo.karmayogi@gov.in, + cto-karmayogi@gov.in, + astha.sharma@tarento.com, + ritesh.kumar80@karmayogi.in, + kumar.abhinav@gov.in, + """ +BCC = """ + mathew.pallan@tarento.com, + chandni.babu@nic.in, + amandeep.singh@fosteringlinux.com, + kunwara.jha@fosteringlinux.com, + madhu.jaswal@fosteringlinux.com, + rohtash.kumar@fosteringlinux.com, + manoj.cchounsali@fosteringlinux.com, + raju@fosteringlinux.com, + igot.operations@tarento.com, + somya.jaiswal@tarento.com, + rangabashyam.krishnamachari@tarento.com, + vikram.subramanyam@tarento.com, + devendra.maurya@fosteringlinux.com, + shobhit.vashistha@tarento.com, + himanshu.gupta@tarento.com, + wilky.singh@tarento.com, + anand.varada@tarento.com, + mohit.agarwal@tarento.com, + vijay.sunkeswari@tarento.com, + haridas.kakunje@tarento.com, + spvijay@gmail.com, + rangabashyam.pk@gmail.com, + naushad.ali@fosteringlinux.com, + igot.noc@tarento.com + """ +FROM_EMAIL = "mission.karmayogi@gov.in" +EMAIL_RELAY = "relay.nic.in" +EMAIL_SUBJECT = f"IGOT Mission Karmayogi - [ Automated DSR Dated ] - {datetime.now().strftime('%d/%m/%Y')}" + +# Read mail body from file +with open('/var/lib/jenkins/ansible_play/aman-inventory/PMU_Report_Playbook/automate_data/Dsr-data/pmu.html', 'r') as file: + mail_body = file.read() + +# Create MIME message +message = MIMEMultipart() +message['From'] = FROM_EMAIL +message['To'] = TO +message['Bcc'] = BCC +message['Subject'] = EMAIL_SUBJECT + +# Attach HTML content to the email +message.attach(MIMEText(mail_body, 'html')) + +# Connect to the SMTP server and send the email +try: + smtp_server = smtplib.SMTP(EMAIL_RELAY) + smtp_server.send_message(message) + smtp_server.quit() + print("Email sent successfully!") +except Exception as e: + print(f"Failed to send email: {e}") diff --git a/ansible/manual-scripts/dsr-mailer-update/mail-redis-dsr.sh b/ansible/manual-scripts/dsr-mailer-update/mail-redis-dsr.sh new file mode 100644 index 0000000000..da3c36b2a4 --- /dev/null +++ b/ansible/manual-scripts/dsr-mailer-update/mail-redis-dsr.sh @@ -0,0 +1,20 @@ +#!/bin/bash +MAIL=/usr/bin/swaks +TO="S.Ramadorai@tcs.com,secy_mop@nic.in,secretary@meity.gov.in,ceo.karmayogi@gov.in,chairperson-ncvet@gov.in,govindiyer09@gmail.com,debjani@nasscom.in,santrupt.misra@adityabirla.com,pankaj@peoplestrong.com,som.sharma@gov.in,shyama.roy@nic.in,nalini@gov.in,Zachariahthomas.edu@nic.in,pawan.joshi@nic.in,jitesh.gupta@nic.in,rakesh.verma@karmayogi.in,shubho.gupta@karmayogi.in,monojeet.chakravorty@karmayogi.in,ap.singh@karmayogi.in,priyanka.agarwal@karmayogi.in,bm@keenable.in,memberhr-cbc@gov.in,memberadmn.cbc-dopt@gov.in,chairperson-cbc@gov.in,rajul.bhatt@gov.in,anshu.aggarwal28@karmayogi.in,nalini.nautiyal@gmail.com,singhr11@ias.nic.in,cfo.karmayogi@gov.in,ranapratap.singh@karmayogi.in" +BCC="mathew.pallan@tarento.com,chandni.babu@nic.in,amandeep.singh@fosteringlinux.com,kunwara.jha@fosteringlinux.com,madhu.jaswal@fosteringlinux.com,rohtash.kumar@fosteringlinux.com,manoj.cchounsali@fosteringlinux.com,raju@fosteringlinux.com,igot.operations@tarento.com,somya.jaiswal@tarento.com,rangabashyam.krishnamachari@tarento.com,vikram.subramanyam@tarento.com,devendra.maurya@fosteringlinux.com,shobhit.vashistha@tarento.com,himanshu.gupta@tarento.com,wilky.singh@tarento.com,anand.varada@tarento.com,mohit.agarwal@tarento.com,vijay.sunkeswari@tarento.com,haridas.kakunje@tarento.com,spvijay@gmail.com,rangabashyam.pk@gmail.com,naushad.ali@fosteringlinux.com,astha.sharma@tarento.com,igot.noc@tarento.com" + +DATE=`date +%d/%m/%Y` +MAIL_BODY=`cat /var/lib/jenkins/ansible_play/aman-inventory/PMU_Report_Playbook/automate_data/Dsr-data/pmu.html` +EMAIL_RELAY="relay.nic.in" +################################################################################################# + + +$MAIL \ + --to "$TO,$BCC" \ + --from "mission.karmayogi@gov.in" \ + --server "$EMAIL_RELAY" \ + --timeout 300 \ + --header "To:$TO" \ + --header "Subject: IGOT Karmayogi - [ Automated DSR Dated ] - $DATE" \ + --header "Content-Type: text/html" \ + --body "$MAIL_BODY" diff --git a/ansible/manual-scripts/dsr-mailer-update/main-redis.yml b/ansible/manual-scripts/dsr-mailer-update/main-redis.yml new file mode 100644 index 0000000000..630e1b98a7 --- /dev/null +++ b/ansible/manual-scripts/dsr-mailer-update/main-redis.yml @@ -0,0 +1,29 @@ +--- + +- hosts: localhost + vars_files: + - /var/lib/jenkins/ansible_play/aman-inventory/PMU_Report_Playbook/automate_data/Dsr-data/vars/main.yml + tasks: + + - name: "DSRRedissripts" + include: dsr-redisscript.yml + delegate_to: 192.168.3.82 + tags: + - dsr-redisscript + + + - name: "TotalUserCount" + include: totaluser_count.yml + delegate_to: localhost + tags: + - totaluser_count + + - name: "generate the report in html" + template: + src: /var/lib/jenkins/ansible_play/aman-inventory/PMU_Report_Playbook/automate_data/Dsr-data/final-report.html.j2 + dest: /var/lib/jenkins/ansible_play/aman-inventory/PMU_Report_Playbook/automate_data/Dsr-data/pmu.html + owner: admin + group: users + tags: + - html + diff --git a/ansible/manual-scripts/dsr-mailer-update/pmu.html b/ansible/manual-scripts/dsr-mailer-update/pmu.html new file mode 100644 index 0000000000..dfc679ca24 --- /dev/null +++ b/ansible/manual-scripts/dsr-mailer-update/pmu.html @@ -0,0 +1,84 @@ + + + + + +

Dear Sir/Mam, +

Please find enclosed here with IGOT Daily Status Report Dated hrs.

+
S.No.Information DetailsCounts
1.Central Ministries Onboarded {{central_ministries}}
2.Department/Organisations Onboarded {{ department_organisations_onboarded }}
3.States/UTs Onboarded (with registered users) {{state_ut}}
4.Course Publishers {{ course_publishers }}
5.Courses Published {{ courses_published }}
6.Events Published {{ dashboard_events_published_count }}
7.Duration of Courses Published (Hours) {{ course_duration | float | int }}
8.Course Enrolments {{ course_enrolments }}
9.Course Completions {{ course_completions }}
10.Event Enrolments {{ dashboard_events_enrolment_count }}
11.Event Completions {{ dashboard_events_completed_count }}
12.Users Registered {{ user_count }}
13.New User Registrations yesterday {{ newuser_registrations_yesterday }}
14.User Logged in yesterday {{ user_loggedin_yesterday }}
15.Users Enrolled in at least one course {{ users_enrolledin_atleast_one_course }}
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/ansible/manual-scripts/dsr-mailer-update/script-logs.txt b/ansible/manual-scripts/dsr-mailer-update/script-logs.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ansible/manual-scripts/dsr-mailer-update/totaluser_count.yml b/ansible/manual-scripts/dsr-mailer-update/totaluser_count.yml new file mode 100644 index 0000000000..0d8dd1538e --- /dev/null +++ b/ansible/manual-scripts/dsr-mailer-update/totaluser_count.yml @@ -0,0 +1,54 @@ +--- +- name: Generate authentication token for Total User Count list + uri: + url: "{{generate_token}}" + method: POST + body_format: form-urlencoded + body: + username: "{{username}}" + password: "{{password}}" + client_id: admin-cli + grant_type: password + status_code: [200,201,308,302,401] + register: data + until: data is not failed + retries: 10 + delay: 10 + +- name: store token in a variable + set_fact: + access_token: "{{data.json.access_token}}" + +- name: TotalUserCount + uri: + url: "{{usercount_domain}}" + method: POST + headers: + Content-Type: "application/json" + authorization: "{{token}}" + x-authenticated-user-token: "{{access_token}}" + body_format: json + body: | + { + "request": { + "query": "", + "filters": { + "status":1 + }, + "limit":1 + } + } + register: total_user_count + until: total_user_count is not failed + retries: 10 + delay: 10 + + +- debug: + var: total_user_count + +- set_fact: + total_user_count: "{{ total_user_count.json.result.response.count }}" + +- debug: + var: total_user_count diff --git a/ansible/manual-scripts/dsr-mailer-update/vars/main.yml b/ansible/manual-scripts/dsr-mailer-update/vars/main.yml new file mode 100644 index 0000000000..d3b7d458e0 --- /dev/null +++ b/ansible/manual-scripts/dsr-mailer-update/vars/main.yml @@ -0,0 +1,97 @@ +state_domain: "https://portal.igotkarmayogi.gov.in/api/user/v1/search" +Org_domain: "https://portal.igotkarmayogi.gov.in/api/org/v1/search" +live_domain: "https://igotkarmayogi.gov.in/api/composite/v1/search" +publish_domain: "https://igotkarmayogi.gov.in/api/content/v1/search" +token: "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI5a04xTW1TcGVuVTAyam8zVHg1U2p0amhTOFVXeGVSUiJ9.LWAgFust4e0wntxqY8_MQjf5WQ9RSD6Hg45jX_NoCXY" +druid_sql_url: http://192.168.3.21:8888/druid/v2/sql + +generate_token: "https://portal.igotkarmayogi.gov.in/auth/realms/sunbird/protocol/openid-connect/token" +usercount_domain: "https://portal.igotkarmayogi.gov.in/api/user/v1/search?mode=edit" +total_user_count_domain: "https://portal.igotkarmayogi.gov.in/api/user/v1/search" +username: nalini@nic.in +password: Nautiyal@123 +user_output: "/roles/automate_data/files/userscount.txt" +mdo_output: "/roles/automate_data/files/mdocount.txt" +replaced: [] +org_publish_course_domain: "https://portal.igotkarmayogi.gov.in/api/composite/v1/search" +token_file_path: "/var/lib/jenkins/ansible_play/aman-inventory/PMU_Report_Playbook/automate_data/files/token.txt" + +#########################################Bugzila Variable######################################################## +bug_token: "http://bugzilla.keenable.in/bugzilla/rest/login?" +bug_domain: "https://bugzilla.keenable.in/bugzilla/rest/bug?" +userid: reema.pal@fosteringlinux.com +password1: abc@123 +classy: iGoT +listid: "4515" +queryformate: advanced +limit: "0" +bug_status: resolved + + +##########################Cassandra Variable####################################################################### +group: users +owner: jenkins +lchost: 127.0.0.1 + + +yesterday_time: "{{ lookup('pipe', 'date -d \"1 day ago\" +\"%Y-%m-%d\"') }}" + +query1: "SELECT userid FROM sunbird_courses.user_content_consumption where last_access_time >= '{{ yesterday_time }} 00:00:00+0200' AND last_access_time <= '{{ yesterday_time }} 23:59:00+0200' ALLOW FILTERING ;" +#query2: "select * from sunbird_courses.user_content_consumption limit 1;" +query2: "select distinct userid from sunbird_courses.user_enrolments ;" +query3: "select count(*) from sunbird_courses.user_content_consumption where last_access_time >= '2022-03-08' and last_access_time <= '2022-10-05' group by userid, courseid, batchid ALLOW FILTERING;" +query4: "SELECT userid FROM sunbird.user where createddate >= '{{ yesterday_time }} 00:00:00+0200' AND createddate <= '{{ yesterday_time }} 23:59:00+0200' ALLOW FILTERING ;" +#query4: "SELECT userid FROM sunbird.user where createddate >= '2022-08-19 00:00:00+0200' AND createddate <= '2022-08-19 23:59:00+0200' ALLOW FILTERING ;" +#query5: "select count(*) from sunbird_courses.user_enrolments where active=true and status=0 ALLOW FILTERING;" +query5: "select userid from sunbird_courses.user_enrolments where active=true and courseid='{{ item.identifier }}' and status=0 ALLOW FILTERING;" +#query6: "select count(*) from sunbird_courses.user_enrolments where active=true and status=1 ALLOW FILTERING;" +query6: "select userid from sunbird_courses.user_enrolments where active=true and courseid='{{ item.identifier }}' and status=1 ALLOW FILTERING;" +#query7: "select count(*) from sunbird_courses.user_enrolments where active=true and status=2 ALLOW FILTERING;" +#query7: "select count(*) from sunbird_courses.user_enrolments where active=true and courseid='{{ item.identifier }}' and status=2 ALLOW FILTERING;" +query7: "select userid from sunbird_courses.user_enrolments where active=true and courseid='{{ item.identifier }}' and status=2 ALLOW FILTERING;" + + + + +################################MYSQL Variable######################################################################### +mysql_db: frac_tool +mysql_user: root +mysql_pass: Nicigot@1234 + +competency_query: "SELECT count(*) FROM data_node where type = 'COMPETENCY' AND status = 'VERIFIED' AND is_active=1;" +role_query: "SELECT COUNT(*) FROM data_node where type = 'ROLE' AND status = 'VERIFIED' AND is_active=1;" +activity_query: "SELECT COUNT(*) FROM data_node where type = 'ACTIVITY' AND status = 'VERIFIED' AND is_active=1;" +position_query: "SELECT COUNT(*) FROM data_node where type = 'POSITION' AND status = 'VERIFIED' AND is_active=1;" + +########################################################################################################################## +postgres_db: igot_reports4 +postgres_user: postgres +#host: localhost +host: 192.168.3.204 +postgres_port: 5433 +#postgres_query: "select count(distinct(user_course_enrolment.user_id)) from user_course_enrolment where completion_status='Completed' or completion_status='In-Progress' or completion_status='Not Started' ;" +postgres_query: select count(distinct(user_course_enrolment.user_id)) from user_course_enrolment where completion_status IN ('Completed','In-Progress','Not Started') ; + + + #######################################mail Variables################################################################ + #receivers: secy-mop@nic.in,hemang.jani@gov.in,shirahatti.deepak@nic.in,nalini@gov.in,deepak.shirahatti@nic.in,pawan.joshi@nic.in,ceo-karmayogi@gov.in,arushimathur@ipeglobal.com,g.d.jayanthi@nic.in,apsingh@ipeglobal.com,Zachariahthomas.edu@nic.in,jitesh.gupta@nic.in,bharat.kumar13@gov.in,priyanka.agarwal@govcontractor.in,prachi.gupta@tarento.com,wilky.singh@tarento.com,rakesh.verma@karmayogi.in,vikas.bagri@karmayogi.in,manthan.sharma@tarento.com,s.ramadorai@tcs.com,monojeet.chakravorty@karmayogi.in,govindiyer09@gmail.com,pankaj@peoplestrong.com,bm@keenable.in + +#bcc_receivers: bm@keenable.in,chandni.babu@nic.in,amandeep.singh@fosteringlinux.com,kunwara.jha@fosteringlinux.com,madhu.jaswal@fosteringlinux.com,rohtash.kumar@fosteringlinux.com,manoj.cchounsali@fosteringlinux.com,raju@fosteringlinux.com + +#receivers: nalini@nic.in,bm@keenable.in,mayank.01@fosteringlinux.com,amandeep.singh@fosteringlinux.com +receivers: amandeep.singh@fosteringlinux.com,kunwara.jha@fosteringlinux.com +bcc_receivers: amandeep.singh@fosteringlinux.com,kunwara.jha@fosteringlinux.com,nandakishor.sudarshan@tarento.com + +###################################################################################################################### + + + +#SearchLive: "{\"locale\":[\"en\"],\"query\":\"\",\"request\":{\"query\":\"\",\"filters\":{\"status\":[\"live\"],\"contentType\":\"course\",\"primaryCategory\":\"Course\"},\"sort_by\":{\"lastUpdatedOn\":\"desc\"},\"fields\":[\"name\",\"identifier\"],\"facets\":[\"primaryCategory\",\"status\",\"mimeType\"],\"limit\":1000}}" +SearchLive: "{\"locale\":[\"en\"],\"query\":\"\",\"request\":{\"query\":\"\",\"filters\":{\"status\":[\"live\"],\"contentType\":\"course\",\"primaryCategory\":\"Course\"},\"sort_by\":{\"lastUpdatedOn\":\"desc\"},\"fields\":[\"name\",\"identifier\",\"createdOn\",\"duration\"],\"facets\":[\"primaryCategory\",\"status\",\"mimeType\"],\"limit\":2000}}" + +SearchUserBody: "{\"request\":{\"filters\":{\"email\":\"{{ user.email }}\"},\"fields\":[],\"sortBy\":{\"updatedDate\":\"Desc\"}}}" +api_key: "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI5a04xTW1TcGVuVTAyam8zVHg1U2p0amhTOFVXeGVSUiJ9.LWAgFust4e0wntxqY8_MQjf5WQ9RSD6Hg45jX_NoCXY" +domain: "https://portal.igotkarmayogi.gov.in" +auth_username: nalini@nic.in +auth_passwd: Nautiyal@123 +query99: "select count(*) from sunbird_courses.user_enrolments where active=true and courseid='{{ item.identifier }}'and status=1 ALLOW FILTERING;" diff --git a/ansible/manual-scripts/landing-page-update/automated-dashboard/configurations.json.j2 b/ansible/manual-scripts/landing-page-update/automated-dashboard/configurations.json.j2 new file mode 100644 index 0000000000..83eba99e8f --- /dev/null +++ b/ansible/manual-scripts/landing-page-update/automated-dashboard/configurations.json.j2 @@ -0,0 +1,67 @@ +{ + "portalURL": "{{ domain }}/", + "loginPageRedirection": false, + "landingPageUpdateTime": "{{ processing_time_human }}", + "karmayogiOnboarded": "{{ user_count }}", + "registeredMdo": "{{ organization_count }}", + "courses": "{{ live_course_count }}", + "availableContent": "{{ aggregated_course_duration }}", + "monthyActiveUsers": "{{ monthly_active_user_count }}", + "courseProgramCompletionCount": "{{ course_program_completion_count }}", + "courseProgramCompletionYesterdayCount": "{{ course_program_completion_in_last_24_hrs_count }}", + "courseEnrollments": "4166", + "activeUsers": "1200000", + "contentBucket": "assets/public", + "certImage": "assets/certimage/content", + "meetLink": "https://teams.microsoft.com/l/meetup-join/19%3ameeting_M2Y3ZDE2ZDMtMWQwYS00OWQzLWE3NDctNDRkNTdjOGI4Yzll%40thread.v2/0?context=%7b%22Tid%22%3a%2240cfb65c-9b71-435f-8bc2-bc2c69df1aca%22%2c%22Oid%22%3a%22cbd37bc9-5c33-401f-b590-9decb3c370f8%22%7d", + "updateData": [ + { + "english": [ + { + "updateTitle": "User Manual Karmayogi Portal", + "updateUrl": "docs/Updated_user_manual-Karmayogi_portal.docx.pdf" + }, + { + "updateTitle": "User Manual MDO Portal", + "updateUrl": "docs/updated_user_manual-MDO_Portal.docx.pdf" + }, + { + "updateTitle": "User Manual SPV Portal", + "updateUrl": "docs/updated_user_manual-SPV_Portal.docx.pdf" + }, + { + "updateTitle": "User Manual FRAC Toolkit", + "updateUrl": "docs/Updated_user_manual-FRAC_Toolkit.docx.pdf" + }, + { + "updateTitle": "Guidelines for Content Creation", + "updateUrl": "docs/Guidelines_for_Content_Creation.pdf" + } + ] + }, + { + "hindi": [ + { + "updateTitle": "उपयोगकर्ता पुस्तिका कर्मयोगी पोर्टल", + "updateUrl": "docs/Updated_user_manual-Karmayogi_portal.docx.pdf" + }, + { + "updateTitle": "उपयोगकर्ता पुस्तिका एमडीओ पोर्टल", + "updateUrl": "docs/updated_user_manual-MDO_Portal.docx.pdf" + }, + { + "updateTitle": "उपयोगकर्ता पुस्तिका एसपीवी पोर्टल", + "updateUrl": "docs/updated_user_manual-SPV_Portal.docx.pdf" + }, + { + "updateTitle": "उपयोगकर्ता पुस्तिका FRAC टूलकिट", + "updateUrl": "docs/Updated_user_manual-FRAC_Toolkit.docx.pdf" + }, + { + "updateTitle": "Content निर्माण के लिए दिशानिर्देश", + "updateUrl": "docs/Guidelines_for_Content_Creation.pdf" + } + ] + } + ] +} diff --git a/ansible/manual-scripts/landing-page-update/automated-dashboard/dashboard.html.j2 b/ansible/manual-scripts/landing-page-update/automated-dashboard/dashboard.html.j2 new file mode 100644 index 0000000000..105529b257 --- /dev/null +++ b/ansible/manual-scripts/landing-page-update/automated-dashboard/dashboard.html.j2 @@ -0,0 +1 @@ +

Hi team,

Dashboard data has been updated on the static page.

Please find the below updates.

S.No.Information DetailsCounts
1.Central Ministries Onboarded 89
2.Department/Organisations Onboarded 3824
3.States/UTs Onboarded (with registered users) 36
4.Course Publishers 90
5.Courses Published 902
6.Duration of Courses Published (Hours) 1669
7.Course Enrolments 8000035
8.Course Completions 5693730
9.Users Registered 3093700
10.New User Registrations yesterday 3072
11.User Logged in yesterday 26598
12.Users Enrolled in at least one course 945923

Sno

Dashboard Parameters

Value

1.

Total No of Users Created

"{{ user_count }}"

2.

Total No of Organization

"{{ organization_count }}"

3.

Available Content Time in Hrs

"{{ aggregated_course_duration }}"

4.

Total No of Courses Published

"{{ live_course_count }}"

\ No newline at end of file diff --git a/ansible/manual-scripts/landing-page-update/automated-dashboard/playbook.yml b/ansible/manual-scripts/landing-page-update/automated-dashboard/playbook.yml new file mode 100644 index 0000000000..8b7866e6ed --- /dev/null +++ b/ansible/manual-scripts/landing-page-update/automated-dashboard/playbook.yml @@ -0,0 +1,300 @@ +--- +- hosts: localhost + connection: local + vars_files: + - variable.yml + tasks: + + - name: User Count API call + uri: + url: "{{ es_host }}/user_v2/_search" + method: POST + headers: + Content-Type: "application/json" + body_format: json + body: "{{ usercount_body }}" + status_code: 200 + register: user_out + until: user_out is not failed + retries: 10 + delay: 2 + + - name: Count Output + set_fact: + user_count: "{{ user_out.json.hits.total }}" + failed_when: + - user_count == "0" + + - debug: + var: user_count + + - name: Total no. of Organizations API call + uri: + url: "{{ es_host }}/org_v2/_search" + method: POST + headers: + Content-Type: "application/json" + body_format: json + body: "{{ orgcount_body }}" + status_code: [201,200,302,308] + register: org_out + until: org_out is not failed + retries: 10 + delay: 2 + + - name: Organization Count + set_fact: + organization_count: "{{ org_out.json.hits.total }}" + failed_when: + - organization_count == "0" + + - debug: + var: organization_count + + - name: Total no. of Draft Courses API call + uri: + url: "{{ es_host }}/compositesearch/_search" + method: POST + headers: + Content-Type: "application/json" + body_format: json + body: "{{ draft_course_count_api_body }}" + status_code: [201,200,302,308] + register: draft_course_count_out + until: draft_course_count_out is not failed + retries: 10 + delay: 2 + + - name: Draft Course Count + set_fact: + draft_course_count: "{{ draft_course_count_out.json.hits.total }}" + + - debug: + var: draft_course_count + + - name: Total no. of Review Courses API call + uri: + url: "{{ es_host }}/compositesearch/_search" + method: POST + headers: + Content-Type: "application/json" + body_format: json + body: "{{ review_course_count_api_body }}" + status_code: [201,200,302,308] + register: review_course_count_out + until: review_course_count_out is not failed + retries: 10 + delay: 2 + + - name: Review Course Count + set_fact: + review_course_count: "{{ review_course_count_out.json.hits.total }}" + + - debug: + var: review_course_count + + - name: Total no. of Retired Courses API call + uri: + url: "{{ es_host }}/compositesearch/_search" + method: POST + headers: + Content-Type: "application/json" + body_format: json + body: "{{ retired_course_count_api_body }}" + status_code: [ 201,200,302,308 ] + register: retired_course_count_out + until: retired_course_count_out is not failed + retries: 10 + delay: 2 + + - name: Retired Course Count + set_fact: + retired_course_count: "{{ retired_course_count_out.json.hits.total }}" + + - debug: + var: retired_course_count + + - name: Total no. of Pending Publish Courses API call + uri: + url: "{{ es_host }}/compositesearch/_search" + method: POST + headers: + Content-Type: "application/json" + body_format: json + body: "{{ pending_publish_course_count_api_body }}" + status_code: [ 201,200,302,308 ] + register: pending_publish_course_count_out + until: pending_publish_course_count_out is not failed + retries: 10 + delay: 2 + + - name: Pending Publish Course Count + set_fact: + pending_publish_course_count: "{{ pending_publish_course_count_out.json.hits.total }}" + + - debug: + var: pending_publish_course_count + + - name: Live Course Query Elasticsearch and Process Response + uri: + url: "{{ es_host }}/compositesearch/_search" + method: POST + headers: + Content-Type: "application/json" + body_format: json + body: "{{ live_course_api_body }}" + status_code: [201,200,302,308] + register: live_course_output + until: live_course_output is not failed + retries: 10 + delay: 2 + + - name: Extract Document Count and Aggregate Duration + set_fact: + live_course_count: "{{ live_course_output.json.hits.total }}" + failed_when: + - live_course_count == "0" + + - name: course duration list + set_fact: + course_duration_list: "{{ live_course_output.json.hits.hits | map(attribute='_source.duration') | list }}" + + - name: unique course publisher id list + set_fact: + unique_course_publisher_id_list: "{{ live_course_output.json.hits.hits | map(attribute='_source.createdFor') | select() | sum(start=[]) | unique | select() | list }}" + + - debug: + var: unique_course_publisher_id_list + + - name: unique course publisher count + set_fact: + unique_course_publisher_count: "{{ unique_course_publisher_id_list | length }}" + + - debug: + var: unique_course_publisher_count + + - name: course id list + set_fact: + course_id_list: "{{ live_course_output.json.hits.hits | map(attribute='_source.identifier') | list | join(',') }}" + + - debug: + var: course_duration_list + + - debug: + var: course_id_list + + - name: Total Sum of Duration + set_fact: + aggregated_course_duration_float: "{{ course_duration_list | map('int') | sum / 3600 | float }}" + + - name: Total Sum of Round Off value + set_fact: + aggregated_course_duration: "{{ aggregated_course_duration_float | float | round | int }}" + failed_when: + - aggregated_course_duration == "0" + + - debug: + var: aggregated_course_duration + + - name: Display Results + debug: + msg: + - "Total Live Course Count: {{ live_course_count }}" + - "Aggregated Duration: {{ aggregated_course_duration }}" + + - debug: + msg: "{{ user_count }}, {{ organization_count }}, {{ aggregated_course_duration }}, {{ live_course_count }}" + + - name: Get IST Date + command: "date '+%Y-%m-%d %H:%M:%S'" + register: date_ist + + - name: Get IST Date ISO format + command: "date '+%Y-%m-%dT%H:%M:%SZ'" + register: date_ist_iso + + - name: Processing Time + set_fact: + processing_time: "{{ date_ist_iso.stdout }}" + + - name: Processing Time Human Format + set_fact: + processing_time_human: "{{ date_ist.stdout }}" + + - name: Redis Get Monthly Active User Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} GET {{ redis_key_lp_monthly_active_users }}" + register: redis_monthly_active_users_out + + - name: Monthly Active User Count + set_fact: + monthly_active_user_count: "{{ redis_monthly_active_users_out.stdout }}" + + - name: Redis Get Course Program Completion + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} GET {{ redis_key_dashboard_completed_count }}" + register: redis_course_program_completed_out + + - name: Course Program Completion Count + set_fact: + course_program_completion_count: "{{ redis_course_program_completed_out.stdout }}" + + - name: Redis Get Course Program Completion In Last 24 Hrs Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} GET {{ redis_key_lp_completed_yesterday_count }}" + register: redis_course_program_completed_yesterday_out + + - name: Course Program Completion In Last 24 Hrs Count + set_fact: + course_program_completion_in_last_24_hrs_count: "{{ redis_course_program_completed_yesterday_out.stdout }}" + + - name: copy file + copy: + dest: "{{ root_path }}/automated-dashboard/dashbord_history/configurations.json_{{ ansible_date_time.date }}" + src: "{{ root_path }}/html/configurations.json" + backup: yes + register: copy_file + + - name: Apply Page Template + template: + src: configurations.json.j2 + dest: "{{ root_path }}/html/configurations.json" + + - name: Redis Update Processing time + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_update_time }} {{ processing_time }}" + + - name: Redis Update User Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_user_count }} {{ user_count }}" + + - name: Redis Update Org Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_org_count }} {{ organization_count }}" + + - name: Redis Update Live Course Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_live_course_count }} {{ live_course_count }}" + + - name: Redis Update Live Course Duration + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_agg_course_duration }} {{ aggregated_course_duration }}" + + - name: Redis Update Live Course ID List + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_live_course_id_list }} {{ course_id_list }}" + + - name: Redis Update Live Course Publisher Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_live_course_publisher_count }} {{ unique_course_publisher_count }}" + + - name: Redis Update Draft Course Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_draft_course_count }} {{ draft_course_count }}" + + - name: Redis Update Review Course Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_review_course_count }} {{ review_course_count }}" + + - name: Redis Update Retired Course Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_retired_course_count }} {{ retired_course_count }}" + + - name: Redis Update Pending Publish Course Count + command: "redis-cli -h {{ redis_host }} -p {{ redis_port }} -n {{ redis_db }} SET {{ redis_key_lp_es_pending_publish_course_count }} {{ pending_publish_course_count }}" + + - mail: + host: relay.nic.in + port: 25 + from: mission.karmayogi@gov.in + to: "{{ mail_to }}" + subject: "Dashboard data status Production dated {{ ansible_date_time.date }}" + body: "{{ lookup('template','dashboard.html.j2') }}" + subtype: html diff --git a/ansible/manual-scripts/landing-page-update/automated-dashboard/variable.bm.yml b/ansible/manual-scripts/landing-page-update/automated-dashboard/variable.bm.yml new file mode 100644 index 0000000000..c7c80d2193 --- /dev/null +++ b/ansible/manual-scripts/landing-page-update/automated-dashboard/variable.bm.yml @@ -0,0 +1,65 @@ +usercount_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status\":1}}]}}}" +orgcount_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status\":1}}]}}}" +live_course_api_body: "{\"from\":0,\"size\":10000,\"_source\":[\"identifier\",\"duration\",\"createdFor\"],\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Live\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" + +draft_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Draft\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" +review_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Review\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" +retired_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Retired\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" +pending_publish_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Review\"}},{\"match\":{\"reviewStatus.raw\":\"Reviewed\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" + +domain: "https://portal.karmayogibm.nic.in" +static_domain_name: "portal.karmayogibm.nic.in" + +mail_to: "shobhit.vashistha@tarento.com" + +root_path: "/home/admin/prod_static" +es_host: "http://192.168.3.79:9200" +redis_host: "192.168.3.211" +redis_port: "6379" +redis_db: "12" + +# redis keys updated by LP script +redis_key_lp_update_time: "lp_update_time" +redis_key_lp_es_user_count: "lp_es_user_count" +redis_key_lp_es_org_count: "lp_es_org_count" +redis_key_lp_es_live_course_count: "lp_es_live_course_count" +redis_key_lp_es_agg_course_duration: "lp_es_agg_course_duration" +redis_key_lp_es_live_course_id_list: "lp_es_live_course_id_list" +redis_key_lp_es_live_course_publisher_count: "lp_es_live_course_publisher_count" + +redis_key_lp_es_draft_course_count: "lp_es_draft_course_count" +redis_key_lp_es_review_course_count: "lp_es_review_course_count" +redis_key_lp_es_retired_course_count: "lp_es_retired_course_count" +redis_key_lp_es_pending_publish_course_count: "lp_es_pending_publish_course_count" + +# redis keys not updated by LP script, here for reference +redis_key_dashboard_update_time: "dashboard_update_time" +redis_key_dashboard_courses_enrolled_in_at_least_once: "dashboard_courses_enrolled_in_at_least_once" +redis_key_dashboard_courses_completed_at_least_once: "dashboard_courses_completed_at_least_once" +redis_key_dashboard_courses_enrolled_in_at_least_once_id_list: "dashboard_courses_enrolled_in_at_least_once_id_list" +redis_key_dashboard_courses_completed_at_least_once_id_list: "dashboard_courses_completed_at_least_once_id_list" + +redis_key_dashboard_enrolment_count: "dashboard_enrolment_count" +redis_key_dashboard_completed_count: "dashboard_completed_count" + +redis_key_dashboard_unique_users_enrolled_count: "dashboard_unique_users_enrolled_count" +redis_key_dashboard_unique_users_completed_count: "dashboard_unique_users_completed_count" + +redis_key_dashboard_new_users_registered_yesterday: "dashboard_new_users_registered_yesterday" + +# redis hashmap keys not updated by LP script, here for reference +redis_key_dashboard_user_count_by_user_org: "dashboard_user_count_by_user_org" +redis_key_dashboard_live_course_count_by_course_org: "dashboard_live_course_count_by_course_org" + +redis_key_dashboard_draft_course_count_by_course_org: "dashboard_draft_course_count_by_course_org" +redis_key_dashboard_review_course_count_by_course_org: "dashboard_review_course_count_by_course_org" +redis_key_dashboard_retired_course_count_by_course_org: "dashboard_retired_course_count_by_course_org" +redis_key_dashboard_pending_publish_course_count_by_course_org: "dashboard_pending_publish_course_count_by_course_org" + +redis_key_dashboard_enrolment_count_by_user_org: "dashboard_enrolment_count_by_user_org" +redis_key_dashboard_completed_count_by_user_org: "dashboard_completed_count_by_user_org" + +redis_key_dashboard_enrolment_count_by_course_org: "dashboard_enrolment_count_by_course_org" +redis_key_dashboard_completed_count_by_course_org: "dashboard_completed_count_by_course_org" + +redis_key_dashboard_courses_completed_at_least_once_by_user_org: "dashboard_courses_completed_at_least_once_by_user_org" \ No newline at end of file diff --git a/ansible/manual-scripts/landing-page-update/automated-dashboard/variable.pp.yml b/ansible/manual-scripts/landing-page-update/automated-dashboard/variable.pp.yml new file mode 100644 index 0000000000..fb019f640d --- /dev/null +++ b/ansible/manual-scripts/landing-page-update/automated-dashboard/variable.pp.yml @@ -0,0 +1,69 @@ +usercount_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status\":1}}]}}}" +orgcount_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status\":1}}]}}}" +live_course_api_body: "{\"from\":0,\"size\":10000,\"_source\":[\"identifier\",\"duration\",\"createdFor\"],\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Live\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" + +draft_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Draft\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" +review_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Review\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" +retired_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Retired\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" +pending_publish_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Review\"}},{\"match\":{\"reviewStatus.raw\":\"Reviewed\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" + +domain: "https://portal.karmayogi.nic.in" +static_domain_name: "portal.karmayogi.nic.in" + +mail_to: "shobhit.vashistha@tarento.com" + +root_path: "/home/admin/PreProd_static" +es_host: "http://192.168.3.211:9200" +redis_host: "192.168.3.249" +redis_port: "6379" +redis_db: "12" + +# redis keys updated by LP script +redis_key_lp_update_time: "lp_update_time" +redis_key_lp_es_user_count: "lp_es_user_count" +redis_key_lp_es_org_count: "lp_es_org_count" +redis_key_lp_es_live_course_count: "lp_es_live_course_count" +redis_key_lp_es_agg_course_duration: "lp_es_agg_course_duration" +redis_key_lp_es_live_course_id_list: "lp_es_live_course_id_list" +redis_key_lp_es_live_course_publisher_count: "lp_es_live_course_publisher_count" +redis_key_lp_es_new_users_registered_yesterday: "lp_es_new_users_registered_yesterday" +redis_key_lp_monthly_active_users: "lp_monthly_active_users" +redis_key_lp_completed_count: "lp_completed_count" +redis_key_lp_completed_yesterday_count: "lp_completed_yesterday_count" + +redis_key_lp_es_draft_course_count: "lp_es_draft_course_count" +redis_key_lp_es_review_course_count: "lp_es_review_course_count" +redis_key_lp_es_retired_course_count: "lp_es_retired_course_count" +redis_key_lp_es_pending_publish_course_count: "lp_es_pending_publish_course_count" + +# redis keys not updated by LP script, here for reference +redis_key_dashboard_update_time: "dashboard_update_time" +redis_key_dashboard_courses_enrolled_in_at_least_once: "dashboard_courses_enrolled_in_at_least_once" +redis_key_dashboard_courses_completed_at_least_once: "dashboard_courses_completed_at_least_once" +redis_key_dashboard_courses_enrolled_in_at_least_once_id_list: "dashboard_courses_enrolled_in_at_least_once_id_list" +redis_key_dashboard_courses_completed_at_least_once_id_list: "dashboard_courses_completed_at_least_once_id_list" + +redis_key_dashboard_enrolment_count: "dashboard_enrolment_count" +redis_key_dashboard_completed_count: "dashboard_completed_count" + +redis_key_dashboard_unique_users_enrolled_count: "dashboard_unique_users_enrolled_count" +redis_key_dashboard_unique_users_completed_count: "dashboard_unique_users_completed_count" + +redis_key_dashboard_new_users_registered_yesterday: "dashboard_new_users_registered_yesterday" + +# redis hashmap keys not updated by LP script, here for reference +redis_key_dashboard_user_count_by_user_org: "dashboard_user_count_by_user_org" +redis_key_dashboard_live_course_count_by_course_org: "dashboard_live_course_count_by_course_org" + +redis_key_dashboard_draft_course_count_by_course_org: "dashboard_draft_course_count_by_course_org" +redis_key_dashboard_review_course_count_by_course_org: "dashboard_review_course_count_by_course_org" +redis_key_dashboard_retired_course_count_by_course_org: "dashboard_retired_course_count_by_course_org" +redis_key_dashboard_pending_publish_course_count_by_course_org: "dashboard_pending_publish_course_count_by_course_org" + +redis_key_dashboard_enrolment_count_by_user_org: "dashboard_enrolment_count_by_user_org" +redis_key_dashboard_completed_count_by_user_org: "dashboard_completed_count_by_user_org" + +redis_key_dashboard_enrolment_count_by_course_org: "dashboard_enrolment_count_by_course_org" +redis_key_dashboard_completed_count_by_course_org: "dashboard_completed_count_by_course_org" + +redis_key_dashboard_courses_completed_at_least_once_by_user_org: "dashboard_courses_completed_at_least_once_by_user_org" \ No newline at end of file diff --git a/ansible/manual-scripts/landing-page-update/automated-dashboard/variable.yml b/ansible/manual-scripts/landing-page-update/automated-dashboard/variable.yml new file mode 100644 index 0000000000..e89147bcd8 --- /dev/null +++ b/ansible/manual-scripts/landing-page-update/automated-dashboard/variable.yml @@ -0,0 +1,65 @@ +usercount_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status\":1}}]}}}" +orgcount_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status\":1}}]}}}" +live_course_api_body: "{\"from\":0,\"size\":10000,\"_source\":[\"identifier\",\"duration\",\"createdFor\"],\"query\":{\"bool\":{\"must\":[{\"match\":{\"status\":\"Live\"}},{\"bool\":{\"should\":[{\"match\":{\"courseCategory\":\"Moderated Course\"}},{\"match\":{\"courseCategory\":\"Course\"}}]}}]}}}" + +draft_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Draft\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" +review_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Review\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" +retired_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Retired\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" +pending_publish_course_count_api_body: "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"match\":{\"status.raw\":\"Review\"}},{\"match\":{\"reviewStatus.raw\":\"Reviewed\"}},{\"match\":{\"primaryCategory.raw\":\"Course\"}}]}}}" + +domain: "https://portal.igotkarmayogi.gov.in" +static_domain_name: "igotkarmayogi.gov.in" + +mail_to: "igot_team@fosteringlinux.com, nandakishor.sudarshan@tarento.com, shobhit.vashistha@tarento.com, haridas.kakunje@tarento.com" + +root_path: "/home/admin/prod_static" +es_host: "http://192.168.3.90:9200" +redis_host: "192.168.3.82" +redis_port: "6379" +redis_db: "12" + +# redis keys updated by LP script +redis_key_lp_update_time: "lp_update_time" +redis_key_lp_es_user_count: "lp_es_user_count" +redis_key_lp_es_org_count: "lp_es_org_count" +redis_key_lp_es_live_course_count: "lp_es_live_course_count" +redis_key_lp_es_agg_course_duration: "lp_es_agg_course_duration" +redis_key_lp_es_live_course_id_list: "lp_es_live_course_id_list" +redis_key_lp_es_live_course_publisher_count: "lp_es_live_course_publisher_count" + +redis_key_lp_es_draft_course_count: "lp_es_draft_course_count" +redis_key_lp_es_review_course_count: "lp_es_review_course_count" +redis_key_lp_es_retired_course_count: "lp_es_retired_course_count" +redis_key_lp_es_pending_publish_course_count: "lp_es_pending_publish_course_count" + +# redis keys not updated by LP script, here for reference +redis_key_dashboard_update_time: "dashboard_update_time" +redis_key_dashboard_courses_enrolled_in_at_least_once: "dashboard_courses_enrolled_in_at_least_once" +redis_key_dashboard_courses_completed_at_least_once: "dashboard_courses_completed_at_least_once" +redis_key_dashboard_courses_enrolled_in_at_least_once_id_list: "dashboard_courses_enrolled_in_at_least_once_id_list" +redis_key_dashboard_courses_completed_at_least_once_id_list: "dashboard_courses_completed_at_least_once_id_list" + +redis_key_dashboard_enrolment_count: "dashboard_enrolment_count" +redis_key_dashboard_completed_count: "dashboard_completed_count" + +redis_key_dashboard_unique_users_enrolled_count: "dashboard_unique_users_enrolled_count" +redis_key_dashboard_unique_users_completed_count: "dashboard_unique_users_completed_count" + +redis_key_dashboard_new_users_registered_yesterday: "dashboard_new_users_registered_yesterday" + +# redis hashmap keys not updated by LP script, here for reference +redis_key_dashboard_user_count_by_user_org: "dashboard_user_count_by_user_org" +redis_key_dashboard_live_course_count_by_course_org: "dashboard_live_course_count_by_course_org" + +redis_key_dashboard_draft_course_count_by_course_org: "dashboard_draft_course_count_by_course_org" +redis_key_dashboard_review_course_count_by_course_org: "dashboard_review_course_count_by_course_org" +redis_key_dashboard_retired_course_count_by_course_org: "dashboard_retired_course_count_by_course_org" +redis_key_dashboard_pending_publish_course_count_by_course_org: "dashboard_pending_publish_course_count_by_course_org" + +redis_key_dashboard_enrolment_count_by_user_org: "dashboard_enrolment_count_by_user_org" +redis_key_dashboard_completed_count_by_user_org: "dashboard_completed_count_by_user_org" + +redis_key_dashboard_enrolment_count_by_course_org: "dashboard_enrolment_count_by_course_org" +redis_key_dashboard_completed_count_by_course_org: "dashboard_completed_count_by_course_org" + +redis_key_dashboard_courses_completed_at_least_once_by_user_org: "dashboard_courses_completed_at_least_once_by_user_org" diff --git a/ansible/monit-deploy.yml b/ansible/monit-deploy.yml index 69808b3667..c851142051 100644 --- a/ansible/monit-deploy.yml +++ b/ansible/monit-deploy.yml @@ -31,7 +31,7 @@ - ['{{inventory_dir}}/secrets.yml'] name: Installing and configuring monit roles: - - { role: monit, service: 'monit_assess-events-backup,monit_channel-telemetry-backup,monit_denorm-events-backup,monit_derived-telemetry-backup,monit_device-profile-backup,monit_extractor-failed-backup,monit_failed-telemetry-backup,monit_ingestion-telemetry-backup,monit_learning-events-backup,monit_learning-failed-backup,monit_pipeline-metrics-backup,monit_raw-telemetry-backup,monit_unique-telemetry-backup' } + - { role: monit, service: 'monit_assess-events-backup,monit_denorm-events-backup,monit_derived-telemetry-backup,monit_device-profile-backup,monit_extractor-failed-backup,monit_failed-telemetry-backup,monit_ingestion-cluster-telemetry-backup,monit_learning-events-backup,monit_learning-failed-backup,monit_raw-telemetry-backup,monit_unique-telemetry-backup,monit_assess-raw-events-backup,monit_telemetry-duplicate-backup,monit_extractor-duplicate-backup,monit_content-consumption-events-backup,monit_derived-denorm-events-backup,monit_channel-summary-backup' } tags: secor - hosts: kafka-indexer:analytics-api diff --git a/ansible/portal.yml b/ansible/portal.yml index 6a25409041..02c080c034 100644 --- a/ansible/portal.yml +++ b/ansible/portal.yml @@ -5,4 +5,4 @@ roles: - portal-dashboard environment: - API_KEY: "{{dp_search_service_authorization_token}}" \ No newline at end of file + API_KEY: "{{ sunbird_api_auth_token }}" diff --git a/ansible/postgres-db-update.yml b/ansible/postgres-db-update.yml new file mode 100644 index 0000000000..6c39b05630 --- /dev/null +++ b/ansible/postgres-db-update.yml @@ -0,0 +1,7 @@ +--- +- hosts: postgres + vars_files: + - "{{inventory_dir}}/secrets.yml" + become: yes + roles: + - postgres-db-update diff --git a/ansible/postgres-manage.yml b/ansible/postgres-manage.yml deleted file mode 100644 index e6abda59c2..0000000000 --- a/ansible/postgres-manage.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- hosts: local - vars_files: - - "{{inventory_dir}}/secrets.yml" - become: yes - roles: - - postgres diff --git a/ansible/postgres-provision.yml b/ansible/postgres-provision.yml index 6949f21fca..64bb5f78ba 100644 --- a/ansible/postgres-provision.yml +++ b/ansible/postgres-provision.yml @@ -1,10 +1,19 @@ --- - hosts: postgres - vars_files: - - "{{inventory_dir}}/secrets.yml" become: yes + vars: + # Ref: https://github.com/geerlingguy/ansible-role-postgresql/issues/136 + postgresql_global_config_options: + - option: listen_addresses + value: "{{ ansible_default_ipv4.address }}" + - option: max_connections + value: 99999 + vars_files: + - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] + pre_tasks: + - name: install setfacl support + become: yes + apt: + name: ['acl', 'python3-psycopg2'] roles: - postgres-provision - # become_user: postgres - # roles: - # - postgres diff --git a/ansible/redis-backup.yml b/ansible/redis-backup.yml index 248367c3fb..8e1198185b 100644 --- a/ansible/redis-backup.yml +++ b/ansible/redis-backup.yml @@ -1,10 +1,6 @@ -- hosts: redisall-backup +- hosts: redisall become: yes vars_files: - ['{{inventory_dir}}/secrets.yml'] - environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}" - AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}" roles: - redis-backup - run_once: true diff --git a/ansible/redis-restore.yml b/ansible/redis-restore.yml index 69cc454f99..44c96511b0 100644 --- a/ansible/redis-restore.yml +++ b/ansible/redis-restore.yml @@ -6,4 +6,4 @@ vars_files: - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] roles: - - redis-restore + - redis-multiprocess-restore diff --git a/ansible/redis_provision.yml b/ansible/redis_provision.yml index 0efa7b9365..3bd750765e 100644 --- a/ansible/redis_provision.yml +++ b/ansible/redis_provision.yml @@ -16,6 +16,29 @@ groups: sudo shell: /bin/bash roles: - - redis + - redis-multiprocess serial: 1 + tags: + - metadata_redis # this tag installs multiple redis process in single server +- hosts: redis + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + pre_tasks: + - name: Create group + group: + name: analytics + state: present + - name: Create user + user: + name: analytics + comment: "analytics" + group: analytics + groups: sudo + shell: /bin/bash + roles: + - redis + serial: 1 + tags: + - single_redis_process # this tags installs single redis process. diff --git a/ansible/roles/analytics-api-deploy/defaults/main.yml b/ansible/roles/analytics-api-deploy/defaults/main.yml deleted file mode 100644 index e32185b9b6..0000000000 --- a/ansible/roles/analytics-api-deploy/defaults/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -kafka_broker: "{{groups['processing-cluster-kafka'][0]}}:9092" - -analytics: - home: /mount/data/analytics -analytics_api: - root: "{{ analytics.home }}/api" - home: "{{ analytics.home }}/api/analytics-api-2.0" - service_args: -Dconfig.file=/mount/data/analytics/api/{{ env }}.conf - paths: ['/home/analytics/sbin', '/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/api-service', '/mount/data/analytics/api', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts/monitor-data'] -cassandra: - keyspace_prefix: "{{ env }}_" - -geo_location_city: "{{ env }}_geo_location_city" -geo_location_city_ipv4: "{{ env }}_geo_location_city_ipv4" -report_config: "{{env}}_report_config" -default_channel_id: "in.ekstep" -consumer_channel_cache_type: ConsumerChannel -artifact_name: "analytics-api-2.0-dist.zip" - -analytics: - home: /mount/data/analytics diff --git a/ansible/roles/analytics-api-deploy/tasks/main.yml b/ansible/roles/analytics-api-deploy/tasks/main.yml deleted file mode 100644 index 52805ce030..0000000000 --- a/ansible/roles/analytics-api-deploy/tasks/main.yml +++ /dev/null @@ -1,91 +0,0 @@ -- name: checking the list of installed services - service_facts: - -- name: Stop the Monit service - service: - name: monit - state: stopped - when: ansible_facts.services.monit is defined - -- name: Status of the api service - command: "{{ sbin_path }}/api-service status" - become_user: "{{analytics_user}}" - register: api_status - -- name: Stop the api service - command: "{{ sbin_path }}/api-service stop" - become_user: "{{analytics_user}}" - when: "api_status.stdout.find('RUNNING') != -1" - async: 10 - poll: 5 - -- name: Delete the pid file - file: path={{ sbin_path }}/api-service.pid state=absent - become_user: "{{analytics_user}}" - -- name: Delete the play pid file - file: path={{ analytics_api.home }}/RUNNING_PID state=absent - become_user: "{{analytics_user}}" - -- name: Remove lib directory - file: - path: "{{ analytics_api.home }}/lib" - state: absent - become_user: "{{analytics_user}}" - -- name: Remove the old artifacts - file: - path: "{{ analytics_api.root }}/{{ artifact_name }}" - state: absent - become_user: "{{analytics_user}}" - -- name: Copy API artifact - copy: - src: "{{ artifact_name }}" - dest: "{{ analytics_api.root }}" - become_user: "{{analytics_user}}" - -- name: Unarchive api artifact - unarchive: src={{ analytics_api.root }}/{{ artifact_name }} dest={{ analytics_api.root }} copy=no group={{ analytics_user }} owner={{ analytics_user }} - become_user: "{{analytics_user}}" - -- name: Delete slf4j-log4j jar file - file: path={{ analytics_api.home }}/lib/org.slf4j.slf4j-log4j12-1.7.16.jar state=absent - become_user: "{{analytics_user}}" - -- name: Copy API configuration file - template: src=conf.j2 dest={{ analytics_api.root }}/{{ env }}.conf mode=755 owner={{ analytics_user }} group={{ analytics_group }} - become_user: "{{analytics_user}}" - -- name: Remove netty-all 4.0.29 from lib folder - file: path={{ analytics_api.home }}/lib/io.netty.netty-all-4.0.29.Final.jar state=absent - become_user: "{{analytics_user}}" - -- name: Start the api service - command: "{{ sbin_path }}/api-service start" - become_user: "{{analytics_user}}" - async: 600 - poll: 120 - -- name: Check the API health status - uri: - url: http://localhost:9000/health - become_user: "{{analytics_user}}" - register: api_health_status - -- name: Print the health status to console - debug: - var: api_health_status - -#Update ConsumerChannel Cache -- uri: - url: "http://localhost:9000/refresh-cache/{{ consumer_channel_cache_type }}" - method: GET - return_content: yes - tags: refresh-cache - -- name: Start the Monit service - service: - name: monit - state: started - when: ansible_facts.services.monit is defined diff --git a/ansible/roles/analytics-api-deploy/templates/conf.j2 b/ansible/roles/analytics-api-deploy/templates/conf.j2 deleted file mode 100644 index 22afa19538..0000000000 --- a/ansible/roles/analytics-api-deploy/templates/conf.j2 +++ /dev/null @@ -1,257 +0,0 @@ -application.env="{{ env }}" -spark.cassandra.connection.host={{groups['dp-cassandra'][0]}} -spark.cassandra.connection.port="9042" -service.search.url="{{lp_search}}" -service.search.path="/v2/search" -service.search.requestbody="""{"request":{"filters":{"objectType":["Content"],"contentType":["Story","Worksheet","Collection","Game"],"status":["Live"]},"limit":1000}}""" -service.search.limit="10" - -metrics.creation.es.url="{{lp_composite_search}}" -metrics.creation.es.indexes="compositesearch" - - -recommendation.enable=true -recommendation.limit="10" -recommendation.surprise_find.enable=true -recommendation.surprise_find.index="15" -recommendation.surprise_find.serendipity_factor="20" - -dataproduct.scripts_path="/mount/data/analytics/scripts" - -# Metrics API -# metrics.period.format.day="MMM dd EEE" -# metrics.period.format.month="MMM YYYY" -# metrics.period.format.year="YYYY" - - -# Data Exhaust API -data_exhaust.list.limit="100" -data_exhaust.retry.limit="3" -data_exhaust.dataset.list=["eks-consumption-raw", "eks-consumption-summary", "eks-consumption-metrics","eks-creation-raw", "eks-creation-summary", "eks-creation-metrics"] -data_exhaust.dataset.default="eks-consumption-raw" -data_exhaust.output_format="json" - -# Log4j Kafka appender config -log4j.appender.kafka.enable="false" -log4j.appender.kafka.broker_host="{{ kafka_broker }}" -log4j.appender.kafka.topic="{{env}}.telemetry.log" - -cassandra.service.embedded.enable=false -cassandra.keyspace_prefix="{{ cassandra.keyspace_prefix }}" - -device-register-controller-dispatcher { - type = "Dispatcher" - executor = "fork-join-executor" - fork-join-executor { - # The parallelism factor is used to determine thread pool size using the - # following formula: ceil(available processors * factor). Resulting size - # is then bounded by the parallelism-min and parallelism-max values. - parallelism-factor = 3.0 - - # Min number of threads to cap factor-based parallelism number to - parallelism-min = 4 - - # Max number of threads to cap factor-based parallelism number to - parallelism-max = 8 - } - # Throughput for default Dispatcher, set to 1 for as fair as possible - throughput = 1 -} - -device-register-actor-dispatcher { - type = "Dispatcher" - executor = "thread-pool-executor" - thread-pool-executor { - fixed-pool-size = 8 - } - throughput = 1 -} - -device-profile-actor-dispatcher { - type = "Dispatcher" - executor = "thread-pool-executor" - thread-pool-executor { - fixed-pool-size = 8 - } - throughput = 1 -} - -experiment-actor { - type = "Dispatcher" - executor = "fork-join-executor" - fork-join-executor { - # The parallelism factor is used to determine thread pool size using the - # following formula: ceil(available processors * factor). Resulting size - # is then bounded by the parallelism-min and parallelism-max values. - parallelism-factor = 3.0 - - # Min number of threads to cap factor-based parallelism number to - parallelism-min = 8 - - # Max number of threads to cap factor-based parallelism number to - parallelism-max = 16 - } - # Throughput for default Dispatcher, set to 1 for as fair as possible - throughput = 1 -} - -default-dispatcher { - executor = "fork-join-executor" - fork-join-executor { - # The parallelism factor is used to determine thread pool size using the - # following formula: ceil(available processors * factor). Resulting size - # is then bounded by the parallelism-min and parallelism-max values. - parallelism-factor = 3.0 - - # Min number of threads to cap factor-based parallelism number to - parallelism-min = 8 - - # Max number of threads to cap factor-based parallelism number to - parallelism-max = 16 - } - # Throughput for default Dispatcher, set to 1 for as fair as possible - throughput = 1 -} - -#AKKA Configuration -akka { - actor { - deployment { - - /druid-health-actor { - router = smallest-mailbox-pool - nr-of-instances = 10 - } - /job-service-actor { - router = smallest-mailbox-pool - nr-of-instances = 10 - } - /device-register-actor { - router = smallest-mailbox-pool - dispatcher = device-register-actor-dispatcher - nr-of-instances = 6 - } - /device-profile-actor { - router = smallest-mailbox-pool - dispatcher = device-profile-actor-dispatcher - nr-of-instances = 6 - } - /client-log-actor { - router = smallest-mailbox-pool - nr-of-instances = 4 - } - /experiment-actor { - router = smallest-mailbox-pool - nr-of-instances = 2 - } - - } - } -} - -#Netty Configuration -play.server { - - # The server provider class name - provider = "play.core.server.NettyServerProvider" - - netty { - - # The number of event loop threads. 0 means let Netty decide, which by default will select 2 times the number of - # available processors. - eventLoopThreads = 30 - - # Whether the Netty wire should be logged - log.wire = true - - # The transport to use, either jdk or native. - # Native socket transport has higher performance and produces less garbage but are only available on linux - transport = "native" - } -} - -play.modules.enabled+="modules.ActorInjector" -play.filters.enabled+="filter.RequestInterceptor" -play.filters.enabled+="play.filters.hosts.AllowedHostsFilter" -play.filters.disabled+="play.filters.csrf.CSRFFilter" - -# Secret key -# ~~~~~ -# The secret key is used to secure cryptographics functions. -# If you deploy your application to several instances be sure to use the same key! -play.http.secret.key="{{dp_play_http_secret_key}}" - -play.filters.hosts { - # A list of valid hosts (e.g. "example.com") or suffixes of valid hosts (e.g. ".example.com") - # Note that ".example.com" will match example.com and any subdomain of example.com, with or without a trailing dot. - # "." matches all domains, and "" matches an empty or nonexistent host. - allowed = ["."] -} - -# body parser -play.http.parser.maxMemoryBuffer=10M - -default.consumption.app.id="no_value" -default.channel.id="{{default_channel_id}}" -default.creation.app.id="no_value" - -postgres.db="{{postgres.db_name}}" -postgres.url="jdbc:postgresql://{{ postgres.db_url }}:5432/" -postgres.user="{{ postgres.db_username }}" -postgres.pass="{{ postgres.db_password }}" -postgres.table_name="{{ postgres.db_table_name }}" -postgres.table.geo_location_city.name="{{ geo_location_city }}" -postgres.table.geo_location_city_ipv4.name="{{ geo_location_city_ipv4 }}" -postgres.table.report_config.name="{{ report_config }}" - -default.channel="{{ default_channel }}" - -elasticsearch.service.endpoint="http://{{groups['composite-search-cluster'][0]}}:9200" -elasticsearch.index.compositesearch.name="compositesearch" -elasticsearch.index.dialcodemetrics.name="dialcodemetrics" -metrics.dialcodemetrics.request.limit=1000 - -#channel specifix telemetry exhaust - -dataexhaust.authorization_check=true - -channel.data_exhaust.bucket="{{channel_data_exhaust_bucket}}" -channel.data_exhaust.basePrefix="channel/" -channel.data_exhaust.expiryMins=30 -storage-service.request-signature-version="AWS4-HMAC-SHA256" -s3service.region="ap-south-1" - -cloud_storage_type="azure" - -metrics.time.interval.min=30 -cache.refresh.time.interval.min=1440 - -redis.host="{{metadata_redis_host}}" -#redis.host="localhost" -redis.port=6379 -#redis.port=__redis_port__ -redis.connection.max=20 -redis.connection.idle.max=20 -redis.connection.idle.min=10 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 - -elasticsearch.host="{{lp_composite_search_host}}" -elasticsearch.port=9200 - -# experiment service settings -elasticsearch.searchExperiment.index="experiment" -elasticsearch.searchExperiment.fieldWeight="{\"userId\":3.0, \"deviceId\":3.0, \"url\":3.0 }" -elasticsearch.searchExperiment.matchQueryScore=9.0 -deviceRegisterAPI.experiment.enable=false -experimentService.redisEmptyValueExpirySeconds=86400 -redis.experimentIndex=9 -redis.deviceIndex=2 - -druid.coordinator.host="http://{{groups['raw-coordinator'][0]}}:8081/" -druid.healthcheck.url="druid/coordinator/v1/loadstatus" - -device.api.enable.debug.log=true -kafka.device.register.topic={{ env }}.events.deviceprofile -kafka.metrics.event.topic={{ env }}.pipeline_metrics -kafka.broker.list="{{ ingestion_kafka_brokers }}" diff --git a/ansible/roles/analytics-api-provision/defaults/main.yml b/ansible/roles/analytics-api-provision/defaults/main.yml deleted file mode 100644 index 36875b3a47..0000000000 --- a/ansible/roles/analytics-api-provision/defaults/main.yml +++ /dev/null @@ -1,25 +0,0 @@ -analytics_user_home: /home/{{ analytics_user }} -sbin_path: "{{ analytics_user_home }}/sbin" -storage_provider_env: /etc/environment -script_name: api-service -script_dir: "{{ sbin_path }}" -description: Analytics API service -grep_key: play.core.server.ProdServerStart -process: "/usr/bin/java" -analytics_api_min_heap: 1g -analytics_api_max_heap: 2g -java_options: "-XX:+UseG1GC -XX:+UseStringDeduplication" -process_home: "" -process_args: -- -cp '{{ analytics_api.home }}/lib/*' {{ analytics_api.service_args }} play.core.server.ProdServerStart {{ analytics_api.home }} -analytics_api_version: "2.0" -app_home: "/mount/data/analytics/api/analytics-api-{{ analytics_api_version }}" - - -analytics_api: - root: "{{analytics.home}}/api" - home: "{{ analytics.home }}/api/analytics-api-{{ analytics_api_version }}" - service_args: -Dconfig.file=/mount/data/analytics/api/{{ env }}.conf -Xms{{analytics_api_min_heap}} -Xmx{{analytics_api_max_heap}} {{java_options}} - paths: ['/home/analytics/sbin', '/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/api-service', '/mount/data/analytics/api', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts/monitor-data'] - -analytics: - home: /mount/data/analytics diff --git a/ansible/roles/analytics-api-provision/tasks/main.yml b/ansible/roles/analytics-api-provision/tasks/main.yml deleted file mode 100644 index 87a666ef8a..0000000000 --- a/ansible/roles/analytics-api-provision/tasks/main.yml +++ /dev/null @@ -1,45 +0,0 @@ -## Bootstrap for analytics-api ## - -- name: Adding ENV Vars to bashrc file of spark. - lineinfile: - path: '{{ analytics_user_home }}/.bashrc' - line: 'export {{item.var}}={{item.value}}' - regexp: "export {{ item.var }}.*" - with_items: - - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'} - - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'} - -- name: Adding ENV Vars to spark servers environment. - lineinfile: - path: '/etc/environment' - line: '{{item.var}}={{item.value}}' - regexp: "{{ item.var }}.*" - with_items: - - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'} - - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'} - -- name: Install libraries for spark bootstrap - become: yes - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - build-essential - - git - -- name: Create directories for analytics API - file: - path: "{{ item }}" - owner: "{{ analytics_user }}" - group: "{{ analytics_group }}" - state: directory - with_items: "{{ analytics_api.paths }}" - -- name: Setup analytics API as service - template: - src: servicify-process.j2 - dest: "{{ sbin_path }}/api-service" - mode: 0755 - owner: "{{ analytics_user }}" - group: "{{ analytics_group }}" diff --git a/ansible/roles/analytics-api-provision/templates/servicify-process.j2 b/ansible/roles/analytics-api-provision/templates/servicify-process.j2 deleted file mode 100644 index a203423cef..0000000000 --- a/ansible/roles/analytics-api-provision/templates/servicify-process.j2 +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - -# chkconfig: 2345 95 05 -# description: {{ description }} - -# Source function library. -. /lib/lsb/init-functions - -prog="{{ script_name }}" -SCRIPT_DIR="{{ script_dir }}" #/home/ubuntu/ -PIDFILE={{ script_dir }}/$prog.pid -DESC="{{ description }}" -GREP_KEY="{{ grep_key }}" -DAEMON={{ process }} -DAEMON_HOME="{{ process_home }}" -LOGS_HOME="{{ analytics.home }}/logs" -APP_HOME="{{ app_home }}" - - -#echo "All value sets" - -start() { - if [ -f $PIDFILE ]; then - PID=`cat $PIDFILE` - if [ -z "`pgrep $PID`" ] && [ "$PID" != "`ps aux|grep -vE 'grep|runuser|bash'|grep -w "$GREP_KEY"|awk '{print $2}'`" ]; then - printf "%s\n" "Process dead but pidfile exists" - else - printf "$prog is already running!\n" - fi - else - printf "%-50s" "Starting $prog ..." - export azure_storage_key="{{sunbird_private_storage_account_name}}" - export azure_storage_secret="{{sunbird_private_storage_account_key}}" - cd $APP_HOME - if [ -z "$DAEMON_HOME" ]; then - start-stop-daemon --start --quiet --background --no-close --name $prog --exec $DAEMON {{ process_args }} >$LOGS_HOME/services/$prog.log 2>&1 - else - start-stop-daemon --start --quiet --background -d $DAEMON_HOME --exec $DAEMON {{ process_args }} >$LOGS_HOME/services/$prog.log 2>&1 - fi - sleep 10 - PID=`ps aux|grep -vE 'grep|runuser|bash'|grep -w "$GREP_KEY"|awk '{print $2}'` - if [ -z "$PID" ]; then - printf "[ \e[31mFAIL\033[0m ]\n" - exit 1 - else - echo $PID > $PIDFILE - printf "[ \e[32mOK\033[0m ]\n" - fi - fi -} - -stop() { - printf "%-50s" "Shutting down $prog:" - if [ -f $PIDFILE ]; then - PID=`cat $PIDFILE` - start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE 2>/dev/null - sleep 5 - PID=`ps aux|grep -vE 'grep|runuser|bash'|grep -w "$GREP_KEY"|awk '{print $2}'` - if [ ! -z "$PID" ]; then - printf "[ \e[31mFAIL\033[0m ]\n" - exit 1 - else - rm -f $PIDFILE - printf "[ \e[32mOK\033[0m ]\n" - fi - else - printf "[ \e[32mNOT RUNNING\033[0m ]\n" - fi -} - -check_status() { - printf "%-50s" "Checking $prog ..." - if [ -f $PIDFILE ]; then - PID=`cat $PIDFILE` - if [ -z "`pgrep $PID`" ] && [ "$PID" != "`ps aux|grep -vE 'grep|runuser|bash'|grep -w "$GREP_KEY"|awk '{print $2}'`" ]; then - printf "%s\n" "Process dead but pidfile exists" - else - printf "[ \e[32mRUNNING\033[0m ]\n" - fi - else - printf "[ \e[31mSTOPPED\033[0m ]\n" - fi -} - -case "$1" in - start) - start - ;; - stop) - stop - ;; - status) - check_status - ;; - restart) - stop - start - ;; - *) - echo "Usage: $prog {start|stop|status|restart}" - exit 1 - ;; -esac -exit 0 diff --git a/ansible/roles/analytics-bootstrap-always/meta/main.yml b/ansible/roles/analytics-bootstrap-always/meta/main.yml index ce1479d7c0..08c74c6586 100644 --- a/ansible/roles/analytics-bootstrap-always/meta/main.yml +++ b/ansible/roles/analytics-bootstrap-always/meta/main.yml @@ -1,6 +1,3 @@ --- dependencies: - - { role: jdk-1.8.0_121 , become: yes } - - { role: azure-cli , become: yes } - - + - { role: openjdk , become: true } \ No newline at end of file diff --git a/ansible/roles/analytics-bootstrap-always/tasks/main.yml b/ansible/roles/analytics-bootstrap-always/tasks/main.yml index d7e1594440..5c8da116e9 100644 --- a/ansible/roles/analytics-bootstrap-always/tasks/main.yml +++ b/ansible/roles/analytics-bootstrap-always/tasks/main.yml @@ -26,7 +26,7 @@ with_items: - rsync - zip - - python-pip + - python3-pip - name: Set locale LC_ALL=en_US.UTF-8 become: yes diff --git a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml index 8af0e60754..4036f68aef 100644 --- a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml +++ b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml @@ -1,16 +1,5 @@ ## Bootstrap for spark ## -- name: Adding ENV Vars to bashrc file of spark. - become: yes - become_user: "{{ analytics_user }}" - lineinfile: - path: '{{ analytics_user_home }}/.bashrc' - line: 'export {{item.var}}={{item.value}}' - regexp: "export {{ item.var }}.*" - with_items: - - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'} - - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'} - - name: Adding ENV Vars to spark servers environment. become: yes lineinfile: @@ -18,12 +7,6 @@ line: '{{item.var}}={{item.value}}' regexp: "{{ item.var }}.*" with_items: - - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'} - - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'} - - {var: 'AZURE_STORAGE_ACCOUNT', value: '{{ sunbird_private_storage_account_name }}'} - - {var: 'AZURE_STORAGE_ACCESS_KEY', value: '{{ sunbird_private_storage_account_key }}'} - - {var: 'GOOGLE_CREDENTIALS_PATH', value: '/home/analytics/credentials'} - - {var: 'STORAGE_PROVIDER', value: 'AZURE'} - {var: 'ENV', value: '{{env}}'} - {var: 'KAFKA_BROKER_HOST', value: "{{groups['processing-cluster-kafka'][0]}}:9092"} diff --git a/ansible/roles/analytics-druid/defaults/main.yml b/ansible/roles/analytics-druid/defaults/main.yml index 6a3d80e197..98900320f3 100644 --- a/ansible/roles/analytics-druid/defaults/main.yml +++ b/ansible/roles/analytics-druid/defaults/main.yml @@ -1,71 +1,56 @@ --- # Druid Template variables - -zookeeper_group: "{{ groups['druid-zookeeper'] }}" +druid_monitoring: true +cluster: "{{ remote }}" +zookeeper_group: "groups[{{ cluster }}-zookeeper]" druid_user: druid druid_directory: "/data" -druid_version: "0.18.0" -druid_url: "https://downloads.apache.org/druid/0.18.0/apache-druid-0.18.0-bin.tar.gz" +druid_version: "0.22.1" +druid_url: "https://archive.apache.org/dist/druid/{{druid_version}}/apache-druid-{{druid_version}}-bin.tar.gz" druid_folder: "{{ druid_directory }}/apache-druid-{{ druid_version }}" druid_symlink: "{{ druid_directory }}/druid" druid_path: "{{ druid_symlink }}/" -druid_checksum: "807581d54fa4c5a90eec2a230e2a7fc4c6daf18eb8136009bf36a775d793d6f6" +druid_checksum: "716b83e07a76b5c9e0e26dd49028ca088bde81befb070989b41e71f0e8082d11a26601f4ac1e646bf099a4bc7420bdfeb9f7450d6da53d2a6de301e08c3cab0d" +druid_open_file_limit: 65535 +druid_process_limit: 128596 #Common Varibles -druid_zookeeper_host: "{{ groups['druid-zookeeper']|join(':2181,')}}:2181" +druid_zookeeper_host: "{{ groups[cluster+'-zookeeper']|join(':2181,')}}:2181" +druid_postgres_user: druid #druid_postgres_host: "{{ groups['druid-postgres'][0] }}" -druid_postgres_host: "{{ postgres.db_url }}" -druid_postgres_port: "{{ postgres.db_port }}" -druid_postgres_user: "druid@{{ postgres.db_url }}" -druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}" + druid_default_tmp_dir: "/var/tmp" druid_gc_logdir: "/var/log/druid/crash_logs" druid_crash_logdir: "/var/log/druid/crash_logs" druid_log_dir: "/var/log/druid/" -#Writing request query logs to file +#Writing request query logs to file druid_request_logging_type: "file" -#azure config in group vars - -druid_azure_container_name: "telemetry-data-store" -azure_account_name: "{{sunbird_private_storage_account_name}}" -azure_storage_secret: "{{sunbird_private_storage_account_key}}" -azure_container: "{{ druid_azure_container_name }}" - -#Logging the indexing logs to azure - -druid_log_azure_container: "{{ druid_azure_container_name }}" -druid_log_azure_folder: "druidlogs" - #Druid Extensions -druid_storge_type: "azure" +druid_storage_type: "google" -druid_extensions_list : '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service"' +druid_extensions_list : '"druid-google-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches"' druid_community_extensions: - graphite-emitter # End of druid_extensions -druid_indexing_logs_type: azure +druid_indexing_logs_type: "{{ druid_storage_type }}" druid_indexing_log_dir: /var/druid/indexing-logs druid_indexing_storage_type : metadata druid_indexing_task_basedir : "/var/task" -druid_indexing_queue_startDelay: PT30S druid_common_monitors: '"com.metamx.metrics.JvmMonitor","org.apache.druid.java.util.metrics.JvmMonitor"' druid_common_emitters: '"logging","graphite"' -druid_graphite_host: "{{ groups['druid-graphite'][0] }}" -druid_graphite_port : 2004 druid_graphite_prefix: "druid.metric" druid_whitelist_filepath: "{{ druid_path }}whitelist" -druid_postgress_graphite_pass: "{{ dp_vault_druid_postgress_graphite_pass }}" #End of Common variables @@ -74,67 +59,44 @@ druid_postgress_graphite_pass: "{{ dp_vault_druid_postgress_graphite_pass }}" druid_coordinator_port: 8081 druid_coordinator_service : druid/coordinator -druid_coordinator_heap_size: 128m -druid_coordinator_startDelay: PT30S -druid_coordinator_period: PT30S druid_coordinator_tmp_dir: "{{ druid_default_tmp_dir }}" -druid_coordinator_gc_logfile: "{{ druid_crash_logdir }}/gc.log" +druid_coordinator_gc_logfile: "{{ druid_crash_logdir }}/gc.coordinator.log" +druid_coordinator_heap_dump_file: "{{ druid_crash_logdir }}/coordinator.hprof" #Overlord Configurations druid_overlord_port: 8090 druid_overlord_service : druid/overlord druid_overlord_heap_size: 256m +druid_overlord_min_heap_size: 256m +druid_overlord_max_heap_size: 256m + druid_overlord_tmp_dir: "{{ druid_default_tmp_dir }}" -druid_overlord_gc_logfile: "{{ druid_crash_logdir }}/gc_overlord.log" +druid_overlord_gc_logfile: "{{ druid_crash_logdir }}/gc.overlord.log" +druid_overlord_heap_dump_file: "{{ druid_crash_logdir }}/overlord.hprof" # Configurations Broker druid_broker_port: 8082 druid_broker_service: druid/broker -druid_broker_heap_size: 128m druid_broker_max_direct_size: 800m druid_broker_tmp_dir: /var/tmp -druid_broker_gc_logdir: "{{ druid_crash_logdir }}/gc.log" - -druid_broker_http_numConnections: 5 -druid_broker_server_http_numThread: 25 - -druid_broker_processing_bufferBytes: 134217728 -druid_broker_processing_threads: 2 - +druid_broker_gc_logfile: "{{ druid_crash_logdir }}/gc.broker.log" +druid_broker_heap_dump_file: "{{ druid_crash_logdir }}/broker.hprof" # Historical Configurations druid_historical_port: 8083 druid_historical_service: druid/historical -druid_historical_heap_size: 1048m -druid_historical_max_direct_size: 800m + druid_historical_tmp_dir: "{{ druid_default_tmp_dir }}" -druid_historical_gc_logfile: "{{ druid_crash_logdir }}/historical.gc.log" +druid_historical_gc_logfile: "{{ druid_crash_logdir }}/gc.historical.log" druid_historical_heap_dump_file: "{{ druid_crash_logdir }}/historical.hprof" -druid_broker_heap_dump_file : "{{ druid_crash_logdir }}/broker.hprof" -druid_coordinator_heap_dump_file : "{{ druid_crash_logdir }}/coordinator.hprof" -druid_overlord_heap_dump_file : "{{ druid_crash_logdir }}/overlord.hprof" -druid_mm_heap_dump_file : "{{ druid_crash_logdir }}/middlemanager.hprof" - -druid_historical_http_numConnections: 5 -druid_historical_server_http_numThread: 25 - -druid_historical_processing_bufferBytes: 134217728 -druid_historical_processing_threads: 2 -druid_historical_processing_num_merge_buffers: 2 - druid_historical_monitoring_monitors: '"org.apache.druid.server.metrics.HistoricalMetricsMonitor","com.metamx.metrics.JvmMonitor"' -druid_historical_segmentcache_path: "/var/segmentstore" -druid_historical_segmentcache_size: 10000000000 - -druid_historical_server_maxsize: 10000000000 - -druid_historical_enable_cache: false +druid_historical_segmentcache_path: "/var/druid/segments/segmentstore" druid_historical_cache_size: 3221225472 druid_historical_cache_expiry: 3600000 @@ -143,26 +105,177 @@ druid_historical_cache_expiry: 3600000 druid_middlemanager_port: 8091 druid_middlemanager_service: druid/middlemanager -druid_middlemanager_heap_size: 128m druid_middlemanager_tmp_dir: "{{ druid_default_tmp_dir }}" -druid_middlemanager_gc_logdir: "{{ druid_crash_logdir }}/gc.log" - -druid_middlemanager_worker_cap: 4 -druid_mm_java_opts_array: "-server -Xmx900m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" - -druid_middlemanager_peon_server_http_numThread: 25 -druid_middlemanager_peon_processing_bufferBytes: 25000000 -druid_middlemanager_peon_processing_threads: 2 - - -druid_middlemanager_peon_segmentcache_path: "/var/segment_cache" -druid_middlemanager_peon_server_maxsize: 0 +druid_middlemanager_gc_logfile: "{{ druid_crash_logdir }}/gc.middlemanager.log" +druid_mm_heap_dump_file : "{{ druid_crash_logdir }}/middlemanager.hprof" -# Router configurations +druid_middlemanager_peon_segmentcache_path: "/var/druid/segments/segment_cache" +druid_azure_container_name: "telemetry-data-store" -druid_router_heap_size: 1g -druid_router_http_numConnections: 50 -druid_router_http_readTimeout: PT5M -druid_router_http_numMaxThreads: 100 -druid_server_http_numThreads: 100 -druid_router_managementProxy_enabled: true +# Router Configuration +druid_router_port: 8888 +druid_router_service: druid/router + +druid_router_tmp_dir: "{{ druid_default_tmp_dir }}" +druid_router_gc_logfile: "{{ druid_crash_logdir }}/gc.router.log" +druid_router_heap_dump_file: "{{ druid_crash_logdir }}/router.hprof" + + +default_druid_configs: + raw: + #Druid Postgres Details + druid_postgres_db: "druid" + druid_postgres_host: "{{ postgres.db_url }}" + druid_postgres_port: "{{ postgres.db_port }}" + druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}" + druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}" + #Druid Azure Details + azure_account_name: "{{ sunbird_druid_storage_account_name | default('') }}" + azure_storage_secret: "{{ sunbird_druid_storage_account_key | default('') }}" + azure_container: "{{ druid_azure_container_name | default('') }}" + #Logging the indexing logs to azure + druid_log_azure_container: "{{ druid_azure_container_name | default('') }}" + druid_log_azure_folder: "druidlogs" + #Druid S3 Details + druid_storage_type: "{{ druid_storage_type }}" + druid_gcloud_bucket: "{{ druid_gcloud_bucket }}" + druid_gcloud_prefix: "druid/raw/segments" + druid_gcloud_indexer_logs_bucket: "{{ druid_gcloud_bucket }}" + druid_gcloud_indexer_logs_prefix: "druid/raw/logs/indexer" + druid_gcloud_max_list_length: 1024 + #Druid coordinator node configuration + druid_coordinator_min_heap_size: 2048m + druid_coordinator_max_heap_size: 5120m + druid_coordinator_period: PT30S + druid_coordinator_startDelay: PT30S + druid_coordinator_balance_strategy: diskNormalized + #Druid overlord node configuration + druid_overlord_min_heap_size: 1024m + druid_overlord_max_heap_size: 2560m + #Druid broker node configuration + druid_broker_min_heap_size: 2048m + druid_broker_max_heap_size: 5120m + druid_broker_max_direct_size: 2560m + druid_broker_http_numConnections: 5 + druid_broker_server_http_numThread: 25 + druid_broker_processing_bufferBytes: 536870912 + druid_broker_processing_threads: 2 + druid_broker_processing_num_merge_buffers: 2 + #Druid historical node configuration + druid_historical_min_heap_size: 2560m + druid_historical_max_heap_size: 5120m + druid_historical_max_direct_size: 5120m + druid_historical_http_numConnections: 5 + druid_historical_server_http_numThread: 25 + druid_historical_processing_bufferBytes: 536870912 + druid_historical_processing_num_merge_buffers: 2 + druid_historical_processing_threads: 7 + druid_historical_enable_cache: true + druid_historical_cache_size: 3221225472 + druid_historical_cache_expiry: 3600000 + druid_historical_segmentcache_size: 40000000000 + druid_historical_segmentcache_locations__count: 10 # deliberate double underscore to indicate that this does not map to a druid config + druid_historical_segmentcache_numloadingthreads: 4 + druid_historical_segmentcache_path: "/var/druid/segments/segmentstore" + druid_historical_server_maxsize: 650000000000 # > 10 * 40000000000 + druid_query_ondiskstorage_enabled: true + druid_historical_maxOnDiskStorage: 21474836480 + druid_historical_maxMergingDictionarySize: 100000000 + #Druid middlemanager configuration + druid_middlemanager_min_heap_size: 128m + druid_middlemanager_max_heap_size: 128m + druid_middlemanager_worker_cap: 18 + druid_mm_java_opts_array: "-server -Xms512m -Xmx1024m -XX:MaxDirectMemorySize=1024m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" + druid_middlemanager_peon_server_http_numThread: 25 + druid_middlemanager_peon_processing_bufferBytes: 104857600 + druid_middlemanager_peon_processing_num_merge_buffers: 2 + druid_middlemanager_peon_processing_threads: 2 + druid_middlemanager_peon_server_maxsize: 0 + druid_indexing_queue_startDelay: PT30S + #Druid router configuration + druid_router_min_heap_size: 1024m + druid_router_max_heap_size: 1024m + druid_router_http_numConnections: 50 + druid_router_http_readTimeout: PT5M + druid_router_http_numMaxThreads: 100 + druid_server_http_numThreads: 100 + druid_router_managementProxy_enabled: true + rollup: + #Druid Postgres Details + druid_postgres_db: "druid" + druid_postgres_host: "{{ postgres.db_url }}" + druid_postgres_port: "{{ postgres.db_port }}" + druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}" + druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}" + + azure_account_name: "{{ sunbird_druid_storage_account_name | default('') }}" + azure_storage_secret: "{{ sunbird_druid_storage_account_key | default('') }}" + azure_container: "{{ druid_azure_container_name | default('') }}" + #Logging the indexing logs to azure + druid_log_azure_container: "{{ druid_azure_container_name | default('') }}" + druid_log_azure_folder: "druidlogs" + #Druid S3 Details + druid_storage_type: "{{ druid_storage_type }}" + druid_gcloud_bucket: "{{ druid_gcloud_bucket }}" + druid_gcloud_prefix: "druid/raw/segments" + druid_gcloud_indexer_logs_bucket: "{{ druid_gcloud_bucket }}" + druid_gcloud_indexer_logs_prefix: "druid/raw/logs/indexer" + druid_gcloud_max_list_length: 1024 + + #Druid coordinator node configuration + druid_coordinator_min_heap_size: 128m + druid_coordinator_max_heap_size: 128m + druid_coordinator_period: PT30S + druid_coordinator_startDelay: PT30S + druid_coordinator_balance_strategy: cost + #Druid overlord node configuration + druid_overlord_min_heap_size: 256m + druid_overlord_max_heap_size: 256m + #Druid broker node configuration + druid_broker_min_heap_size: 128m + druid_broker_max_heap_size: 128m + druid_broker_max_direct_size: 700m + druid_broker_http_numConnections: 5 + druid_broker_server_http_numThread: 25 + druid_broker_processing_bufferBytes: 134217728 + druid_broker_processing_threads: 2 + druid_broker_processing_num_merge_buffers: 2 + #Druid historical node configuration + druid_historical_min_heap_size: 1048m + druid_historical_max_heap_size: 1048m + druid_historical_max_direct_size: 800m + druid_historical_http_numConnections: 5 + druid_historical_server_http_numThread: 25 + druid_historical_processing_bufferBytes: 134217728 + druid_historical_processing_threads: 2 + druid_historical_enable_cache: false + druid_historical_segmentcache_size: 2000000000 + druid_historical_segmentcache_locations__count: 5 # deliberate double underscore to indicate that this does not map to a druid config + druid_historical_server_maxsize: 10000000000 + druid_historical_processing_num_merge_buffers: 2 + druid_query_ondiskstorage_enabled: false + druid_historical_segmentcache_numloadingthreads: 4 + druid_historical_segmentcache_path: "/var/druid/segments/segmentstore" + druid_historical_cache_size: 3221225472 + druid_historical_cache_expiry: 3600000 + #Druid middlemanager configuration + druid_middlemanager_min_heap_size: 128m + druid_middlemanager_max_heap_size: 128m + druid_middlemanager_worker_cap: 12 + druid_mm_java_opts_array: "-server -Xmx900m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" + druid_middlemanager_peon_server_http_numThread: 25 + druid_middlemanager_peon_processing_bufferBytes: 25000000 + druid_middlemanager_peon_processing_threads: 2 + druid_middlemanager_peon_processing_num_merge_buffers: 2 + druid_middlemanager_peon_server_maxsize: 0 + druid_indexing_queue_startDelay: PT30S + druid_router_min_heap_size: 1g + druid_router_max_heap_size: 1g + druid_router_http_numConnections: 50 + druid_router_http_readTimeout: PT5M + druid_router_http_numMaxThreads: 100 + druid_server_http_numThreads: 100 + druid_router_managementProxy_enabled: true + +enable_druid_sql: true +gcs_service_account_file: "/root/.gcs_service_acount.json" diff --git a/ansible/roles/analytics-druid/files/whitelist b/ansible/roles/analytics-druid/files/whitelist index e96d9c8679..9ce2d7bf71 100644 --- a/ansible/roles/analytics-druid/files/whitelist +++ b/ansible/roles/analytics-druid/files/whitelist @@ -1,111 +1,11 @@ { - "jetty/numOpenConnections" :[], - "query/time" : ["dataSource", "type"], + "tier/historical/count":["tier"], + "segment/unavailable/count": ["dataSource"], + "task/success/count":["dataSource"], + "task/failed/count":["dataSource"], + "ingest/kafka/lag" : ["dataSource"], + "ingest/handoff/failed" : ["dataSource"], "query/success/count" : [], "query/failed/count" : [], - "query/interrupted/count" : [], - "query/node/time" : ["server"], - "query/node/ttfb" : ["server"], - "query/intervalChunk/time" : [], - - "query/segment/time" :[], - "query/wait/time" : [], - "segment/scan/pending" : [], - "query/segmentAndCache/time" : [], - "query/cpu/time" : ["dataSource", "type"], - - "query/cache/delta/numEntries" : [], - "query/cache/delta/sizeBytes" : [], - "query/cache/delta/hits" : [], - "query/cache/delta/misses" : [], - "query/cache/delta/evictions" : [], - "query/cache/delta/hitRate" : [], - "query/cache/delta/averageBytes" : [], - "query/cache/delta/timeouts" : [], - "query/cache/delta/errors" : [], - - "query/cache/total/numEntries" : [], - "query/cache/total/sizeBytes" : [], - "query/cache/total/hits" : [], - "query/cache/total/misses" : [], - "query/cache/total/evictions" : [], - "query/cache/total/hitRate" : [], - "query/cache/total/averageBytes" : [], - "query/cache/total/timeouts" : [], - "query/cache/total/errors" : [], - - "ingest/events/thrownAway" : ["dataSource", "taskType"], - "ingest/events/unparseable" : ["dataSource", "taskType"], - "ingest/events/processed" : ["dataSource", "taskType"], - "ingest/rows/output" : ["dataSource"], - "ingest/persist/count" : ["dataSource"], - "ingest/persist/time" : ["dataSource"], - "ingest/persist/cpu" : ["dataSource"], - "ingest/persist/backPressure" : ["dataSource"], - "ingest/persist/failed" : ["dataSource"], - "ingest/handoff/failed" : ["dataSource"], - "ingest/merge/time" : ["dataSource"], - "ingest/merge/cpu" : ["dataSource"], - - "task/run/time" : ["dataSource", "taskType"], - "segment/added/bytes" : ["dataSource", "taskType"], - "segment/moved/bytes" : ["dataSource", "taskType"], - "segment/nuked/bytes" : ["dataSource", "taskType"], - - "segment/assigned/count" : ["tier"], - "segment/moved/count" : ["tier"], - "segment/dropped/count" : ["tier"], - "segment/deleted/count" : ["tier"], - "segment/unneeded/count" : ["tier"], - "segment/cost/raw" : ["tier"], - "segment/cost/normalization" : ["tier"], - "segment/cost/normalized" : ["tier"], - "segment/loadQueue/size" : ["server"], - "segment/loadQueue/failed" : ["server"], - "segment/loadQueue/count" : ["server"], - "segment/dropQueue/count" : ["server"], - "segment/size" : ["dataSource"], - "segment/count" : ["dataSource"], - "segment/overShadowed/count" : [], - - "segment/max" : [], - "segment/used" : ["dataSource"], - "segment/usedPercent" : ["dataSource"], - - "jvm/pool/committed" : ["poolKind", "poolName"], - "jvm/pool/init" :["poolKind", "poolName"], - "jvm/pool/max" : ["poolKind", "poolName"], - "jvm/pool/used" : ["poolKind", "poolName"], - "jvm/bufferpool/count" : ["bufferPoolName"], - "jvm/bufferpool/used" : ["bufferPoolName"], - "jvm/bufferpool/capacity" : ["bufferPoolName"], - "jvm/mem/init" : ["memKind"], - "jvm/mem/max" : ["memKind"], - "jvm/mem/used" : ["memKind"], - "jvm/mem/committed" : ["memKind"], - "jvm/gc/count" : ["gcName"], - "jvm/gc/time" : ["gcName"], - - "ingest/events/buffered" : ["dataSource"], - "ingest/kafka/lag" : ["dataSource"], - - "sys/swap/free" : [], - "sys/swap/max" : [], - "sys/swap/pageIn" :[], - "sys/swap/pageOut" : [], - "sys/disk/write/count" : ["fsDevName"], - "sys/disk/read/count" : ["fsDevName"], - "sys/disk/write/size" : ["fsDevName"], - "sys/disk/read/size" : ["fsDevName"], - "sys/net/write/size" : [], - "sys/net/read/size" : [], - "sys/fs/used" : ["fsDevName", "fsDirName", "fsTypeName", "fsSysTypeName", "fsOptions"], - "sys/fs/max" : ["fsDevName", "fsDirName", "fsTypeName", "fsSysTypeName", "fsOptions"], - "sys/mem/used" : [], - "sys/mem/max" : [], - "sys/storage/used" : ["fsDirName"], - "sys/cpu" : ["cpuName", "cpuTime"], - - "coordinator-segment/count" : ["dataSource"], - "historical-segment/count" : ["dataSource", "tier", "priority"] -} \ No newline at end of file + "segment/usedPercent" : ["dataSource"] +} diff --git a/ansible/roles/analytics-druid/meta/main.yaml b/ansible/roles/analytics-druid/meta/main.yaml new file mode 100644 index 0000000000..2d92bdf2e3 --- /dev/null +++ b/ansible/roles/analytics-druid/meta/main.yaml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: graphite-exporter diff --git a/ansible/roles/analytics-druid/tasks/install.yml b/ansible/roles/analytics-druid/tasks/install.yml index 3bba463166..e3a2efaa24 100644 --- a/ansible/roles/analytics-druid/tasks/install.yml +++ b/ansible/roles/analytics-druid/tasks/install.yml @@ -1,5 +1,7 @@ --- # file: install.yml +- name: Set the druid configs + set_fact: druid_configs="{{ default_druid_configs | combine(private_druid_configs,recursive=True) if private_druid_configs is defined else default_druid_configs }}" - name: Create druid user user: @@ -104,7 +106,7 @@ owner: "{{ druid_user }}" mode: 755 -- name: Copy over systemd service file for druid role {{ druid_role }} +- name: Copy over systemd service file for druid role {{ remote }}-{{ druid_role }} template: src=druid.service dest=/lib/systemd/system/druid_{{ druid_role }}.service - name: Start druid {{ druid_role }} service diff --git a/ansible/roles/analytics-druid/tasks/main.yml b/ansible/roles/analytics-druid/tasks/main.yml index deba8959e7..8f50b233a9 100644 --- a/ansible/roles/analytics-druid/tasks/main.yml +++ b/ansible/roles/analytics-druid/tasks/main.yml @@ -1,4 +1,10 @@ # include install and test for druid role - include_vars: "{{inventory_dir}}/secrets.yml" +- name: create the service account file + copy: + dest: "{{ gcs_service_account_file }}" + content: "{{ druid_gcs_service_account }}" + with_items: + - {var: 'GOOGLE_APPLICATION_CREDENTIALS', value: '{{ gcs_service_account_file }}'} - include: install.yml - include: test.yml diff --git a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties index 464c2d4e48..b2faad45f3 100644 --- a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties +++ b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties @@ -38,23 +38,37 @@ druid.zk.paths.base=/druid # Metadata storage # For PostgreSQL: druid.metadata.storage.type=postgresql -druid.metadata.storage.connector.connectURI=jdbc:postgresql://{{ druid_postgres_host }}:{{ druid_postgres_port }}/{{ druid_postgres_db }} -druid.metadata.storage.connector.user={{ druid_postgres_user }} -druid.metadata.storage.connector.password={{ druid_postgres_pass }} +druid.metadata.storage.connector.connectURI=jdbc:postgresql://{{ druid_configs[cluster].druid_postgres_host }}:{{ druid_configs[cluster].druid_postgres_port }}/{{ druid_configs[cluster].druid_postgres_db }} +druid.metadata.storage.connector.user={{ druid_configs[cluster].druid_postgres_user }} +druid.metadata.storage.connector.password={{ druid_configs[cluster].druid_postgres_pass }} # Deep storage #Using azure -druid.storage.type={{ druid_storge_type }} -druid.azure.account={{sunbird_druid_storage_account_name}} -druid.azure.key={{sunbird_druid_storage_account_key}} -druid.azure.container={{ azure_container }} +druid.storage.type={{ druid_storage_type }} +{% if druid_storage_type == "azure" %} +druid.azure.account={{ druid_configs[cluster].azure_account_name }} +druid.azure.key={{ druid_configs[cluster].azure_storage_secret }} +druid.azure.container={{ druid_configs[cluster].azure_container }} +{% elif druid_storage_type == "google" %} +druid.google.bucket={{ druid_configs[cluster].druid_gcloud_bucket }} +druid.google.prefix={{ druid_configs[cluster].druid_gcloud_prefix }} +druid.indexer.logs.bucket={{ druid_configs[cluster].druid_gcloud_indexer_logs_bucket }} +druid.indexer.logs.prefix={{ druid_configs[cluster].druid_gcloud_indexer_logs_prefix }} +druid.google.maxListingLength={{ druid_configs[cluster].druid_gcloud_max_list_length }} +{% endif %} # Indexing service logs # For local disk (only viable in a cluster if this is a network mount): druid.indexer.logs.type={{ druid_indexing_logs_type }} +{% if druid_indexing_logs_type == "azure" %} #druid.indexer.logs.directory={{ druid_indexing_log_dir }} -druid.indexer.logs.container={{ druid_log_azure_container }} -druid.indexer.logs.prefix= {{ druid_log_azure_folder }} +druid.indexer.logs.container={{ druid_configs[cluster].druid_log_azure_container }} +druid.indexer.logs.prefix= {{ druid_configs[cluster].druid_log_azure_folder }} +{% elif druid_indexing_logs_type == "s3" %} +druid.indexer.logs.s3Bucket={{ druid_configs[cluster].s3_logging_bucket }} +# path to logs within the bucker +druid.indexer.logs.s3Prefix={{ druid_configs[cluster].s3_indexer_logs_dir }} +{% endif %} # Service discovery druid.selectors.indexing.serviceName={{ druid_overlord_service }} @@ -66,19 +80,22 @@ druid.emitter=composing druid.emitter.composing.emitters=[ {{ druid_common_emitters }}] druid.emitter.logging.logLevel=info -{% if druid_graphite_host is defined %} -druid.emitter.graphite.port={{ druid_graphite_port }} -druid.emitter.graphite.hostname={{ druid_graphite_host }} +{% if druid_monitoring %} +druid.emitter.graphite.port={{ druid_graphite_port | d(9109) }} +druid.emitter.graphite.hostname={{ druid_graphite_host | d("localhost") }} +druid.emitter.graphite.protocol=plaintext druid.emitter.graphite.eventConverter={"type":"whiteList", "namespacePrefix": "{{ druid_graphite_prefix }}", "ignoreHostname":false, "ignoreServiceName":false ,"mapPath":"{{ druid_whitelist_filepath }}"} {% endif %} + # Storage type of double columns # ommiting this will lead to index double as float at the storage layer druid.indexing.doubleStorage=double -#Writing query logs into file +#Writing query logs into file druid.request.logging.type={{ druid_request_logging_type }} druid.request.logging.dir={{ druid_log_dir }} druid.javascript.enabled=true +druid.sql.enable={{ enable_druid_sql | lower }} diff --git a/ansible/roles/analytics-druid/templates/broker/jvm.config b/ansible/roles/analytics-druid/templates/broker/jvm.config index 530561493a..b04197578c 100644 --- a/ansible/roles/analytics-druid/templates/broker/jvm.config +++ b/ansible/roles/analytics-druid/templates/broker/jvm.config @@ -1,11 +1,11 @@ -server --Xms{{ druid_broker_heap_size }} --Xmx{{ druid_broker_heap_size }} --XX:MaxDirectMemorySize={{ druid_broker_max_direct_size }} +-Xms{{ druid_configs[cluster].druid_broker_min_heap_size }} +-Xmx{{ druid_configs[cluster].druid_broker_max_heap_size }} +-XX:MaxDirectMemorySize={{ druid_configs[cluster].druid_broker_max_direct_size }} -XX:+PrintGCDetails -XX:+UseG1GC -XX:+PrintGCTimeStamps --Xloggc:{{ druid_broker_gc_logdir }} +-Xloggc:{{ druid_broker_gc_logfile }} -XX:HeapDumpPath={{ druid_broker_heap_dump_file }} -Duser.timezone=UTC -Dfile.encoding=UTF-8 diff --git a/ansible/roles/analytics-druid/templates/broker/runtime.properties b/ansible/roles/analytics-druid/templates/broker/runtime.properties index bb4ce59d99..ec2c5d25d8 100644 --- a/ansible/roles/analytics-druid/templates/broker/runtime.properties +++ b/ansible/roles/analytics-druid/templates/broker/runtime.properties @@ -3,12 +3,13 @@ druid.host={{ ansible_host }} druid.port={{ druid_broker_port }} # HTTP server threads -druid.broker.http.numConnections={{ druid_broker_http_numConnections }} -druid.server.http.numThreads={{ druid_broker_server_http_numThread }} +druid.broker.http.numConnections={{ druid_configs[cluster].druid_broker_http_numConnections }} +druid.server.http.numThreads={{ druid_configs[cluster].druid_broker_server_http_numThread }} # Processing threads and buffers -druid.processing.buffer.sizeBytes={{ druid_broker_processing_bufferBytes }} -druid.processing.numThreads={{ druid_broker_processing_threads }} +druid.processing.buffer.sizeBytes={{ druid_configs[cluster].druid_broker_processing_bufferBytes }} +druid.processing.numThreads={{ druid_configs[cluster].druid_broker_processing_threads }} +druid.processing.numMergeBuffers={{ druid_configs[cluster].druid_broker_processing_num_merge_buffers }} druid.javascript.enabled=true druid.sql.enable=true diff --git a/ansible/roles/analytics-druid/templates/coordinator/jvm.config b/ansible/roles/analytics-druid/templates/coordinator/jvm.config index dc6d765efb..21f8c5068c 100644 --- a/ansible/roles/analytics-druid/templates/coordinator/jvm.config +++ b/ansible/roles/analytics-druid/templates/coordinator/jvm.config @@ -1,6 +1,6 @@ -server --Xms{{ druid_coordinator_heap_size }} --Xmx{{ druid_coordinator_heap_size }} +-Xms{{ druid_configs[cluster].druid_coordinator_min_heap_size }} +-Xmx{{ druid_configs[cluster].druid_coordinator_max_heap_size }} -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCDateStamps diff --git a/ansible/roles/analytics-druid/templates/coordinator/runtime.properties b/ansible/roles/analytics-druid/templates/coordinator/runtime.properties index 8c176e83d9..a4a99948ed 100644 --- a/ansible/roles/analytics-druid/templates/coordinator/runtime.properties +++ b/ansible/roles/analytics-druid/templates/coordinator/runtime.properties @@ -2,10 +2,11 @@ druid.service={{ druid_coordinator_service }} druid.host={{ ansible_host }} druid.port={{ druid_coordinator_port }} -druid.coordinator.startDelay={{ druid_coordinator_startDelay }} -druid.coordinator.period={{ druid_coordinator_period }} +druid.coordinator.startDelay={{ druid_configs[cluster].druid_coordinator_startDelay }} +druid.coordinator.period={{ druid_configs[cluster].druid_coordinator_period }} +druid.coordinator.balancer.strategy={{ druid_configs[cluster].druid_coordinator_balance_strategy }} {% if druid_coordinator_monitoring_monitors is defined %} -# Logging monitors +#Logging monitors druid.monitoring.monitors=[{{ druid_coordinator_monitoring_monitors }}] {% endif %} diff --git a/ansible/roles/analytics-druid/templates/druid.service b/ansible/roles/analytics-druid/templates/druid.service index 8e7503b812..ecb132f1b0 100644 --- a/ansible/roles/analytics-druid/templates/druid.service +++ b/ansible/roles/analytics-druid/templates/druid.service @@ -3,8 +3,11 @@ Description=druid_{{druid_role}}_service After=network.target [Service] +Environment="GOOGLE_APPLICATION_CREDENTIALS={{ gcs_service_account_file }}" WorkingDirectory={{ druid_path }} ExecStart=/bin/sh -c "/usr/bin/java $(/bin/cat {{ druid_path }}conf/druid/{{ service }}/jvm.config | /usr/bin/xargs) -cp {{ druid_path }}conf/druid/{{ service }}:{{ druid_path }}conf/druid/_common:{{ druid_path }}lib/* org.apache.druid.cli.Main server {{ service }}" +LimitNOFILE={{ druid_open_file_limit }} +LimitNPROC={{ druid_process_limit }} Restart=always [Install] diff --git a/ansible/roles/analytics-druid/templates/historical/jvm.config b/ansible/roles/analytics-druid/templates/historical/jvm.config index f076e6af60..18fd757a37 100644 --- a/ansible/roles/analytics-druid/templates/historical/jvm.config +++ b/ansible/roles/analytics-druid/templates/historical/jvm.config @@ -1,9 +1,10 @@ -server --Xms{{ druid_historical_heap_size }} --Xmx{{ druid_historical_heap_size }} --XX:MaxDirectMemorySize={{ druid_historical_max_direct_size }} +-Xms{{ druid_configs[cluster].druid_historical_min_heap_size }} +-Xmx{{ druid_configs[cluster].druid_historical_max_heap_size }} +-XX:MaxDirectMemorySize={{ druid_configs[cluster].druid_historical_max_direct_size }} -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps diff --git a/ansible/roles/analytics-druid/templates/historical/runtime.properties b/ansible/roles/analytics-druid/templates/historical/runtime.properties index ee8e889d7c..a71e80d8de 100644 --- a/ansible/roles/analytics-druid/templates/historical/runtime.properties +++ b/ansible/roles/analytics-druid/templates/historical/runtime.properties @@ -4,27 +4,41 @@ druid.port={{ druid_historical_port }} # HTTP server threads -druid.server.http.numThreads={{ druid_historical_server_http_numThread }} +druid.server.http.numThreads={{ druid_configs[cluster].druid_historical_server_http_numThread }} # Processing threads and buffers -druid.processing.buffer.sizeBytes={{ druid_historical_processing_bufferBytes }} -druid.processing.numThreads={{ druid_historical_processing_threads }} -druid.processing.numMergeBuffers={{ druid_historical_processing_num_merge_buffers }} +druid.processing.buffer.sizeBytes={{ druid_configs[cluster].druid_historical_processing_bufferBytes }} +druid.historical.http.numConnections={{ druid_configs[cluster].druid_historical_http_numConnections }} +druid.processing.numThreads={{ druid_configs[cluster].druid_historical_processing_threads }} +druid.processing.numMergeBuffers={{ druid_configs[cluster].druid_historical_processing_num_merge_buffers }} -# Segment storage -druid.segmentCache.locations=[{"path": "{{ druid_historical_segmentcache_path }}", "maxSize": {{ druid_historical_segmentcache_size }}}] -druid.server.maxSize={{ druid_historical_server_maxsize }} +# Segmentstorage +#druid.segmentCache.locations=[{"path": "{{ druid_historical_segmentcache_path }}", "maxSize": {{ druid_configs[cluster].druid_historical_segmentcache_size }}}] +# Set the segment size to 9% of historical max size +{% set max_size = (druid_configs[cluster].druid_historical_server_maxsize * 0.09) | int %} + +druid.segmentCache.locations=[{% for number in range(druid_configs[cluster].druid_historical_segmentcache_locations__count) %}{"path": "{{ druid_configs[cluster].druid_historical_segmentcache_path }}{{number }}", "maxSize": {{ max_size }}}{% if not loop.last %}, {% endif %}{% endfor %}] +druid.server.maxSize={{ druid_configs[cluster].druid_historical_server_maxsize }} + +druid.segmentCache.numLoadingThreads={{ druid_configs[cluster].druid_historical_segmentcache_numloadingthreads }} {% if druid_historical_monitoring_monitors is defined %} # Monitors druid.monitoring.monitors=[{{ druid_historical_monitoring_monitors }}] {% endif %} +{% if druid_configs[cluster].druid_query_ondiskstorage_enabled %} +druid.query.groupBy.maxMergingDictionarySize={{ druid_configs[cluster].druid_historical_maxMergingDictionarySize }} +druid.query.groupBy.maxOnDiskStorage={{ druid_configs[cluster].druid_historical_maxOnDiskStorage }} +{% endif %} + + # Caching -druid.historical.cache.useCache={{ druid_historical_enable_cache }} +druid.historical.cache.useCache={{ druid_configs[cluster].druid_historical_enable_cache }} druid.historical.cache.populateCache=true druid.historical.cache.unCacheable=["select","scan"] druid.cache.type=caffeine -druid.cache.sizeInBytes={{ druid_historical_cache_size }} -druid.cache.expireAfter={{ druid_historical_cache_expiry }} \ No newline at end of file +druid.cache.sizeInBytes={{ druid_configs[cluster].druid_historical_cache_size }} +druid.cache.expireAfter={{ druid_configs[cluster].druid_historical_cache_expiry }} + diff --git a/ansible/roles/analytics-druid/templates/middlemanager/jvm.config b/ansible/roles/analytics-druid/templates/middlemanager/jvm.config index 7a9c216f2e..68a5a5d73b 100644 --- a/ansible/roles/analytics-druid/templates/middlemanager/jvm.config +++ b/ansible/roles/analytics-druid/templates/middlemanager/jvm.config @@ -1,10 +1,10 @@ -server --Xmx{{ druid_middlemanager_heap_size }} --Xms{{ druid_middlemanager_heap_size }} +-Xms{{ druid_configs[cluster].druid_middlemanager_min_heap_size }} +-Xmx{{ druid_configs[cluster].druid_middlemanager_max_heap_size }} -XX:+PrintGCDetails -XX:+UseG1GC -XX:+PrintGCTimeStamps --Xloggc:{{ druid_middlemanager_gc_logdir }} +-Xloggc:{{ druid_middlemanager_gc_logfile }} -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/ansible/roles/analytics-druid/templates/middlemanager/log4j2.xml b/ansible/roles/analytics-druid/templates/middlemanager/log4j2.xml index fa9f9a688b..15ede15fd9 100644 --- a/ansible/roles/analytics-druid/templates/middlemanager/log4j2.xml +++ b/ansible/roles/analytics-druid/templates/middlemanager/log4j2.xml @@ -18,9 +18,6 @@ - - - diff --git a/ansible/roles/analytics-druid/templates/middlemanager/runtime.properties b/ansible/roles/analytics-druid/templates/middlemanager/runtime.properties index 0559cb03b4..a7ff978cb6 100644 --- a/ansible/roles/analytics-druid/templates/middlemanager/runtime.properties +++ b/ansible/roles/analytics-druid/templates/middlemanager/runtime.properties @@ -3,17 +3,17 @@ druid.host={{ ansible_host }} druid.port={{ druid_middlemanager_port }} # Number of tasks per middleManager -druid.worker.capacity={{ druid_middlemanager_worker_cap }} +druid.worker.capacity={{ druid_configs[cluster].druid_middlemanager_worker_cap }} # Task launch parameters -druid.indexer.runner.javaOpts={{ druid_mm_java_opts_array }} +druid.indexer.runner.javaOpts={{ druid_configs[cluster].druid_mm_java_opts_array }} druid.indexer.task.baseTaskDir={{ druid_indexing_task_basedir }} # Peon properties -druid.indexer.fork.property.druid.processing.buffer.sizeBytes={{ druid_middlemanager_peon_processing_bufferBytes }} -druid.indexer.fork.property.druid.processing.numThreads={{ druid_middlemanager_peon_processing_threads }} -druid.indexer.fork.property.druid.segmentCache.locations=[{"path": "{{ druid_middlemanager_peon_segmentcache_path }}", "maxSize": {{ druid_middlemanager_peon_server_maxsize }}}] -druid.indexer.fork.property.druid.server.http.numThreads={{ druid_middlemanager_peon_server_http_numThread }} +druid.indexer.fork.property.druid.processing.buffer.sizeBytes={{ druid_configs[cluster].druid_middlemanager_peon_processing_bufferBytes }} +druid.indexer.fork.property.druid.processing.numThreads={{ druid_configs[cluster].druid_middlemanager_peon_processing_threads }} +druid.indexer.fork.property.druid.processing.numMergeBuffers={{ druid_configs[cluster].druid_middlemanager_peon_processing_num_merge_buffers }} +druid.indexer.fork.property.druid.server.http.numThreads={{ druid_configs[cluster].druid_middlemanager_peon_server_http_numThread }} {% if druid_middlemanager_monitoring_monitors is defined %} diff --git a/ansible/roles/analytics-druid/templates/overlord/jvm.config b/ansible/roles/analytics-druid/templates/overlord/jvm.config index fb642350e5..db02f4a56d 100644 --- a/ansible/roles/analytics-druid/templates/overlord/jvm.config +++ b/ansible/roles/analytics-druid/templates/overlord/jvm.config @@ -1,6 +1,6 @@ -server --Xms{{ druid_overlord_heap_size }} --Xmx{{ druid_overlord_heap_size }} +-Xms{{ druid_configs[cluster].druid_overlord_min_heap_size }} +-Xmx{{ druid_configs[cluster].druid_overlord_max_heap_size }} -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps diff --git a/ansible/roles/analytics-druid/templates/overlord/runtime.properties b/ansible/roles/analytics-druid/templates/overlord/runtime.properties index 314fc008cc..ac9d3eeec2 100644 --- a/ansible/roles/analytics-druid/templates/overlord/runtime.properties +++ b/ansible/roles/analytics-druid/templates/overlord/runtime.properties @@ -2,7 +2,7 @@ druid.service={{ druid_overlord_service }} druid.port={{ druid_overlord_port }} druid.host={{ ansible_host }} -druid.indexer.queue.startDelay={{ druid_indexing_queue_startDelay }} +druid.indexer.queue.startDelay={{ druid_configs[cluster].druid_indexing_queue_startDelay }} druid.indexer.runner.type=remote druid.indexer.storage.type={{ druid_indexing_storage_type }} diff --git a/ansible/roles/analytics-druid/templates/router/jvm.config b/ansible/roles/analytics-druid/templates/router/jvm.config index a50bef1a10..f78c112b59 100644 --- a/ansible/roles/analytics-druid/templates/router/jvm.config +++ b/ansible/roles/analytics-druid/templates/router/jvm.config @@ -1,10 +1,13 @@ -server --Xms{{ druid_router_heap_size }} --Xmx{{ druid_router_heap_size }} --XX:+UseG1GC --XX:+ExitOnOutOfMemoryError +-Xms{{ druid_configs[cluster].druid_router_min_heap_size }} +-Xmx{{ druid_configs[cluster].druid_router_max_heap_size }} -XX:+UseG1GC +-XX:+HeapDumpOnOutOfMemoryError +-XX:+PrintGCTimeStamps +-Xloggc:{{ druid_router_gc_logfile }} +-XX:HeapDumpPath={{ druid_router_heap_dump_file }} -Duser.timezone=UTC -Dfile.encoding=UTF-8 --Djava.io.tmpdir=var/tmp --Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \ No newline at end of file +-Djava.io.tmpdir={{ druid_router_tmp_dir }} +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dlog4j.configurationFile={{ druid_path }}conf/druid/{{ service }}/log4j2.xml diff --git a/ansible/roles/analytics-druid/templates/router/runtime.properties b/ansible/roles/analytics-druid/templates/router/runtime.properties index 85f3395f8c..e7144c9416 100644 --- a/ansible/roles/analytics-druid/templates/router/runtime.properties +++ b/ansible/roles/analytics-druid/templates/router/runtime.properties @@ -17,18 +17,18 @@ # under the License. # -druid.service=druid/router -druid.plaintextPort=8888 +druid.service={{ druid_router_service }} +druid.plaintextPort={{ druid_router_port }} # HTTP proxy -druid.router.http.numConnections={{ druid_router_http_numConnections }} -druid.router.http.readTimeout={{ druid_router_http_readTimeout }} -druid.router.http.numMaxThreads={{ druid_router_http_numMaxThreads }} -druid.server.http.numThreads={{ druid_server_http_numThreads }} +druid.router.http.numConnections={{ druid_configs[cluster].druid_router_http_numConnections }} +druid.router.http.readTimeout={{ druid_configs[cluster].druid_router_http_readTimeout }} +druid.router.http.numMaxThreads={{ druid_configs[cluster].druid_router_http_numMaxThreads }} +druid.server.http.numThreads={{ druid_configs[cluster].druid_server_http_numThreads }} # Service discovery -druid.router.defaultBrokerServiceName=druid/broker -druid.router.coordinatorServiceName=druid/coordinator +druid.router.defaultBrokerServiceName={{ druid_broker_service }} +druid.router.coordinatorServiceName={{ druid_coordinator_service }} # Management proxy to coordinator / overlord: required for unified web console. -druid.router.managementProxy.enabled={{ druid_router_managementProxy_enabled }} +druid.router.managementProxy.enabled={{ druid_configs[cluster].druid_router_managementProxy_enabled }} diff --git a/ansible/roles/analytics-druid/tests/test.yml b/ansible/roles/analytics-druid/tests/test.yml index 85778d7824..bf02bb68f4 100644 --- a/ansible/roles/analytics-druid/tests/test.yml +++ b/ansible/roles/analytics-druid/tests/test.yml @@ -23,3 +23,4 @@ - {role: analytics-druid,when: "'broker' in group_names",druid_role: 'broker', service: 'broker'} - {role: analytics-druid,when: "'historical' in group_names",druid_role: 'historical', service: 'historical'} - {role: analytics-druid,when: "'middlemanager' in group_names",druid_role: 'middlemanager',service: 'middleManager' } + - {role: analytics-druid,when: "'router' in group_names",druid_role: 'router',service: 'router' } diff --git a/ansible/roles/analytics-run-report/tasks/main.yml b/ansible/roles/analytics-run-report/tasks/main.yml index 34a6bc9120..e939316e7c 100644 --- a/ansible/roles/analytics-run-report/tasks/main.yml +++ b/ansible/roles/analytics-run-report/tasks/main.yml @@ -1,5 +1,5 @@ - name: Get the report_config - shell: 'curl -H "Content-Type: application/json" http://{{ analytics_host }}:9000/report/jobs/{{ report_id }}' + shell: 'curl -H "Content-Type: application/json" -H "Authorization: {{ druid_report_token }}" {{ druid_report_url_endpoint }}/{{ report_id }}' register : output - set_fact: @@ -8,6 +8,22 @@ - set_fact: report_config: "{{ config.result.config | to_json }}" +- set_fact: + dynamic: true + when: report_start_date is defined and report_end_date is defined and report_start_date != "" and report_end_date != "" + +- set_fact: + dateRange: "{{config.result.config.reportConfig.dateRange}}" + when: dynamic is defined and dynamic == true + +- name: Load the dynamic interval json + set_fact: {"dynamic_interval":"{{ lookup('template', 'interval.j2')}}"} + when: dynamic is defined and dynamic == true + +- set_fact: + report_config: "{{ config.result.config.reportConfig | combine(dynamic_interval) | to_json }}" + when: dynamic is defined and dynamic == true + - name: Update report config template: src=config.json.j2 dest={{ analytics.home }}/scripts/report_config.json mode=755 owner={{ analytics_user }} group={{ analytics_group }} diff --git a/ansible/roles/analytics-run-report/templates/druid-report-job.j2 b/ansible/roles/analytics-run-report/templates/druid-report-job.j2 index 713a4f74b9..7be2234aff 100644 --- a/ansible/roles/analytics-run-report/templates/druid-report-job.j2 +++ b/ansible/roles/analytics-run-report/templates/druid-report-job.j2 @@ -6,12 +6,13 @@ export SCRIPTS_HOME={{ analytics.home }}/scripts export DP_LOGS={{ analytics.home }}/logs/data-products cd {{ analytics.home }}/scripts +today=$(date "+%Y-%m-%d") libs_path="{{ analytics.home }}/models-{{ model_version }}/data-products-1.0" job_config=`cat $SCRIPTS_HOME/report_config.json` echo "Job Config - $job_config" >> "$DP_LOGS/$today-job-execution.log" -nohup $SPARK_HOME/bin/spark-submit --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.3.2.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "druid_reports" --config "$job_config" >> "$DP_LOGS/$today-job-execution.log" 2>&1 +nohup $SPARK_HOME/bin/spark-submit --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.4.0.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "druid_reports" --config "$job_config" >> "$DP_LOGS/$today-job-execution.log" 2>&1 echo "Job execution completed - $1" >> "$DP_LOGS/$today-job-execution.log" diff --git a/ansible/roles/analytics-run-report/templates/interval.j2 b/ansible/roles/analytics-run-report/templates/interval.j2 new file mode 100644 index 0000000000..0901b42ad8 --- /dev/null +++ b/ansible/roles/analytics-run-report/templates/interval.j2 @@ -0,0 +1 @@ +{"dateRange":{"interval":{"startDate":"{{ report_start_date }}","endDate":"{{ report_end_date }}"},"granularity":"{{ dateRange.granularity | default('all') }}","intervalSlider":{{ dateRange.intervalSlider | default(0) | int }}}} diff --git a/ansible/roles/analytics-spark-provision/defaults/main.yml b/ansible/roles/analytics-spark-provision/defaults/main.yml index c05b0addaf..b7c406b313 100644 --- a/ansible/roles/analytics-spark-provision/defaults/main.yml +++ b/ansible/roles/analytics-spark-provision/defaults/main.yml @@ -3,8 +3,11 @@ analytics: soft_path: /mount/data/analytics base_path: /home/analytics scala_version: 2.11.8 + spark_version: 2.4.4 -model_version: "2.0" +model_version: "2.0" +heap_memory: "-Xmx5120m" + spark: home: "{{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7" public_dns: 54.255.154.146 @@ -13,17 +16,24 @@ spark: host: 172.31.11.117 worker: instances: 1 - cores: 2 - memory: 4g + cores: 6 + memory: 18g driver: - memory: 3g + memory: 12g + cores: 2 + memoryOverhead: 2g executor: - memory: 4g - driver_memory: 7g + instances: 1 + cores: 6 + memory: 14g + memoryOverhead: 2g + driver_memory: 12g executor_extraJavaOptions: -Dconfig.file=/mount/data/analytics/models-{{ model_version }}/{{ env }}.conf -Dlog4j.configurationFile=file:////mount/data/analytics/models-{{ model_version }}/log4j2.xml driver_extraJavaOptions: -Dconfig.file=/mount/data/analytics/models-{{ model_version }}/{{ env }}.conf -Dlog4j.configurationFile=file:////mount/data/analytics/models-{{ model_version }}/log4j2.xml - executor_memory: 2g - heap_conf_str: -Xmx5120m + memory_fraction: 0.3 + storage_fraction: 0.5 + executor_memory: 14g + heap_conf_str: '"-XX:+UseG1GC -XX:MaxGCPauseMillis=100 -Xms250m {{ heap_memory }} -XX:+UseStringDeduplication"' spark_url: https://archive.apache.org/dist/spark/spark-{{ spark_version }}/spark-{{ spark_version }}-bin-hadoop2.7.tgz guava_url: https://repo1.maven.org/maven2/com/google/guava/guava/19.0/guava-19.0.jar @@ -32,3 +42,12 @@ hadoop_aws_url: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.7. java_xmlbuilder_url: https://repo1.maven.org/maven2/com/jamesmurty/utils/java-xmlbuilder/1.1/java-xmlbuilder-1.1.jar hadoop_azure_url: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/2.7.3/hadoop-azure-2.7.3.jar azure_storage: https://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/3.0.0/azure-storage-3.0.0.jar + + +# jets3t s3 config, allows us to configure for s3-like object stores +jets3t_s3_request_signature_version: "{{ s3_request_signature_version }}" +jets3t_s3_endpoint_host: "{% if s3_storage_endpoint %}{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}{% endif %}" +jets3t_s3_disable_dns_buckets: "{{ s3_path_style_access }}" +jets3t_s3_https_only: "{{ s3_https_only }}" +jets3t_s3_default_bucket_location: "{{ s3_default_bucket_location }}" + diff --git a/ansible/roles/analytics-spark-provision/tasks/main.yml b/ansible/roles/analytics-spark-provision/tasks/main.yml index 07bd81d8a8..257aa7c210 100644 --- a/ansible/roles/analytics-spark-provision/tasks/main.yml +++ b/ansible/roles/analytics-spark-provision/tasks/main.yml @@ -1,147 +1,198 @@ ## Provision Spark ## - name: Download Apache Spark {{ spark_version }} hadoop 2.7 - become: yes + become: true become_user: "{{ analytics_user }}" get_url: url={{ spark_url }} dest={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7.tgz force=no owner={{ analytics_user }} group={{ analytics_group }} - name: Unarchive Spark {{ spark_version }} hadoop 2.7 - become: yes + become: true become_user: "{{ analytics_user }}" unarchive: src={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7.tgz dest={{ analytics.home }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7 - name: Download Scala 2.11.8 - become: yes + become: true become_user: "{{ analytics_user }}" shell: wget -O {{ analytics.soft_path }}/scala-{{ scala_version }}.tgz https://downloads.lightbend.com/scala/{{ scala_version }}/scala-{{ scala_version }}.tgz - name: Unarchive Scala 2.11.8 - become: yes + become: true become_user: "{{ analytics_user }}" unarchive: src={{ analytics.soft_path }}/scala-{{ scala_version }}.tgz dest={{ analytics.soft_path }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.soft_path }}/scala-{{ scala_version }} - name: Remove Guava 14.0.1 from Spark jars folder - become: yes + become: true become_user: "{{ analytics_user }}" file: path={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/jars/guava-14.0.1.jar state=absent - name: Download Guava 19.0 and copy to Spark jars folder - become: yes + become: true become_user: "{{ analytics_user }}" get_url: url={{ guava_url }} dest={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/jars/guava-19.0.jar timeout=1000 force=no owner={{ analytics_user }} group={{ analytics_group }} - name: Remove jets3t 0.9.3 from Spark jars folder - become: yes + become: true become_user: "{{ analytics_user }}" file: path={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/jars/jets3t-0.9.3.jar state=absent - name: Download jets3t 0.9.4 and copy to Spark jars folder - become: yes + become: true become_user: "{{ analytics_user }}" get_url: url={{ jets3t_url }} dest={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/jars/jets3t-0.9.4.jar timeout=1000 force=no owner={{ analytics_user }} group={{ analytics_group }} - name: Download hadoop-aws 2.7.3 and copy to Spark jars folder - become: yes + become: true become_user: "{{ analytics_user }}" get_url: url={{ hadoop_aws_url }} dest={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/jars/hadoop-aws-2.7.3.jar timeout=1000 force=no owner={{ analytics_user }} group={{ analytics_group }} - name: Download hadoop-azure 2.7.3 and copy to Spark jars folder - become: yes + become: true become_user: "{{ analytics_user }}" get_url: url={{ hadoop_azure_url }} dest={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/jars/hadoop-azure-2.7.3.jar timeout=1000 force=no owner={{ analytics_user }} group={{ analytics_group }} - - name: Download azure_storage 3.0.0 and copy to Spark jars folder - become: yes + become: true become_user: "{{ analytics_user }}" get_url: url={{ azure_storage }} dest={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/jars/azure-storage-3.0.0.jar timeout=1000 force=no owner={{ analytics_user }} group={{ analytics_group }} - - name: Remove java-xmlbuilder 1.0 from Spark jars folder - become: yes + become: true become_user: "{{ analytics_user }}" file: path={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/jars/java-xmlbuilder-1.0.jar state=absent - name: Download java-xmlbuilder 1.1 in to Spark jars folder - become: yes + become: true become_user: "{{ analytics_user }}" get_url: url={{ java_xmlbuilder_url }} dest={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/jars/java-xmlbuilder-1.1.jar timeout=1000 force=no owner={{ analytics_user }} group={{ analytics_group }} - name: Update spark default conf - become: yes + become: true become_user: "{{ analytics_user }}" template: src=spark-defaults.j2 dest={{ spark.home }}/conf/spark-defaults.conf mode=755 owner={{ analytics_user }} group={{ analytics_group }} - name: Update spark env - become: yes + become: true become_user: "{{ analytics_user }}" template: src=spark-env.j2 dest={{ spark.home }}/conf/spark-env.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }} - name: Copy Jets3t properties file - become: yes + become: true become_user: "{{ analytics_user }}" template: src=jets3t.j2 dest={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/conf/jets3t.properties mode=755 owner={{ analytics_user }} group={{ analytics_group }} - ## Provision DS ## - name: Install packages - become: yes + become: true apt: - name: "{{ item }}" - state: installed + name: + [ + "gcc", + "autoconf", + "curl", + "g++", + "gnupg", + "automake", + "bison", + "libc6-dev", + "libffi-dev", + "libgdbm-dev", + "libncurses5-dev", + "libsqlite3-dev", + "pkg-config", + "sqlite3", + "zlib1g-dev", + "libtool", + "libyaml-dev", + "make", + "libgmp-dev", + "libreadline-dev", + "libssl-dev", + "gpgv2", + "gem", + "ruby", + "gnupg2", + "virtualenv", + ] + state: present update_cache: true cache_valid_time: 3600 - with_items: - - gem - - ruby - - gnupg2 -- name: Import rvm keys - become: yes +- name: Change ownership + file: + path: "{{ analytics.base_path }}" + owner: "{{ analytics_user }}" + group: "{{ analytics_user }}" + recurse: yes + become: true + +- name: Import keys + become: true become_user: "{{ analytics_user }}" - shell: command curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - + shell: |- + curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - + curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - + gpg2 --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB - name: Install rvm - become: yes + become: true become_user: "{{ analytics_user }}" - shell: curl -sSL https://get.rvm.io | bash -s stable + shell: "curl -sSL https://get.rvm.io | bash -s stable" -- name: source ruby script - become: yes +- name: Add RVM to PATH and source it in .bashrc + become: true become_user: "{{ analytics_user }}" - shell: echo "source {{ analytics.base_path }}/.rvm/scripts/rvm" >> ~/.bashrc - args: - executable: /bin/bash + lineinfile: + path: "{{ analytics.base_path }}/.bashrc" + line: 'export PATH="{{ analytics.base_path }}/.rvm/bin:$PATH"' + create: yes + state: present -- name: Install latest ruby - become: yes - shell: "{{ analytics.base_path }}/.rvm/bin/rvm install ruby-2.6" +- name: Source RVM script in .bashrc + become: true + become_user: "{{ analytics_user }}" + lineinfile: + path: "{{ analytics.base_path }}/.bashrc" + line: "source {{ analytics.base_path }}/.rvm/scripts/rvm" + state: present + +- name: Install openssl + become: true + become_user: "{{ analytics_user }}" + shell: "bash -ilc 'rvm pkg install openssl'" -- name: Set latest ruby as default - become: yes +- name: Install latest ruby + become: true become_user: "{{ analytics_user }}" - shell: "{{ analytics.base_path }}/.rvm/bin/rvm --default use ruby-2.6" + shell: "bash -ilc 'rvm install ruby-2.6 --with-openssl-dir={{ analytics.base_path }}/.rvm/usr'" - name: Install ruby-kafka - gem: - name: ruby-kafka - become: yes + become: true become_user: "{{ analytics_user }}" + shell: "bash -ilc 'rvm --default use ruby-2.6 && gem install --user-install --no-document ruby-kafka'" - name: Download Kafka-2.11 - become: yes + become: true become_user: "{{ analytics_user }}" - get_url: url=http://downloads.mesosphere.com/kafka/assets/kafka_2.11-0.10.1.0.tgz dest={{ analytics.home }}/kafka_2.11-0.10.1.0.tgz force=no owner={{ analytics_user }} group={{ analytics_group }} + get_url: url=https://archive.apache.org/dist/kafka/0.10.1.0/kafka_2.11-0.10.1.0.tgz dest={{ analytics.home }}/kafka_2.11-0.10.1.0.tgz force=no owner={{ analytics_user }} group={{ analytics_group }} tags: - kafka-provision -- name: Unarchive Kafka - become: yes +- name: Unarchive Kafka + become: true become_user: "{{ analytics_user }}" unarchive: src={{ analytics.soft_path }}/kafka_2.11-0.10.1.0.tgz dest={{ analytics.home }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.home }}/kafka_2.11-0.10.1.0 tags: - kafka-provision +- name: Add python ppa + become: true + apt_repository: + repo: ppa:deadsnakes/ppa - - +- name: Install python 3.7 + become: true + apt: + name: python3.7 + state: present + update_cache: true + cache_valid_time: 3600 diff --git a/ansible/roles/analytics-spark-provision/templates/jets3t.j2 b/ansible/roles/analytics-spark-provision/templates/jets3t.j2 index df9a223127..b812d12830 100644 --- a/ansible/roles/analytics-spark-provision/templates/jets3t.j2 +++ b/ansible/roles/analytics-spark-provision/templates/jets3t.j2 @@ -1,3 +1,9 @@ -storage-service.request-signature-version=AWS4-HMAC-SHA256 -s3service.s3-endpoint=s3-ap-south-1.amazonaws.com +storage-service.request-signature-version={{ jets3t_s3_request_signature_version }} +s3service.s3-endpoint={% if jets3t_s3_endpoint_host %}{{ jets3t_s3_endpoint_host }}{% else %}s3-ap-south-1.amazonaws.com{% endif %} + +s3service.disable-dns-buckets={{ jets3t_s3_disable_dns_buckets }} +s3service.https-only={{ jets3t_s3_https_only }} +{% if jets3t_s3_default_bucket_location %} +s3service.default-bucket-location={{ jets3t_s3_default_bucket_location }} +{% endif %} uploads.stream-retry-buffer-size=2147483646 diff --git a/ansible/roles/analytics-spark-provision/templates/spark-defaults.j2 b/ansible/roles/analytics-spark-provision/templates/spark-defaults.j2 index 9bc66c466f..40560a320f 100644 --- a/ansible/roles/analytics-spark-provision/templates/spark-defaults.j2 +++ b/ansible/roles/analytics-spark-provision/templates/spark-defaults.j2 @@ -2,8 +2,20 @@ # This is useful for setting default environmental settings. spark.master {{ spark.master.url }} -spark.driver.memory {{ spark.driver_memory }} spark.executor.extraJavaOptions {{ spark.executor_extraJavaOptions }} spark.driver.extraJavaOptions {{ spark.driver_extraJavaOptions }} -spark.executor.memory {{ spark.executor_memory }} spark.local.dir {{ analytics.home }}/tmp + +# spark.worker.instances {{ spark.worker.instances }} +# spark.worker.cores {{ spark.worker.cores }} +# spark.worker.memory {{ spark.worker.memory }} + +# these settings are not applicable in mode=local +# spark.executor.instances {{ spark.executor.instances }} +# spark.executor.cores {{ spark.executor.cores }} +# spark.executor.memory {{ spark.executor.memory }} +# spark.executor.memoryOverhead {{ spark.executor.memoryOverhead }} + +spark.driver.cores {{ spark.driver.cores }} +spark.driver.memory {{ spark.driver.memory }} +# spark.driver.memoryOverhead {{ spark.driver.memoryOverhead }} diff --git a/ansible/roles/analytics-spark-provision/templates/spark-env.j2 b/ansible/roles/analytics-spark-provision/templates/spark-env.j2 index ad8c5c4e9b..e52fb1182c 100644 --- a/ansible/roles/analytics-spark-provision/templates/spark-env.j2 +++ b/ansible/roles/analytics-spark-provision/templates/spark-env.j2 @@ -64,13 +64,16 @@ # - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER) # - SPARK_NICENESS The scheduling priority for daemons. (Default: 0) export SPARK_MASTER_HOST="{{ spark.master.host }}" + export SPARK_DRIVER_MEMORY={{ spark.driver.memory }} +export SPARK_DRIVER_CORES={{ spark.driver.cores }} + export SPARK_WORKER_CORES={{ spark.worker.cores }} export SPARK_WORKER_MEMORY={{ spark.worker.memory }} export SPARK_WORKER_INSTANCES={{ spark.worker.instances }} + export SPARK_EXECUTOR_MEMORY={{ spark.executor.memory }} +export SPARK_EXECUTOR_CORES={{ spark.executor.cores }} +export SPARK_EXECUTOR_INSTANCES={{ spark.executor.instances }} + export SPARK_PUBLIC_DNS="{{ spark.public_dns }}" -export reports_storage_key={{sunbird_private_storage_account_name}} -export reports_storage_secret={{sunbird_private_storage_account_key}} -export azure_storage_key={{sunbird_private_storage_account_name}} -export azure_storage_secret={{sunbird_private_storage_account_key}} diff --git a/ansible/roles/ansible-kafka-upgrade/defaults/main.yml b/ansible/roles/ansible-kafka-upgrade/defaults/main.yml index ec93841925..0fa8d0b5cf 100644 --- a/ansible/roles/ansible-kafka-upgrade/defaults/main.yml +++ b/ansible/roles/ansible-kafka-upgrade/defaults/main.yml @@ -2,7 +2,7 @@ apache_mirror: http://apache.mirrors.tds.net apache_mirror_archive: https://archive.apache.org/dist kafka_hosts: "{{ groups['processing-cluster-kafka']|join(':9092,')}}:9092" -kafka_version: 1.1.0 +kafka_version: 2.8.0 kafka_version_old: 0.10.0.1 kafka_scala_version: 2.12 run_mode: Deploy @@ -53,4 +53,4 @@ user: ecosystem kafka_systemd_service: /etc/systemd/system/kafka.service kafka_auto_create_topics: "false" delete_topic_enable: "true" -kafka_openfile_limit: 32768 +openfile_limit: 40768 diff --git a/ansible/roles/ansible-kafka-upgrade/tasks/start.yml b/ansible/roles/ansible-kafka-upgrade/tasks/start.yml index 814f45e4c5..26d9a1bb5f 100644 --- a/ansible/roles/ansible-kafka-upgrade/tasks/start.yml +++ b/ansible/roles/ansible-kafka-upgrade/tasks/start.yml @@ -3,6 +3,10 @@ wait_for: host={{zookeeper_listen_address| default('localhost')}} port={{zookeeper_port}} state=started timeout=30 when: verify +- name: Waiting 1m for kafka to settle + pause: + minutes: 1 + - name: Enable and Start Kafka service: name=kafka state=started enabled=yes become: yes diff --git a/ansible/roles/ansible-kafka-upgrade/templates/server.properties.j2 b/ansible/roles/ansible-kafka-upgrade/templates/server.properties.j2 index 29a97b924c..639e649b57 100644 --- a/ansible/roles/ansible-kafka-upgrade/templates/server.properties.j2 +++ b/ansible/roles/ansible-kafka-upgrade/templates/server.properties.j2 @@ -5,6 +5,8 @@ {%- set url_host = url.split(':')[0] -%} {%- if url_host == ansible_fqdn or url_host in ansible_all_ipv4_addresses -%} broker.id={{loop.index0 + 1}} +listeners=PLAINTEXT://0.0.0.0:9092 +advertised.listeners=PLAINTEXT://{{ansible_host}}:9092 ############################# Socket Server Settings ############################# @@ -123,13 +125,13 @@ log.cleaner.enable=false # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". # You can also append an optional chroot string to the urls to specify the # root directory for all kafka znodes. -zookeeper.connect={% for host in groups['zookeeper'] %}{{ hostvars[host].ansible_default_ipv4.address }}:{{zookeeper_port}}{% if not loop.last %},{% endif %}{% endfor %} +zookeeper.connect={% for host in groups['zookeeper'] %}{{ hostvars[host].inventory_hostname }}:{{zookeeper_port}}{% if not loop.last %},{% endif %}{% endfor %} # Timeout in ms for connecting to zookeeper zookeeper.connection.timeout.ms={{zookeeper_connection_timeout_ms}} -inter.broker.protocol.version={{kafka_version_old}} -log.message.format.version={{kafka_version_old}} +#inter.broker.protocol.version={{kafka_version}} +#log.message.format.version={{kafka_version}} ############################# Group Coordinator Settings ############################# diff --git a/ansible/roles/artifacts-download-azure/tasks/main.yml b/ansible/roles/artifacts-download-azure/tasks/main.yml index db79bc213f..4531b222e5 100644 --- a/ansible/roles/artifacts-download-azure/tasks/main.yml +++ b/ansible/roles/artifacts-download-azure/tasks/main.yml @@ -1,8 +1,12 @@ --- -- name: Ensure azure blob storage container exists - command: az storage container exists --name {{ artifacts_container }} +- name: Nothing + command: echo "" + +#- name: Ensure azure blob storage container exists +# command: az storage container exists --name {{ artifacts_container }} +# +#- name: Download from azure blob storage +# command: az storage blob download -c {{ artifacts_container }} --name {{ artifact }} -f {{ artifact_path }} +# async: 3600 +# poll: 10 -- name: Download from azure blob storage - command: az storage blob download -c {{ artifacts_container }} --name {{ artifact }} -f {{ artifact_path }} - async: 3600 - poll: 10 diff --git a/ansible/roles/artifacts-upload-azure/tasks/main.yml b/ansible/roles/artifacts-upload-azure/tasks/main.yml index 785dc1a455..cca7cc3e8f 100644 --- a/ansible/roles/artifacts-upload-azure/tasks/main.yml +++ b/ansible/roles/artifacts-upload-azure/tasks/main.yml @@ -1,8 +1,11 @@ --- -- name: Ensure azure blob storage container exists - command: az storage container create --name {{ artifacts_container }} +#- name: Ensure azure blob storage container exists +# command: az storage container create --name {{ artifacts_container }} +# +#- name: Upload to azure blob storage +# command: az storage blob upload -c {{ artifacts_container }} --name {{ artifact }} -f {{ artifact_path }} +# async: 3600 +# poll: 10 -- name: Upload to azure blob storage - command: az storage blob upload -c {{ artifacts_container }} --name {{ artifact }} -f {{ artifact_path }} - async: 3600 - poll: 10 +- name: Nothing + command: echo "" \ No newline at end of file diff --git a/ansible/roles/artifacts-upload-gcp/tasks/main.yml b/ansible/roles/artifacts-upload-gcp/tasks/main.yml new file mode 100644 index 0000000000..2e58569a6d --- /dev/null +++ b/ansible/roles/artifacts-upload-gcp/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- name: Set GCP environment variables + set_fact: + gcp_environment: + GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcp_service_account.json" + +- name: Create temporary directory for GCP service account file + file: + path: /tmp/gcp_service_account.json + state: touch + +- name: Write GCP service account file + copy: + content: "{{ service_account }}" + dest: /tmp/gcp_service_account.json + mode: "0600" + +- name: Authenticate gcloud login + shell: gcloud auth activate-service-account --key-file /tmp/gcp_service_account.json + +- name: Ensure Google Cloud Storage bucket exists + shell: gsutil mb gs://{{ artifacts_container }} || true + environment: "{{ gcp_environment }}" + +- name: Upload to Google Cloud Storage + shell: gsutil cp {{ artifact_path }} gs://{{ artifacts_container }}/{{ artifact }} + async: 3600 + poll: 10 + +- name: Remove temporary GCP service account file + file: + path: /tmp/gcp_service_account.json + state: absent diff --git a/ansible/roles/azure-cli/tasks/main.yml b/ansible/roles/azure-cli/tasks/main.yml index 6c05b3e9f6..60d22aae73 100644 --- a/ansible/roles/azure-cli/tasks/main.yml +++ b/ansible/roles/azure-cli/tasks/main.yml @@ -1,13 +1,19 @@ -#- name: Install azure cli -# shell: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash -# become: yes -- name: Add Azure apt repository - apt_repository: repo='deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ xenial main' state=present - - name: Import Azure signing key become: yes shell: curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +- name: Add Azure apt repository + apt_repository: repo='deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ {{ ansible_distribution_release }} main' state=present + +- name: Add distribution release security apt repository + apt_repository: repo='deb http://security.ubuntu.com/ubuntu bionic-security main' state=present + +- name: install azure cli dependency + apt: name={{ item }} state=present update_cache=yes + with_items: + - libssl1.0-dev + when: ansible_distribution_release == "focal" + - name: ensure azure-cli and apt-transport-https is installed apt: name={{ item }} state=present update_cache=yes with_items: diff --git a/ansible/roles/azure-hdinsight-spark-cluster/defaults/main.yml b/ansible/roles/azure-hdinsight-spark-cluster/defaults/main.yml new file mode 100644 index 0000000000..95b0b73e0d --- /dev/null +++ b/ansible/roles/azure-hdinsight-spark-cluster/defaults/main.yml @@ -0,0 +1,7 @@ + +spark-folder: /usr/hdp/current/spark2-client +guava_version: 19.0 +log4j_version: 2.5 +guava_url: https://repo1.maven.org/maven2/com/google/guava/guava/{{guava_version}}/guava-{{guava_version}}.jar +log4j_core_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/{{log4j_version}}/log4j-core-{{log4j_version}}.jar +log4j_api_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/{{log4j_version}}/log4j-api-{{log4j_version}}.jar diff --git a/ansible/roles/azure-hdinsight-spark-cluster/tasks/main.yml b/ansible/roles/azure-hdinsight-spark-cluster/tasks/main.yml new file mode 100644 index 0000000000..d8f4d3cc50 --- /dev/null +++ b/ansible/roles/azure-hdinsight-spark-cluster/tasks/main.yml @@ -0,0 +1,13 @@ +- name: copy cluster creation script + template: + src: create-cluster.sh.j2 + dest: /tmp/create-cluster.sh + mode: 0755 + when: cluster_state == "create_cluster" + +- name: copy cluster deletion script + template: + src: delete-cluster.sh.j2 + dest: /tmp/delete-cluster.sh + mode: 0755 + when: cluster_state == "delete_cluster" diff --git a/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2 b/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2 new file mode 100644 index 0000000000..8a7703c936 --- /dev/null +++ b/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2 @@ -0,0 +1,53 @@ +#!/bin/bash + +cluster_name="{{env}}-spark-cluster" +resource_group="{{azure_resource_group}}" +cluster_type=spark +component_version="{{component_version}}" +headnode_size="{{headnode_size}}" +location="{{location}}" +http_user=admin +http_password="{{azure_spark_cluster_http_password}}" +storage_account_name="{{sunbird_private_storage_account_name}}" +storage_account_key="{{sunbird_private_storage_account_key}}" +storage_container="{{spark_storage_container}}" +subnet_name="{{subnet_name}}" +vnet_name="{{vnet_name}}" +version="{{version}}" +headnode_size="{{headnode_size}}" +workernode_size="{{workernode_size}}" +workernode_count="{{workernode_count}}" +ssh_user="{{ ansible_ssh_user }}" +ssh_pub_key="{{ssh_public_key_deployer}}" +#ssh_pass="{{azure_spark_cluster_http_password}}" + + + +#az login +#az account set --subscription {{subscription_id}} + + +az login --service-principal -u $1 -p $2 --tenant {{tenant_id}} +az account set --subscription {{subscription_id}} + +az hdinsight create --name $cluster_name --resource-group $resource_group --type $cluster_type --cluster-tier Standard --component-version Spark=$component_version --headnode-size $headnode_size --location $location --http-password $http_password --http-user $http_user --ssh-user $ssh_user --ssh-public-key "$ssh_pub_key" --storage-account $storage_account_name --storage-account-key $storage_account_key --storage-container $storage_container --subnet $subnet_name --vnet-name $vnet_name --version $version --workernode-count $workernode_count --workernode-size $workernode_size --tags Name=$cluster_name + +mapfile -t nic_array < <( az network nic list -g $resource_group -o table | grep 'headnode\|workernode' | awk '{print $5}' ) +chmod 0777 "{{inventory_dir}}/hosts" +host_line=$(sed -n '/\[spark-hdinsight\]/=' "{{inventory_dir}}/hosts") +del_line=$((host_line+1)) +sed -i $del_line'd' "{{inventory_dir}}/hosts" + + +for nic in "${nic_array[@]}"; +do + tag_name=$(az network nic show -n $nic -g $resource_group | grep "$cluster_name" | cut -d':' -f2 | tr -d '"' | tr -d ' ') + if [ "$tag_name" == "$cluster_name" ]; then + private_ip=$(az network nic show -n $nic -g $resource_group | grep '"privateIpAddress":' | cut -d':' -f 2 | cut -d ',' -f1 | tr -d '"' | tr -d ' ') + name=$(echo $nic | cut -d'-' -f2) + count=$(echo $nic | cut -d'-' -f3) + hostname=$name-$count + echo $hostname : $private_ip + sed -i '/^\[spark-hdinsight\]/a '$private_ip'' "{{inventory_dir}}/hosts" + fi +done diff --git a/ansible/roles/azure-hdinsight-spark-cluster/templates/delete-cluster.sh.j2 b/ansible/roles/azure-hdinsight-spark-cluster/templates/delete-cluster.sh.j2 new file mode 100644 index 0000000000..7407049a9a --- /dev/null +++ b/ansible/roles/azure-hdinsight-spark-cluster/templates/delete-cluster.sh.j2 @@ -0,0 +1,13 @@ + +#!/bin/bash + +cluster_name="{{env}}-spark-cluster" +resource_group="{{azure_resource_group}}" + +#az login +#az account set --subscription {{subscription_id}} + +az login --service-principal -u $1 -p $2 --tenant {{tenant_id}} +az account set --subscription {{subscription_id}} + +az hdinsight delete --name $cluster_name --resource-group $resource_group --yes diff --git a/ansible/roles/cassandra-3.7.0/defaults/main.yml b/ansible/roles/cassandra-3.7.0/defaults/main.yml deleted file mode 100644 index 32a1c74999..0000000000 --- a/ansible/roles/cassandra-3.7.0/defaults/main.yml +++ /dev/null @@ -1,16 +0,0 @@ -cassandra_repo: "deb http://debian.datastax.com/datastax-ddc 3.7 main" -data_file_directories: /data/cassandra/data -commitlog_directory: /data/cassandra/commitlog -saved_caches_directory: /data/cassandra/saved_caches -seeds: "{{ inventory_hostname }}" -listen_address: "{{ inventory_hostname }}" -broadcast_rpc_address: "{{ inventory_hostname }}" -paths: ['/data/cassandra', '/data/cassandra/commitlog', '/data/cassandra/data', '/data/cassandra/saved_caches', '/data/cassandra/logs'] -user: cassandra -cassandra_lib: "/usr/share/cassandra/lib/" -cassandra_lib_src: "templates/cassandra-lib.tgz" -cassandra_conf: "/etc/cassandra/" -cassandra_env_file: "/etc/cassandra/cassandra-env.sh" -user_home: /home/analytics -keyspace_prefix: '{{ env }}_' -replication_factor: 1 diff --git a/ansible/roles/cassandra-3.7.0/handlers/main.yml b/ansible/roles/cassandra-3.7.0/handlers/main.yml deleted file mode 100644 index 60bf5669e9..0000000000 --- a/ansible/roles/cassandra-3.7.0/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ -# handlers file for ansible-cassandra -- name: "restart cassandra" - service: - name: "cassandra" - state: "restarted" \ No newline at end of file diff --git a/ansible/roles/cassandra-3.7.0/tasks/main.yml b/ansible/roles/cassandra-3.7.0/tasks/main.yml deleted file mode 100644 index 34c8c0ee57..0000000000 --- a/ansible/roles/cassandra-3.7.0/tasks/main.yml +++ /dev/null @@ -1,84 +0,0 @@ -## Cassandra Provisioning ## -- name: Cassandra | Make sure the ansible required dependencies are installed - become: yes - apt: pkg={{ item }} state=present - with_items: - - python-pycurl - - python-pip - -- name: install cassandra driver - become: yes - pip: name=cassandra-driver state=present - -- name: export cqlsh variable - become: yes - lineinfile: - dest: /etc/environment - line: 'CQLSH_NO_BUNDLED=TRUE' - state: present - -- name: Cassandra | Add the datastax repository apt-key - become: yes - apt_key: - url: "http://debian.datastax.com/debian/repo_key" - state: present - -- name: Cassandra | Add the cassandra 3.7 repository - become: yes - apt_repository: - repo: "{{ cassandra_repo }}" - state: present - -- name: Cassandra | Install the cassandra package - become: yes - apt: - name: cassandra - state: present - update_cache: yes - allow_unauthenticated: yes - -- name: Cassandra | Update the cassandra configuration - become: yes - template: - src: "cassandra.j2" - dest: "/etc/cassandra/cassandra.yaml" - mode: 0644 - -- name: Cassandra | Update the cassandra rackdc configuration - become: yes - template: - src: "cassandra-rackdc.properties.j2" - dest: "/etc/cassandra/cassandra-rackdc.properties" - mode: 0644 - -- name: Cassandra | Create directories for cassandra - become: yes - file: - path: "{{ item }}" - owner: cassandra - group: cassandra - state: directory - with_items: "{{ paths }}" - - -- name: Clear cassandra libraries - file: path={{ item }} state=absent - become: yes - with_fileglob: - - "{{ cassandra_lib }}/metrics-core-*.jar" - - "{{ cassandra_lib }}/reporter-config-base-*.jar" - - "{{ cassandra_lib }}/reporter-config3-*.jar" - -- name: Unarchive the files - unarchive: - src: "{{ cassandra_lib_src }}" - dest: "{{ cassandra_lib }}" - become: yes - -- name: Add JAVA_OPTS - lineinfile: dest="{{ cassandra_env_file }}" line='JVM_OPTS="$JVM_OPTS -Dcassandra.metricsReporterConfigFile=graphite.yaml"' - become: yes - -- name: Cassandra | Restart cassandra - become: yes - meta: flush_handlers diff --git a/ansible/roles/cassandra-3.7.0/templates/cassandra-lib.tgz b/ansible/roles/cassandra-3.7.0/templates/cassandra-lib.tgz deleted file mode 100644 index 2dcfaed218..0000000000 Binary files a/ansible/roles/cassandra-3.7.0/templates/cassandra-lib.tgz and /dev/null differ diff --git a/ansible/roles/cassandra-3.7.0/templates/cassandra-rackdc.properties.j2 b/ansible/roles/cassandra-3.7.0/templates/cassandra-rackdc.properties.j2 deleted file mode 100644 index 180d1f1904..0000000000 --- a/ansible/roles/cassandra-3.7.0/templates/cassandra-rackdc.properties.j2 +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# These properties are used with GossipingPropertyFileSnitch and will -# indicate the rack and dc for this node -dc=ap-south -rack=1a - -# Add a suffix to a datacenter name. Used by the Ec2Snitch and Ec2MultiRegionSnitch -# to append a string to the EC2 region name. -#dc_suffix= - -# Uncomment the following line to make this snitch prefer the internal ip when possible, as the Ec2MultiRegionSnitch does. -# prefer_local=true diff --git a/ansible/roles/cassandra-3.7.0/templates/cassandra.j2 b/ansible/roles/cassandra-3.7.0/templates/cassandra.j2 deleted file mode 100644 index 7114ea120f..0000000000 --- a/ansible/roles/cassandra-3.7.0/templates/cassandra.j2 +++ /dev/null @@ -1,180 +0,0 @@ -cluster_name: 'Cassandra Cluster - {{env}}' - -num_tokens: 256 - -hinted_handoff_enabled: true - -max_hint_window_in_ms: 10800000 - -hinted_handoff_throttle_in_kb: 1024 - -max_hints_delivery_threads: 2 - -batchlog_replay_throttle_in_kb: 1024 - -authenticator: AllowAllAuthenticator - -authorizer: AllowAllAuthorizer - -role_manager: CassandraRoleManager - -roles_validity_in_ms: 2000 - -permissions_validity_in_ms: 2000 - -partitioner: org.apache.cassandra.dht.Murmur3Partitioner - -data_file_directories: - - {{data_file_directories}} - -commitlog_directory: {{commitlog_directory}} - -disk_failure_policy: stop - -commit_failure_policy: stop - -key_cache_size_in_mb: - -key_cache_save_period: 14400 - -row_cache_size_in_mb: 0 - -row_cache_save_period: 0 - -counter_cache_size_in_mb: - -counter_cache_save_period: 7200 - -saved_caches_directory: {{saved_caches_directory}} - -commitlog_sync: periodic - -commitlog_sync_period_in_ms: 10000 - -commitlog_segment_size_in_mb: 32 - -seed_provider: - - class_name: org.apache.cassandra.locator.SimpleSeedProvider - parameters: - - seeds: "{{seeds}}" - -concurrent_reads: 32 - -concurrent_writes: 32 - -concurrent_counter_writes: 32 - -concurrent_materialized_view_writes: 32 - -memtable_allocation_type: heap_buffers - -index_summary_capacity_in_mb: - -index_summary_resize_interval_in_minutes: 60 - -trickle_fsync: false - -trickle_fsync_interval_in_kb: 10240 - -storage_port: 7000 - -ssl_storage_port: 7001 - -listen_address: {{listen_address}} - -start_native_transport: true - -native_transport_port: 9042 - -start_rpc: false - -rpc_address: 0.0.0.0 - -rpc_port: 9160 - -broadcast_rpc_address: {{broadcast_rpc_address}} - -rpc_keepalive: true - -rpc_server_type: sync - -thrift_framed_transport_size_in_mb: 15 - -incremental_backups: false - -snapshot_before_compaction: false - -auto_snapshot: true - -tombstone_warn_threshold: 1000 - -tombstone_failure_threshold: 100000 - -column_index_size_in_kb: 64 - -batch_size_warn_threshold_in_kb: 5 - -batch_size_fail_threshold_in_kb: 50 - -unlogged_batch_across_partitions_warn_threshold: 10 - -compaction_throughput_mb_per_sec: 16 - -compaction_large_partition_warning_threshold_mb: 100 - -sstable_preemptive_open_interval_in_mb: 50 - -read_request_timeout_in_ms: 5000 - -range_request_timeout_in_ms: 10000 - -write_request_timeout_in_ms: 2000 - -counter_write_request_timeout_in_ms: 5000 - -cas_contention_timeout_in_ms: 1000 - -truncate_request_timeout_in_ms: 60000 - -request_timeout_in_ms: 10000 - -cross_node_timeout: false - -phi_convict_threshold: 12 - -endpoint_snitch: GossipingPropertyFileSnitch - -dynamic_snitch_update_interval_in_ms: 100 - -dynamic_snitch_reset_interval_in_ms: 600000 - -dynamic_snitch_badness_threshold: 0.1 - -request_scheduler: org.apache.cassandra.scheduler.NoScheduler - -server_encryption_options: - internode_encryption: none - keystore: conf/.keystore - keystore_password: cassandra - truststore: conf/.truststore - truststore_password: cassandra - -client_encryption_options: - enabled: false - optional: false - keystore: conf/.keystore - keystore_password: cassandra - -internode_compression: all - -inter_dc_tcp_nodelay: false - -tracetype_query_ttl: 86400 - -tracetype_repair_ttl: 604800 - -enable_user_defined_functions: false - -windows_timer_interval: 1 - -auto_bootstrap: false \ No newline at end of file diff --git a/ansible/roles/cassandra-restore/defaults/main.yml b/ansible/roles/cassandra-restore/defaults/main.yml index a58c0a40b2..6353e09287 100644 --- a/ansible/roles/cassandra-restore/defaults/main.yml +++ b/ansible/roles/cassandra-restore/defaults/main.yml @@ -1 +1 @@ -user_home: /home/deployer \ No newline at end of file +user_home: "/home/{{ ansible_ssh_user }}/" diff --git a/ansible/roles/content-snapshot-indexer/defaults/main.yml b/ansible/roles/content-snapshot-indexer/defaults/main.yml index c38ddf411a..51c9979b04 100644 --- a/ansible/roles/content-snapshot-indexer/defaults/main.yml +++ b/ansible/roles/content-snapshot-indexer/defaults/main.yml @@ -2,14 +2,15 @@ analytics_user: analytics analytics_group: analytics content_snapshot_path: "/mount/data/analytics/content-snapshot" content_snapshot_jar_path: "{{content_snapshot_path}}/jars" # Add content snapshot jar file path. -content_snapshot_jar_name: "adhoc-jobs-1.0.jar" -content_snapshot_distribuction_name: "adhoc-jobs-1.0-distribution.tar.gz" +content_snapshot_jar_name: "etl-jobs-1.0.jar" +content_snapshot_distribuction_name: "etl-jobs-1.0-distribution.tar.gz" spark_home: "/mount/data/analytics/spark-2.4.4-bin-hadoop2.7" redis: - host: "{{metadata_redis_host}}" # Redis DP IP + host: "{{metadata2_redis_host}}" # Redis DP IP port: "6379" user: - index: "4" + index: "12" # For temporary will change once verification is done + input_index: "4" # Input index for string format source_key: "id" content: index: "5" @@ -23,8 +24,9 @@ redis: min_evict_idle_seconds: 120 time_evict_run_second: 300 max: 20 - max_pipeline_size: "1000" + max_pipeline_size: "100000" key_expiry_seconds: 3600 + scan_count: "100000" elastic_search: host: "{{ lp_composite_search_host }}" # compositesearch index @@ -44,12 +46,15 @@ cloud_storage: container: "telemetry-data-store" # Container is different in all env so override this. object_key: "druid-content-snapshot/snapshot.txt" provider: "azure" - account_name: "{{ azure_storage_account }}" - account_key: "{{ azure_storage_secret }}" + account_name: "{{sunbird_public_storage_account_name}}" + account_key: "{{sunbird_public_storage_account_key}}" cassandra: - host: "{{groups['lp-cassandra'][0]}}" ## LMS-Cassandra IP Address. - keyspace: 'sunbird' + host: "{{lp_cassandra_host}}" ## LMS-Cassandra IP Address. + keyspace: "sunbird" + read_timeout: "500000" + retry_count: "100" + input_consistency_level: "LOCAL_QUORUM" user: table: "user" device_profile: @@ -57,9 +62,9 @@ cassandra: new_table: "device_profile_temp" druid: - coordinator_host: "{{groups['coordinator'][0]}}" + coordinator_host: "{{ groups['rollup-coordinator'][0] | default(groups['raw-coordinator'][0]) }}" data_source: "content-model-snapshot" - ingestion_spec_path: "{{ content_snapshot_jar_path }}/adhoc-jobs-1.0/druid_models/content_index_batch.json" + ingestion_spec_path: "{{ content_snapshot_jar_path }}/etl-jobs-1.0/druid_models/content_index_batch.json" job_config: es_cloud_uploader_path: "{{ content_snapshot_path }}/config/ESCloudUploader.conf" diff --git a/ansible/roles/content-snapshot-indexer/tasks/main.yml b/ansible/roles/content-snapshot-indexer/tasks/main.yml index fefb9ce0f2..bbee7786bc 100644 --- a/ansible/roles/content-snapshot-indexer/tasks/main.yml +++ b/ansible/roles/content-snapshot-indexer/tasks/main.yml @@ -47,13 +47,13 @@ tags: - deploy -- name: Unarchive adhoc job script +- name: Unarchive etl job script become: yes unarchive: src={{ content_snapshot_path }}/{{ content_snapshot_distribuction_name }} dest={{ content_snapshot_jar_path }} copy=no group={{ analytics_group }} owner={{ analytics_user }} tags: - deploy -- name: Copy adhoc-jobs jar file +- name: Copy etl-jobs jar file become: yes copy: src: "{{ content_snapshot_jar_name }}" @@ -69,7 +69,7 @@ path: '{{ content_snapshot_path }}/config' state: absent tags: - - deploy + - execute-script - name: Create config directory file: @@ -77,7 +77,7 @@ state: directory become: yes tags: - - deploy + - execute-script - name: Ensure copying all conf files from the templates/conf folder become: yes @@ -88,7 +88,7 @@ group: "{{ analytics_group }}" with_items: "{{ config_files }}" tags: - - deploy + - execute-script - name: Ensure removing scripts directory become: yes @@ -97,7 +97,7 @@ state: absent become_user: "{{ analytics_user }}" tags: - - deploy + - execute-script - name: Create scripts directory become: yes @@ -106,7 +106,7 @@ state: directory become: yes tags: - - deploy + - execute-script - name: Ensure copying all script files from the templates/script dir become: yes @@ -118,12 +118,12 @@ mode: 0777 with_items: "{{ script_files }}" tags: - - deploy + - execute-script - name: Run Script become: yes become_user: root - shell: "sh {{ content_snapshot_path }}/scripts/run-script.sh {{ script_to_run }} >> {{ content_snapshot_path }}/logs/{{ansible_date_time.date}}-shell-execution.log" + shell: "sh {{ content_snapshot_path }}/scripts/run-script.sh {{ script_to_run }} {{ identifier }} {{ date }} {{ populate_anonymous_user }} {{ refresh_data }} >> {{ content_snapshot_path }}/logs/{{ansible_date_time.date}}-shell-execution.log" poll: 0 async: 45 tags: diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2 index 7826108361..337759f5d1 100644 --- a/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2 +++ b/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2 @@ -1,6 +1,6 @@ # redis redis.host="{{ redis.host }}" -redis.port="{{ redis.port }}" +redis.port="{{ dialcode_port }}" redis.connection.max={{ redis.connection.max }} redis.connection.idle.max={{ redis.connection.idle_max }} redis.connection.idle.min={{ redis.connection.idle_min }} @@ -10,7 +10,7 @@ redis.max.pipeline.size="{{ redis.max_pipeline_size }}" redis.dialcode.database.index=6 -cloudStorage.accountName="{{ azure_storage_account }}" -cloudStorage.accountKey="{{ azure_storage_secret }}" -cloudStorage.container="{{ bucket }}" -cloudStorage.dialCodeDataFile="dialcode-data/dial_code.csv" \ No newline at end of file +cloudStorage.accountName="{{sunbird_private_storage_account_name}}" +cloudStorage.accountKey="{{sunbird_private_storage_account_key}}" +cloudStorage.container="{{ bucket | default('telemetry-data-store') }}" +cloudStorage.dialCodeDataFile="dialcode-data/dial_code.csv" diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/ESContentIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/ESContentIndexer.j2 index 52b7ea2b3f..724051d23b 100755 --- a/ansible/roles/content-snapshot-indexer/templates/conf/ESContentIndexer.j2 +++ b/ansible/roles/content-snapshot-indexer/templates/conf/ESContentIndexer.j2 @@ -1,6 +1,6 @@ # redis redis.host="{{ redis.host }}" -redis.port="{{ redis.port }}" +redis.port="{{ content_port }}" redis.connection.max={{ redis.connection.max }} location.db.redis.key.expiry.seconds={{ redis.key_expiry_seconds }} redis.connection.idle.max={{ redis.connection.idle_max }} diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/cassandraRedis.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/cassandraRedis.j2 index 12db51e672..47681b82c5 100755 --- a/ansible/roles/content-snapshot-indexer/templates/conf/cassandraRedis.j2 +++ b/ansible/roles/content-snapshot-indexer/templates/conf/cassandraRedis.j2 @@ -1,6 +1,6 @@ # redis redis.host="{{ redis.host }}" -redis.port="{{redis.port }}" +redis.port="{{ user_port }}" redis.connection.max={{ redis.connection.max }} location.db.redis.key.expiry.seconds={{ redis.key_expiry_seconds }} redis.connection.idle.max={{ redis.connection.idle_max }} @@ -15,4 +15,10 @@ spark.cassandra.connection.host="{{ cassandra.host }}" cassandra.user.keyspace="{{ cassandra.keyspace }}" cassandra.user.table="{{ cassandra.user.table }}" redis.user.database.index="{{ redis.user.index }}" +redis.user.input.index="{{ redis.user.input_index }}" +redis.user.backup.dir="/mount/data/analytics/content-snapshot/anonymous-user-backup" +redis.scan.count="{{ redis.scan_count }}" redis.user.index.source.key="{{ redis.user.source_key }}" # this will be used as key for redis +cassandra.read.timeoutMS="{{ cassandra.read_timeout }}" +cassandra.query.retry.count="{{ cassandra.retry_count }}" +cassandra.input.consistency.level="{{ cassandra.input_consistency_level }}" \ No newline at end of file diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/DeviceProfileScripts.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/DeviceProfileScripts.j2 index 1da516c8bf..b05762a442 100644 --- a/ansible/roles/content-snapshot-indexer/templates/scripts/DeviceProfileScripts.j2 +++ b/ansible/roles/content-snapshot-indexer/templates/scripts/DeviceProfileScripts.j2 @@ -7,7 +7,7 @@ today=$(date "+%Y-%m-%d") nohup {{ spark_home }}/bin/spark-submit \ --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \ ---class org.ekstep.analytics.jobs.DeviceProfileUpdateCassandra \ +--class org.sunbird.analytics.jobs.DeviceProfileUpdateCassandra \ ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/DruidContentIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/DruidContentIndexer.j2 index 91ca9ba60a..20a6d5ae02 100644 --- a/ansible/roles/content-snapshot-indexer/templates/scripts/DruidContentIndexer.j2 +++ b/ansible/roles/content-snapshot-indexer/templates/scripts/DruidContentIndexer.j2 @@ -10,7 +10,7 @@ home=`echo $HOME` ingestionSpecFilePath="{{ druid.ingestion_spec_path }}" jobJarPath="{{ content_snapshot_jar_path }}/{{ content_snapshot_jar_name }}" jobConfPath="{{ job_config.es_cloud_uploader_path }}" -libs_path="{{ content_snapshot_path }}/jars/adhoc-jobs-1.0" +libs_path="{{ content_snapshot_path }}/jars/etl-jobs-1.0" export SPARK_HOME="{{ spark_home }}" # get list of segments from content model snapshot datasource @@ -28,7 +28,7 @@ echo "STARTED EXECUTING USER DRUID CONTENT INDEXER..." #--class org.ekstep.analytics.jobs.ESCloudUploader \ #${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 -nohup $SPARK_HOME/bin/spark-submit --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',') --class org.ekstep.analytics.jobs.ESCloudUploader ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 +nohup $SPARK_HOME/bin/spark-submit --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',') --class org.sunbird.analytics.jobs.ESCloudUploader ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 printf "\n>>> submit ingestion task to Druid!\n" diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisContentIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisContentIndexer.j2 index 7f86ec9c58..95db9071cd 100644 --- a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisContentIndexer.j2 +++ b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisContentIndexer.j2 @@ -6,10 +6,12 @@ home=`echo $HOME` jobJarPath="{{ content_snapshot_jar_path }}/{{ content_snapshot_jar_name }}" jobConfPath="{{ job_config.es_content_indexer_path }}" today=$(date "+%Y-%m-%d") +libs_path="{{ content_snapshot_path }}/jars/etl-jobs-1.0" echo "STARTED EXECUTING CONTENT CACHE INDEXER..." nohup {{ spark_home }}/bin/spark-submit \ --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \ ---class org.ekstep.analytics.jobs.ESRedisIndexer \ +--class org.sunbird.analytics.jobs.ESRedisIndexer \ +--jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',') \ ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisDialcodeIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisDialcodeIndexer.j2 index 8fe8850554..98f4144af1 100755 --- a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisDialcodeIndexer.j2 +++ b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisDialcodeIndexer.j2 @@ -4,7 +4,7 @@ home=`echo $HOME` jobJarPath="{{ content_snapshot_jar_path }}/{{ content_snapshot_jar_name }}" jobConfPath="{{ job_config.es_dialCode_indexer_path }}" today=$(date "+%Y-%m-%d") -libs_path="{{ content_snapshot_path }}/jars/adhoc-jobs-1.0" +libs_path="{{ content_snapshot_path }}/jars/etl-jobs-1.0" export SPARK_HOME="{{ spark_home }}" echo "STARTED EXECUTING DIALCODE CACHE INDEXER..." @@ -14,4 +14,4 @@ echo "STARTED EXECUTING DIALCODE CACHE INDEXER..." #--class org.ekstep.analytics.jobs.CSVToRedisIndexer \ #${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 -nohup $SPARK_HOME/bin/spark-submit --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',') --class org.ekstep.analytics.jobs.CSVToRedisIndexer ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 +nohup $SPARK_HOME/bin/spark-submit --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',') --class org.sunbird.analytics.jobs.CSVToRedisIndexer ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisUserDataIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisUserDataIndexer.j2 index bc7b5d1d81..5810dfeca1 100644 --- a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisUserDataIndexer.j2 +++ b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisUserDataIndexer.j2 @@ -1,13 +1,20 @@ #!/usr/bin/env bash +#$1 identifier: userid- Especially for UserCache indexer script to cache specific user id details +#$2 date: YYYY-MM-DD- Especially for UserCache indexer script to cache all the user data from this specific updated date. +#$3 populate_anonymous_user: true/false - Especially for UserCacheIndexer to read anonymous user's data +#$4 refresh_user_data: true/false - Especially for UserCacheIndexer to refresh user's data of redis + home=`echo $HOME` jobJarPath="{{ content_snapshot_jar_path }}/{{ content_snapshot_jar_name }}" jobConfPath="{{ job_config.cassandra_redis_path }}" today=$(date "+%Y-%m-%d") +libs_path="{{ content_snapshot_path }}/jars/etl-jobs-1.0" -echo "STARTED EXECUTING USER CACHE INDEXER..." +echo "STARTED EXECUTING USER CACHE INDEXER... $1 $2 $3 $4" nohup {{ spark_home }}/bin/spark-submit \ --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \ ---class org.ekstep.analytics.jobs.CassandraRedisIndexer \ -${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 +--jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',') \ +--class org.sunbird.analytics.jobs.UserCacheIndexer \ +${jobJarPath} $1 $2 $3 $4 >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1 \ No newline at end of file diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/run-script.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/run-script.j2 index 7d274d0ef7..c4b2cc46b1 100644 --- a/ansible/roles/content-snapshot-indexer/templates/scripts/run-script.j2 +++ b/ansible/roles/content-snapshot-indexer/templates/scripts/run-script.j2 @@ -1,10 +1,16 @@ #!/usr/bin/env bash -echo "Executing the script $1" +#$1 script_to_run: It can be USER_CACHE_INDEXER, DRUID_CONTENT_INDEXER etc,. +#$2 identifier: userid- Especially for UserCache indexer script to cache specific user id details +#$3 date: YYYY-MM-DD- Especially for UserCache indexer script to cache all the user data from this specific updated date. +#$4 populate_anonymous_user: true/false - Especially for UserCacheIndexer to read anonymous user's data +#$5 refresh_user_data: true/false - Especially for UserCacheIndexer to refresh user's data of redis + +echo "Executing the script $1 $2 $3 $4 $5" case "$1" in "USER_CACHE_INDEXER") echo "Invoked RedisUserDataIndexer" - nohup {{ content_snapshot_path }}/scripts/RedisUserDataIndexer.sh & + nohup {{ content_snapshot_path }}/scripts/RedisUserDataIndexer.sh $2 $3 $4 $5 & ;; "DRUID_CONTENT_INDEXER") echo "Invoked DruidContentIndexer" diff --git a/ansible/roles/data-products-deploy/defaults/main.yml b/ansible/roles/data-products-deploy/defaults/main.yml old mode 100644 new mode 100755 index 9ac351d19d..048d27f27d --- a/ansible/roles/data-products-deploy/defaults/main.yml +++ b/ansible/roles/data-products-deploy/defaults/main.yml @@ -1,45 +1,186 @@ analytics_user: analytics analytics_group: analytics spark_output_temp_dir: /mount/data/analytics/tmp/ -telemetry_converter_ver: 0.0.8 -bucket: "dev-data-store" +bucket: "telemetry-data-store" +secor_bucket: "telemetry-data-store" +dp_object_store_type: "cephs3" +dp_raw_telemetry_backup_location: "secor-{{ env }}/unique/raw/" +dp_storage_key_config: "aws_storage_key" +dp_storage_secret_config: "aws_storage_secret" +dp_reports_storage_key_config: "aws_storage_key" +dp_reports_storage_secret_config: "aws_storage_secret" + +s3_storage_endpoint: "{{ s3_storage_endpoint | default ('') }}" +s3_storage_key: "{{ s3_storage_key| replace('\n', '\\n')}}" +s3_storage_secret: "{{ s3_storage_secret }}" + kafka_broker_host: "{{groups['processing-cluster-kafka'][0]}}:9092" +ingestion_kafka_broker_host: "{{groups['ingestion-cluster-kafka'][0]}}:9092" brokerlist: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" zookeeper: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" dp_username: dp-monitor analytics_job_queue_topic: "{{ env }}.analytics.job_queue" -datasetRawBucket: ekstep-data-sets-{{ env }} -dataExhaustBucket: ekstep-public-{{ env }} -dataExhaustPrefix: data-exhaust/ -consumptionRawPrefix: datasets/D001/4208ab995984d222b59299e5103d350a842d8d41/ topic: "{{ env }}.telemetry.derived" -learning_topic: "{{ env }}.learning.graph.events" +topic2: "{{ env }}.telemetry.derived2" analytics_metrics_topic: "{{ env }}.analytics_metrics" sink_topic: "{{ env }}.telemetry.sink" +assess_topic: "{{ env }}.telemetry.assess" metrics_topic: "{{ env }}.telemetry.metrics" + +dashboards_courses_topic: "{{ env }}.dashboards.course" +dashboards_role_count_topic: "{{ env }}.dashboards.role.count" +dashboards_org_role_count_topic: "{{ env }}.dashboards.org.role.count" +dashboards_user_course_program_progress_topic: "{{ env }}.dashboards.user.course.program.progress" +dashboards_frac_competency_topic: "{{ env }}.dashboards.competency.frac" +dashboards_course_competency_topic: "{{ env }}.dashboards.competency.course" +dashboards_expected_competency_topic: "{{ env }}.dashboards.competency.expected" +dashboards_declared_competency_topic: "{{ env }}.dashboards.competency.declared" +dashboards_competency_gap_topic: "{{ env }}.dashboards.competency.gap" +dashboards_user_org_topic: "{{ env }}.dashboards.user.org" +dashboards_org_topic: "{{ env }}.dashboards.org" +dashboards_user_assessment_topic: "{{ env }}.dashboards.user.assessment" +dashboards_assessment_topic: "{{ env }}.dashboards.assessment" +dashboards_acbp_enrolment_topic: "{{ env }}.dashboards.acbp.enrolment" +dashboards_cassandra_user_table: "user" +dashboards_cassandra_user_roles_table: "user_roles" +dashboards_cassandra_org_table: "organisation" +dashboards_cassandra_user_enrolments_table: "user_enrolments" +dashboards_cassandra_content_hierarchy_table: "content_hierarchy" +dashboards_cassandra_rating_summary_table: "ratings_summary" +dashboards_cassandra_ratings_table: "ratings" +dashboards_cassandra_user_assessment_table: "user_assessment_data" +dashboards_cassandra_org_hierarchy_table: "org_hierarchy" +dashboards_cassandra_course_batch_table: "course_batch" +dashboards_cassandra_learner_stats_table: "learner_stats" +dashboards_cassandra_acbp_table: "cb_plan" +dashboards_cassandra_karma_points_table: "user_karma_points" +dashboards_cassandra_karma_points_lookup_table: "user_karma_points_credit_lookup" +dashboards_cassandra_mdo_karma_points_table: "mdo_karma_points" +dashboards_cassandra_learner_leaderboard_lookup_table: "learner_leaderboard_lookup" +dashboards_cassandra_learner_leaderboard_table: "learner_leaderboard" +dashboards_cassandra_user_feed_table: "notification_feed" +dashboards_cassandra_user_feed_keyspace: "sunbird_notifications" +dashboards_cassandra_karma_points_summary_table: "user_karma_points_summary" +dashboards_cassandra_mdo_learner_leaderboard_table: "mdo_top_learners" +dashboards_cassandra_old_assessment_table: "user_assessment_master" +dashboards_cassandra_nlw_mdo_table: "nlw_mdo_leaderboard" +dashboards_cassandra_nlw_user_table: "nlw_user_leaderboard" +dashboards_cassandra_public_userassessmenttable: "public_user_assessment_data" +dashboards_cassandra_user_entity_enrolment_table: "user_entity_enrolments" +dashboards_cassandra_group_designation_table: "kb_group_designation_content_data" +dashboards_cassandra_slw_mdo_leaderboard_table: "slw_mdo_leaderboard" +dashboards_cassandra_slw_mdo_top_learners_table: "slw_mdo_top_learners" +dashboards_cassandra_framework_hierarchy_table: "framework_hierarchy" + +app_postgres_host: "{{groups['app-postgres'][0]}}:5432" +dw_postgres_host: "{{groups['dw-postgres'][0]}}:5432" +dw_postgres_credential: "{{dp_vault_pgdb_password}}" + + +mongo_node_bb_host: "{{groups['mongodb-nodebb'][0]}}" +ml_mongo_host: "{{groups['ml-mongodb-host'][0]}}" +dashboards_mongo_discussions_db: "nodebb" +dashboards_mongo_discussions_db_collection: "objects" +dashboards_mongo_survey_observation_db: "ml-survey" +dashboards_mongo_survey_collection: "solutions" +dashboards_mongo_report_config_collection: "dataProductConfigurations" +dashboards_platform_rating_survey_id: "{{ platform_rating_survey_id }}" + +dashboards_frac_backend_host: "frac-dictionary-backend.igot-{{ env }}.in" +dashboards_broker_compression: "none" +dashboards_redis_host: "{{ redis_host }}" +dashboards_redis_port: "6379" +dashboards_redis_db: "12" +dashboards_redis_registered_officer_count_key: "mdo_registered_officer_count" +dashboards_redis_total_officer_count_key: "mdo_total_officer_count" +dashboards_redis_org_name_key: "mdo_name_by_org" +dashboards_redis_total_registered_officer_count_key: "mdo_total_registered_officer_count" +dashboards_redis_total_org_count_key: "mdo_total_org_count" + +user_report_path: "standalone-reports/user-report" +user_enrolment_report_path: "standalone-reports/user-enrollment-report" +course_report_path: "standalone-reports/course-report" +cba_report_path: "standalone-reports/cba-report" +standalone_assessment_report_path: "standalone-reports/user-assessment-report-cbp" +tagged_users_path: "tagged-users/" +blended_report_path: "standalone-reports/blended-program-report" +comms_console_report_path: "standalone-reports/comms-console" +org_hierarchy_report_path: "standalone-reports/org-hierarchy-report" +acbp_report_path: "standalone-reports/cbp-report" +acbp_mdo_enrolment_report_path: "standalone-reports/cbp-report-mdo-enrolment" +acbp_mdo_summary_report_path: "standalone-reports/cbp-report-mdo-summary" +kcm_report_path: "standalone-reports/kcm-report" +ml_report_path: "standalone-reports/ml-report" + +report_solution_ids: "" +report_grace_period: "2" +report_base_url_for_evidences: "{{ report_base_url_for_evidences }}" +report_include_expired_solution_ids: true + +anonymous_assessment_loggedIn_user_content_ids: "{{ anonymous_assessment_loggedIn_user_content_ids }}" +anonymous_assessment_non_loggedIn_user_assessment_ids: "{{ anonymous_assessment_non_loggedIn_user_assessment_ids }}" + +comms_console_prarambh_email_suffix: ".kb@karmayogi.in" +comms_console_num_days_to_consider: "15" +comms_console_num_top_learners_to_consider: "100000" +comms_console_prarambh_tags: "rojgaar,rozgaar,rozgar" +comms_console_prarambh_cbp_ids: "{{ comms_console_prarambh_cbp_ids_for_envs }}" +comms_console_prarambh_n_count: "4" + +report_mdo_ids: "" job_manager_tmp_dir: "transient-data" channel: dev-test druid_broker_host: "{{groups['raw-broker'][0]}}" +druid_router_host: "{{groups['raw-router'][0]}}" druid_rollup_broker_host: "{{groups['raw-broker'][0]}}" +ml_druid_broker_host: "{{groups['ml-raw-broker'][0]}}" +ml_druid_router_host: "{{groups['ml-raw-router'][0]}}" +ml_druid_rollup_broker_host: "{{groups['ml-raw-broker'][0]}}" hierarchySearchServiceUrl: "{{ proto }}://{{ domain_name }}/action/content" hierarchySearchServicEndpoint: /v3/hierarchy/ -analytics_job_list: '"wfs", "ds", "dpu", "content-rating-updater", "audit-metrics-report", "monitor-job-summ"' -analytics_jobs_count: 8 +user_table_keyspace: "sunbird" +course_keyspace: "sunbird_courses" +hierarchy_store_keyspace: "{{ env }}_hierarchy_store" +job_request_table: "{{ env }}_job_request" +dataset_metadata_table: "{{ env }}_dataset_metadata" +report_user_table_keyspace: "sunbird_courses" +report_user_enrolment_table: "report_user_enrolments" +zip_reports_prefix_directory_path: "standalone-reports" +zip_reports_destination_path: "standalone-reports/merged" +zip_reports_password: "123456" +directories_to_select: "blended-program-report-mdo,cbp-report-mdo-summary,course-report,cba-report,cbp-report-mdo-enrolment,user-report,user-enrollment-report" + +bq_script_path: "/mount/data/analytics/bq-scripts.sh" + +es_form_data_ids: "{{ es_form_data_ids }}" +es_form_data_index: "fs-forms-data" +single_node_es_port: 9200 + +analytics_job_list: '"wfs", "dashboard-sync", "assessment-metrics", "monitor-job-summ", "user-report-job", "survey-nps"' +analytics_jobs_count: 6 +operational_report_sync_enable: false +operational_report_zip_sync_enable: true +operational_sl_report_sync_enable: true +survey_question_report_batch_size: 2000 +survey_status_report_batch_size: 40000 +observation_question_report_batch_size: 2000 +observation_status_report_batch_size: 15000 + +national_learning_week_start: "2024-10-19 00:00:00" +national_learning_week_end: "2024-10-25 23:59:59" +state_learning_week_start: "2025-03-18 00:00:00" +state_learning_week_end: "2025-03-23 23:59:59" cassandra_keyspace_prefix: '{{ env }}_' -dp_dispatch_bucket_name: ekstep-dev-data-store -data_exhaust_blob_name: sunbird1p7 -data_exhaust_s3_url: https://s3-ap-south-1.amazonaws.com -spark_version: 2.4.4 +report_cassandra_cluster_host: "{{ report_cassandra_host | default(core_cassandra_host) }}" +cassandra_hierarchy_store_keyspace: "{{ env_name}}_hierarchy_store" +spark_version: 2.4.4 +model_version: "2.0" heap_memory: "-Xmx5120m" -business_metrics: - s3_path: "s3://ekstep-{{ env }}-data-store/business_metrics/" - env: "{{ env }}" - spark: home: "{{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7" public_dns: 54.255.154.146 @@ -48,65 +189,175 @@ spark: host: 172.31.11.117 worker: instances: 1 - cores: 2 - memory: 4g + cores: 6 + memory: 18g driver: - memory: 3g + memory: 12g + cores: 2 + memoryOverhead: 2g executor: - memory: 4g - driver_memory: 7g + instances: 1 + cores: 6 + memory: 14g + memoryOverhead: 2g + driver_memory: 55g + executor_extraJavaOptions: -Dconfig.file=/mount/data/analytics/models-{{ model_version }}/{{ env }}.conf -Dlog4j.configurationFile=file:////mount/data/analytics/models-{{ model_version }}/log4j2.xml + driver_extraJavaOptions: -Dconfig.file=/mount/data/analytics/models-{{ model_version }}/{{ env }}.conf -Dlog4j.configurationFile=file:////mount/data/analytics/models-{{ model_version }}/log4j2.xml memory_fraction: 0.3 - storage_fraction: 0.5 - executor_memory: 2g + storage_fraction: 0.5 + executor_memory: 14g heap_conf_str: '"-XX:+UseG1GC -XX:MaxGCPauseMillis=100 -Xms250m {{ heap_memory }} -XX:+UseStringDeduplication"' submit_jobs: submit-all-jobs: - hour: 00 + hour: 02 minute: 35 start_jobmanager: job-manager: - hour: 00 + hour: 02 minute: 30 have_weekly_jobs: false -video_stream_job_schedule: 10 +course_batch_status_updater_job_schedule: 60 -run_wfs_job: - wfs: +run_data_exhaust_job: + data-exhaust: + hour: 00 + minute: 01 +run_dashboard_sync_job: + dashboard-sync: hour: 00 + minute: 25 +run_assessment_metrics_job: + assessment-metrics: + hour: 01 minute: 30 -run_monitor_job: - monitor-job-summ: +run_course_report_job: + course-report: + hour: 01 + minute: 40 +run_acbp_report_job: + acbp-report: + hour: 01 + minute: 55 +run_user_enrolment_report_job: + user-enrolment-report: hour: 03 minute: 00 - -run_course_metrics_job: - course-dashboard-metrics: - hour: 19 +run_user_report_job: + user-report: + hour: 04 + minute: 15 +run_cba_report_job: + course-based-assessment-report: + hour: 04 minute: 45 - -run_assessment_metrics_job: - assessment-dashboard-metrics: - hour: 20 +run_blended_program_report_job: + blended-program-report: + hour: 05 + minute: 25 +run_kcm_report_job: + kcm-report: + hour: 06 + minute: 25 +run_wfs_job: + wfs: + hour: 06 + minute: 30 +run_in_app_review_feed_job: + in-app-review-feed: + hour: 06 + minute: 45 +run_survey_nps_job: + survey-nps: + hour: 07 + minute: 00 +run_ministry_metrics_job: + ministry-metrics: + hour : 07 + minute: 05 +run_summary_redis_sync_job: + summary-redis-sync: + hour: 07 + minute: 20 +run_weekly_claps_job: + weekly-claps: + hour: 07 + minute: 30 +run_data_warehouse_job: + data-warehouse-sync: + hour: 07 + minute: 40 +run_bq_data_job: + bq-data-push: + hour: 09 + minute: 10 +run_zip_upload_job: + zip-upload: + hour: 09 + minute: 20 +run_comms_console_job: + comms-console: + hour: 10 + minute: 05 +run_survey_question_report_job: + survey-question-report: + hour: 10 minute: 15 - +run_survey_status_report_job: + survey-status-report: + hour: 10 + minute: 25 +run_national_learning_week_job: + national-learning-week: + hour: 10 + minute: 35 +run_ministry_learner_leaderboard_job: + ministry-learner-leaderboard: + day: 01 + hour: 10 + minute: 35 +run_monthly_requests_job: + monthly-requests: + hour: 11 + minute: 00 +run_learner_leaderboard_job: + learner-leaderboard: + day: 01 + hour: 11 + minute: 30 +run_odcs_recomendation_job: + odcs-recomendation: + hour: 12 + minute: 30 +run_user_activity_job: + user-activity: + hour: 12 + minute: 00 +run_monitor_job: + monitor-job-summ: + hour: 03 + minute: 00 run_admin_user_reports_job: admin-user-reports-3AMIST: hour: 21 minute: 30 admin-user-reports-2PMIST: hour: 8 - minute: 30 - + minute: 30 run_admin_geo_reports_job: admin-geo-reports-4AMIST: hour: 22 minute: 30 admin-geo-reports-3PMIST: hour: 9 - minute: 30 + minute: 30 +run_es_form_data_job: + es-form-data: + hour: 10 + minute: 50 + weekday: 1 run_assessment_aggregator_report_job: assessment-aggregator-report: @@ -123,24 +374,55 @@ index_content_model_druid: hour: 1 minute: 00 -run_district_weekly_job: - district-weekly: - hour: 03 - minute: 30 - weekday: 1 - run_etb_metrics_weekly_job: etb-metrics-weekly: hour: 23 minute: 30 weekday: 1 -run_district_monthly_job: - district-monthly: - hour: 04 +# These are the dummy times till sept30 for exhaust reports +#To-Do: Update time after 3.2.7 deployment + +run_progress_exhaust: + progress-exhaust: + hour: 08 + minute: 00 + +run_response_exhaust: + response-exhaust: + hour: 09 + minute: 00 + +run_userinfo_exhaust: + userinfo-exhaust: + hour: 10 + minute: 00 + +run_collection_summary: + collection-summary: + hour: 09 + minute: 30 + +run_sourcing_summary: + sourcing-summary: + hour: 10 minute: 30 - day: 1 - month: 1-12 + +run_cassandra_migration: + cassandra-migration: + hour: 19 + minute: 15 + +run_uci_private_exhaust_job: + uci-private-exhaust: + hour: 03 + minute: 00 + +run_uci_response_exhaust_job: + uci-response-exhaust: + hour: 02 + minute: 00 + service: search: @@ -151,19 +433,96 @@ es_search_index: "compositesearch" analytics: home: /mount/data/analytics soft_path: /mount/data/analytics - paths: ['/home/analytics/sbin', '/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/data-products', '/mount/data/analytics/logs/api-service', '/mount/data/analytics/api', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts/monitor-data', '/mount/data/analytics/models' ] - scripts: ['model-config', 'replay-job', 'replay-updater', 'replay-utils', 'run-job', 'monitor-dp', 'generate-metrics', 'submit-job', 'start-jobmanager'] + paths: ['/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/data-products', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts', '/mount/data/analytics/models' ] + scripts: ['model-config', 'replay-job', 'replay-updater', 'replay-utils', 'run-job', 'submit-job', 'start-jobmanager', 'submit-script'] + dockScripts: ['model-dock-config','run-dock-job'] +# artifact versions +analytics_core_artifact_ver: "2.0" +analytics_ed_dataporducts_artifact_ver: "1.0" +scruid_artifact_ver: "2.4.0" producer_env: "dev.sunbird" -analytics_job_manager_artifact: "job-manager-2.0.jar" -analytics_core_artifact: "analytics-framework-2.0.jar" -scruid_artifact: "scruid_2.11-2.3.2.jar" -analytics_batch_module_artifact: "batch-models-2.0.jar" -analytics_ed_dataporducts_artifact: "data-products-1.0-distribution.tar.gz" -model_version: "2.0" +analytics_job_manager_artifact: "job-manager-{{ analytics_core_artifact_ver }}.jar" +analytics_core_artifact: "analytics-framework-{{ analytics_core_artifact_ver }}.jar" +scruid_artifact: "scruid_2.11-{{ scruid_artifact_ver }}.jar" +analytics_batch_module_artifact: "batch-models-{{ analytics_core_artifact_ver }}.jar" +analytics_ed_dataporducts_artifact: "data-products-{{ analytics_ed_dataporducts_artifact_ver }}-distribution.tar.gz" -submit_jobs_auth_token: "" -report_list_jobs_url: "http://{{groups['analytics-api'][0]}}:9000/" +submit_jobs_auth_token: "{{ sunbird_api_auth_token }}" +report_list_jobs_url: "{{ druid_report_url }}" reports_container: "reports" + +# Cluster vars +spark_cluster_user_password: "" +spark_cluster_user_name: "" +admin_name: "{{ spark_cluster_user_name }}" +admin_password: "{{ spark_cluster_user_password }}" +spark_cluster_name: "{{env}}-spark-cluster" + +spark_cluster: + executor_core: 5 + executor_memory: 19G + num_executors: 5 + +analytics_cluster: + home: "/tmp" + +analytics_ed_dataporducts_jar_artifact: "data-products-{{ analytics_ed_dataporducts_artifact_ver }}.jar" + +spark_enable_dynamic_allocation: false +# Spark Cassandra config-vars +spark_cassandra_connection_timeout_millis: 30000 +spark_cassandra_query_timeout_millis: 180000 +spark_cassandra_query_max_rows_fetch_count: 1000 +spark_sql_shuffle_partitions: 200 + +druid_report_postgres_db_name: druid +druid_report_postgres_db_username: druid + + +#Override this variable in production and point to druid rollup ingestion cluster +# Example: "http://$rollup_cluster_ip:8090" +druid_rollup_cluster_ingestion_task_url: "http://{{groups['raw-overlord'][0]}}:8081" + +# On demand Exhaust throttling vars +exhaust_batches_limit_per_channel: 30 +exhaust_file_size_limit_bytes_per_channel: 1073741824 + +exhaust_parallel_batch_load_limit: 10 +exhaust_user_parallelism: 200 + +data_exhaust_batch_limit_per_request: 20 + +# Start Of UCI Related Variables +uci_postgres_host: "dev-pg11.postgres.database.azure.com" +uci_encryption_key_base64: "" +uci_bot_postgres_database: uci-botdb +uci_fusionauth_postgres_database: uci-fusionauth +uci_postgres_user: "{{postgres.db_username}}" +uci_postgres_password: "{{postgres.db_password}}" + +uci_postgres: + conversation_db_name: "{{ uci_bot_postgres_database }}" + conversation_db_host: "{{ uci_postgres_host }}" + conversation_db_port: "5432" + conversation_db_user: "{{ uci_postgres_user }}" + conversation_db_psss: "{{ uci_postgres_password }}" + conversation_table_name: "bot" + fushionauth_db_name: "{{ uci_fusionauth_postgres_database }}" + fushionauth_db_host: "{{ uci_postgres_host }}" + fushionauth_db_port: "5432" + fushionauth_db_user: "{{ uci_postgres_user }}" + fushionauth_db_psss: "{{ uci_postgres_password }}" + user_table_name: "users" + user_registration_table_name: "user_registrations" + user_identities_table_name: "identities" + +uci_encryption_secret_key: "{{uci_encryption_key_base64}}" +uci_pdata_id: "{{uci_env|default('dev')}}.uci.{{sunbird_instance|default('igot')}}" + +# End Of UCI Related Variables + +# Exhaust sanity check vars +cassandra_migrator_job_name: "Cassandra Migrator" diff --git a/ansible/roles/data-products-deploy/files/collection-summary-ingestion-spec.json b/ansible/roles/data-products-deploy/files/collection-summary-ingestion-spec.json new file mode 100644 index 0000000000..69e13196e2 --- /dev/null +++ b/ansible/roles/data-products-deploy/files/collection-summary-ingestion-spec.json @@ -0,0 +1,251 @@ +{ + "type": "index", + "spec": { + "dataSchema": { + "dataSource": "collection-summary-snapshot", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "content_org", + "expr": "contentorg" + }, + { + "type": "root", + "name": "user_org", + "expr": "orgname" + }, + { + "type": "root", + "name": "batch_start_date", + "expr": "startdate" + }, + { + "type": "root", + "name": "batch_end_date", + "expr": "enddate" + }, + { + "type": "root", + "name": "has_certificate", + "expr": "hascertified" + }, + { + "type": "root", + "name": "collection_id", + "expr": "courseid" + }, + { + "type": "root", + "name": "batch_id", + "expr": "batchid" + }, + { + "type": "root", + "name": "collection_name", + "expr": "collectionname" + }, + { + "type": "root", + "name": "batch_name", + "expr": "batchname" + }, + { + "type": "root", + "name": "total_enrolment", + "expr": "enrolleduserscount" + }, + { + "type": "root", + "name": "total_completion", + "expr": "completionuserscount" + }, + { + "type": "root", + "name": "total_certificates_issued", + "expr": "certificateissuedcount" + }, + { + "type": "root", + "name": "content_status", + "expr": "contentstatus" + }, + { + "type": "root", + "name": "user_state", + "expr": "state" + }, + { + "type": "root", + "name": "user_district", + "expr": "district" + }, + { + "type": "root", + "name": "content_channel", + "expr": "channel" + }, + { + "type": "root", + "name": "keywords", + "expr": "keywords" + }, + { + "type": "root", + "name": "timestamp", + "expr": "timestamp" + }, + { + "type": "root", + "name": "medium", + "expr": "medium" + }, + { + "type": "root", + "name": "subject", + "expr": "subject" + }, + { + "type": "root", + "name": "created_for", + "expr": "createdfor" + }, + { + "type": "root", + "name": "user_type", + "expr": "usertype" + }, + { + "type": "root", + "name": "user_subtype", + "expr": "usersubtype" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "content_org" + }, + { + "name": "user_org" + }, + { + "type": "string", + "name": "batch_id" + }, + { + "type": "string", + "name": "batch_start_date" + }, + { + "type": "string", + "name": "batch_end_date" + }, + { + "type": "string", + "name": "collection_id" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "batch_name" + }, + { + "type": "long", + "name": "total_enrolment" + }, + { + "type": "long", + "name": "total_completion" + }, + { + "type": "long", + "name": "total_certificates_issued" + }, + { + "type": "string", + "name": "content_status" + }, + { + "type": "string", + "name": "user_state" + }, + { + "type": "string", + "name": "user_district" + }, + { + "name": "keywords" + }, + { + "name": "has_certificate" + }, + { + "type": "string", + "name": "content_channel" + }, + { + "name": "medium" + }, + { + "name": "subject" + }, + { + "name": "created_for" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "user_subtype" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "queryGranularity": "none", + "rollup": true + } + }, + "ioConfig": { + "type": "index", + "firehose": { + "type": "static-azure-blobstore", + "blobs": [ + { + "container": "reports", + "path": "/collection-summary-reports-v2/collection-summary-report-latest.json" + } + ], + "fetchTimeout": 300000 + } + }, + "tuningConfig": { + "type": "index", + "targetPartitionSize": 5000000, + "maxRowsInMemory": 25000, + "forceExtendableShardSpecs": false, + "logParseExceptions": true + } + } +} \ No newline at end of file diff --git a/ansible/roles/data-products-deploy/files/sourcing-ingestion-spec.json b/ansible/roles/data-products-deploy/files/sourcing-ingestion-spec.json new file mode 100644 index 0000000000..69e773d457 --- /dev/null +++ b/ansible/roles/data-products-deploy/files/sourcing-ingestion-spec.json @@ -0,0 +1,146 @@ +{ + "type": "index", + "spec": { + "dataSchema": { + "dataSource": "sourcing-summary-snapshot", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "program_id", + "expr": "program_id" + }, + { + "type": "root", + "name": "status", + "expr": "status" + }, + { + "type": "root", + "name": "rootorg_id", + "expr": "rootorg_id" + }, + { + "type": "root", + "name": "user_id", + "expr": "user_id" + }, + { + "type": "root", + "name": "osid", + "expr": "osid" + }, + { + "type": "root", + "name": "user_type", + "expr": "user_type" + }, + { + "type": "root", + "name": "contributor_id", + "expr": "contributor_id" + }, + { + "type": "root", + "name": "total_contributed_content", + "expr": "total_contributed_content" + }, + { + "type": "root", + "name": "primary_category", + "expr": "primary_category" + }, + { + "type": "root", + "name": "created_by", + "expr": "created_by" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "program_id" + }, + { + "type": "string", + "name": "status" + }, + { + "type": "string", + "name": "rootorg_id" + }, + { + "type": "string", + "name": "user_id" + }, + { + "type": "string", + "name": "osid" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "contributor_id" + }, + { + "type": "string", + "name": "primary_category" + }, + { + "type": "string", + "name": "created_by" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [ + { + "name": "total_count", + "type": "count" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "queryGranularity": "none", + "rollup": true + } + }, + "ioConfig": { + "type": "index", + "firehose": { + "type": "static-azure-blobstore", + "blobs": [ + { + "container": "reports", + "path": "/sourcing/SourcingSummaryReport.json" + } + ], + "fetchTimeout": 300000 + } + }, + "tuningConfig": { + "type": "index", + "targetPartitionSize": 5000000, + "maxRowsInMemory": 25000, + "forceExtendableShardSpecs": false, + "logParseExceptions": true + } + } +} diff --git a/ansible/roles/data-products-deploy/tasks/main.yml b/ansible/roles/data-products-deploy/tasks/main.yml index 35f28ffe4d..fac3e98dd2 100644 --- a/ansible/roles/data-products-deploy/tasks/main.yml +++ b/ansible/roles/data-products-deploy/tasks/main.yml @@ -1,24 +1,59 @@ + ## Data products deployment ## +# - name: Ensure azure blob storage container exists +# command: az storage container create --name {{ bucket }} +# when: dp_object_store_type == "azure" +# tags: +# - always + - name: Copy Core Data Products copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: - dataproducts +# - name: Copy Core Data Products to azure blob +# command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} +# async: 3600 +# poll: 10 +# tags: +# - dataproducts-spark-cluster + - name: Unarchive Ed Data Products become: yes unarchive: src={{ playbook_dir}}/{{ analytics_ed_dataporducts_artifact }} dest={{ analytics.home }}/models-{{ model_version }} copy=yes group={{ analytics_group }} owner={{ analytics_user }} tags: - - ed-dataproducts + - ed-dataproducts + +# - name: Copy Ed Data Products to azure blob +# command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar +# async: 3600 +# poll: 10 +# tags: +# - ed-dataproducts-spark-cluster - name: Copy Framework Library copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: - framework +# - name: Copy Framework Library to azure blob +# command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} +# async: 3600 +# poll: 10 +# tags: +# - framework-spark-cluster + - name: Copy Scruid Library copy: src={{ scruid_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: - - framework + - framework + +# - name: Copy Scruid Library to azure blob +# command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} +# async: 3600 +# poll: 10 +# tags: +# - framework-spark-cluster - name: Copy Job Manager copy: src={{ analytics_job_manager_artifact }} dest={{ analytics.home }}/models-{{ model_version }} @@ -31,6 +66,52 @@ - dataproducts - ed-dataproducts - framework + when: dockdataproducts is undefined + +- name: Write GCP service account file + copy: + content: "{{ gcp_artifcats_upload_service_account }}" + dest: "{{ analytics.home }}/models-{{ model_version }}/gcp_service_account.json" + mode: "0640" + tags: + - dataproducts + - default-jobs + +- name: Copy configuration file + template: src=common.conf.j2 dest={{ analytics.home }}/models-{{ model_version }}/dock-{{ env }}.conf mode=755 owner={{ analytics_user }} group={{ analytics_group }} + tags: + - dataproducts + - ed-dataproducts + - framework + when: dockdataproducts is defined + +- name: Copy configuration file as application.conf for cluster + template: src=common.conf.j2 dest={{ analytics.home }}/models-{{ model_version }}/application.conf mode=755 owner={{ analytics_user }} group={{ analytics_group }} + tags: + - framework-spark-cluster + +- name: Update spark temp dir value for cluster + lineinfile: + path: '{{ analytics.home }}/models-{{ model_version }}/application.conf' + regexp: '^spark_output_temp_dir="/mount/data/analytics/tmp/"' + line: 'spark_output_temp_dir="/var/log/sparkapp/tmp/"' + tags: + - framework-spark-cluster + +- name: Update logger kafka config for cluster + lineinfile: + path: '{{ analytics.home }}/models-{{ model_version }}/application.conf' + regexp: '^log.appender.kafka.enable="false"' + line: 'log.appender.kafka.enable="true"' + tags: + - framework-spark-cluster + +# - name: Copy configuration file to azure blob +# command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf +# async: 3600 +# poll: 10 +# tags: +# - framework-spark-cluster - name: Copy log4j2 xml file template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }} @@ -40,6 +121,28 @@ template: src={{ item }}.j2 dest={{ analytics.home }}/scripts/{{ item }}.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }} with_items: "{{ analytics.scripts }}" tags: [ dataproducts, framework, ed-dataproducts ] + when: dockdataproducts is undefined + +- name: Copy bq-scripts file + template: src=bq-scripts.sh.j2 dest={{ analytics.home }}/bq-scripts.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }} + tags: + - dataproducts + +- name: Copy es-form-data file + template: src=es-form-data.j2 dest={{ analytics.home }}/scripts/es-form-data.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }} + tags: + - dataproducts + +- name: Copy python sanity check script file + template: src=exhaust_sanity_check.py.j2 dest={{ analytics.home }}/scripts/exhaust_sanity_check.py + tags: [ dataproducts, framework, ed-dataproducts ] + when: dockdataproducts is undefined + +- name: Copy Dock Scripts + template: src={{ item }}.j2 dest={{ analytics.home }}/scripts/{{ item }}.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }} + with_items: "{{ analytics.dockScripts }}" + tags: [ dataproducts, framework, ed-dataproducts ] + when: dockdataproducts is defined - name: Update model config template: src=model-config.j2 dest={{ analytics.home }}/scripts/model-config.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }} @@ -47,6 +150,15 @@ - dataproducts - update-config - ed-dataproducts + when: dockdataproducts is undefined + +- name: Update model dock config + template: src=model-dock-config.j2 dest={{ analytics.home }}/scripts/model-dock-config.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }} + tags: + - dataproducts + - update-config + - ed-dataproducts + when: dockdataproducts is defined - name: Copy submit-all-jobs ruby file template: src=submit-all-jobs.rb.j2 dest={{ analytics.home }}/scripts/submit-all-jobs.rb mode=755 owner={{ analytics_user }} group={{ analytics_group }} @@ -71,23 +183,221 @@ - spark1-jobs - clean-cronjobs +- name: Create daily cron jobs for data-exhaust + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh data-exhaust" + with_dict: "{{ run_data_exhaust_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for dashboard-sync + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh dashboard-sync" + with_dict: "{{ run_dashboard_sync_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for assessment-metrics + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh assessment-metrics" + with_dict: "{{ run_assessment_metrics_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for course-report + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh course-report" + with_dict: "{{ run_course_report_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for user-enrolment-report + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh user-enrolment-report" + with_dict: "{{ run_user_enrolment_report_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for user-report + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh user-report" + with_dict: "{{ run_user_report_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for ministry-metrics + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh ministry-metrics" + with_dict: "{{ run_ministry_metrics_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for odcs-recomendation + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh odcs-recomendation" + with_dict: "{{ run_odcs_recomendation_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for national-learning-week + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh national-learning-week" + with_dict: "{{ run_national_learning_week_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create weekly cron job for es-form-data + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} weekday={{ item.value.weekday }} job="{{ analytics.home }}/scripts/run-job.sh es-form-data" + with_dict: "{{ run_es_form_data_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for ministry-learner-leaderboard + cron: name="{{env}}-{{ item.key }}" day={{ item.value.day }} minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh ministry-learner-leaderboard" + with_dict: "{{ run_ministry_learner_leaderboard_job }}" + tags: + - spark1-jobs + - default-jobs + - name: Create daily cron jobs for wfs cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh wfs" with_dict: "{{ run_wfs_job }}" tags: - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for course-based-assessment-report + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh course-based-assessment-report" + with_dict: "{{ run_cba_report_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for blended-program-report + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh blended-program-report" + with_dict: "{{ run_blended_program_report_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for summary-redis-sync + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh summary-redis-sync" + with_dict: "{{ run_summary_redis_sync_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for data-warehouse-sync + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh data-warehouse-sync" + with_dict: "{{ run_data_warehouse_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for weekly-claps + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh weekly-claps" + with_dict: "{{ run_weekly_claps_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for acbp-report + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh acbp-report" + with_dict: "{{ run_acbp_report_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for comms-console + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh comms-console" + with_dict: "{{ run_comms_console_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for survey-nps + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh survey-nps" + with_dict: "{{ run_survey_nps_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for user-activity + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh user-activity" + with_dict: "{{ run_user_activity_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for monthly-requests + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh monthly-requests" + with_dict: "{{ run_monthly_requests_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for bq-data-push + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh bq-data-push" + with_dict: "{{ run_bq_data_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for kcm-report + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh kcm-report" + with_dict: "{{ run_kcm_report_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for zip-upload + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh zip-upload" + with_dict: "{{ run_zip_upload_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for in-app-review-feed + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh in-app-review-feed" + with_dict: "{{ run_in_app_review_feed_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for survey-question-report + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh survey-question-report" + with_dict: "{{ run_survey_question_report_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create daily cron jobs for survey-status-report + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh survey-status-report" + with_dict: "{{ run_survey_status_report_job }}" + tags: + - spark1-jobs + - default-jobs + +- name: Create monthly cron jobs for learner-leaderboard + cron: name="{{env}}-{{ item.key }}" day={{ item.value.day }} minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh learner-leaderboard" + with_dict: "{{ run_learner_leaderboard_job }}" + tags: + - spark1-jobs + - default-jobs - name: Create daily cron jobs for monitor job cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh monitor-job-summ" with_dict: "{{ run_monitor_job }}" tags: - spark1-jobs + # - default-jobs - name: Create daily cron jobs using submit-all-jobs cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job='/bin/bash -lc "ruby {{ analytics.home }}/scripts/submit-all-jobs.rb"' with_dict: "{{ submit_jobs }}" tags: - - default-jobs + # - default-jobs - spark-jobs - cronjobs @@ -95,83 +405,255 @@ cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/start-jobmanager.sh" with_dict: "{{ start_jobmanager }}" tags: - - default-jobs - - spark-jobs - - cronjobs - -- name: Create video-streaming cron job - cron: name="{{env}}-video-streaming" minute=*/{{ video_stream_job_schedule }} job="{{ analytics.home }}/scripts/run-job.sh video-streaming" - tags: - - default-jobs + # - default-jobs - spark-jobs - cronjobs -- name: Create course-dashboard-metrics cron job - cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh course-dashboard-metrics" - with_dict: "{{ run_course_metrics_job }}" +- name: Create course-batch-status-updater cron job + cron: name="{{env}}-course-batch-status-updater" minute=*/{{ course_batch_status_updater_job_schedule }} job="{{ analytics.home }}/scripts/run-job.sh course-batch-status-updater" tags: - cronjobs - - default-jobs + # - default-jobs - spark1-jobs -- name: Create assessment-dashboard-metrics cron job - cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh assessment-dashboard-metrics" - with_dict: "{{ run_assessment_metrics_job }}" - tags: - - cronjobs - - default-jobs - - spark-jobs - - name: Create admin-user-reports cron job cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh admin-user-reports" with_dict: "{{ run_admin_user_reports_job }}" tags: - - cronjobs - - default-jobs + - cronjobs + # - default-jobs - spark-jobs - - name: Create admin-geo-reports cron job cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh admin-geo-reports" with_dict: "{{ run_admin_geo_reports_job }}" tags: - cronjobs - - default-jobs - - spark-jobs + # - default-jobs + - spark-jobs - name: Create assessment-aggregator reports cron job cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="/bin/bash {{ analytics.home }}/adhoc-scripts/run_exporter.sh > /home/analytics/output.log" with_dict: "{{ run_assessment_aggregator_report_job }}" tags: - cronjobs - - default-jobs - - spark-jobs + # - default-jobs + - spark-jobs - name: Create etb metrics cron job cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} weekday={{ item.value.weekday }} job="{{ analytics.home }}/scripts/run-job.sh etb-metrics" with_dict: "{{ run_etb_metrics_weekly_job }}" tags: - cronjobs - - default-jobs + # - default-jobs + - spark-jobs + +- name: Create progress-exhaust cron job + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh progress-exhaust" + with_dict: "{{ run_progress_exhaust }}" + tags: + - cronjobs + # - default-jobs + - spark1-jobs + +- name: Create response-exhaust cron job + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh response-exhaust" + with_dict: "{{ run_response_exhaust }}" + tags: + - cronjobs + # - default-jobs + - spark-jobs + +- name: Create cassandra-migration cron job + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh cassandra-migration" + with_dict: "{{ run_cassandra_migration }}" + tags: + - cronjobs + # - default-jobs - spark-jobs -#- name: Create district-weekly cron job -# cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} weekday={{ item.value.weekday }} job="{{ analytics.home }}/scripts/run-job.sh district-weekly" -# with_dict: "{{ run_district_weekly_job }}" -# tags: -# - cronjobs -# - default-jobs -# - ed-dataproducts - -#- name: Create district-monthly cron job -# cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} month={{ item.value.month }} day={{ item.value.day }} job="{{ analytics.home }}/scripts/run-job.sh district-monthly" -# with_dict: "{{ run_district_monthly_job }}" -# tags: -# - cronjobs -# - default-jobs -# - ed-dataproducts + +- name: Create userinfo-exhaust cron job + cron: name="{{ env }}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh userinfo-exhaust" + with_dict: "{{ run_userinfo_exhaust }}" + tags: + - cronjobs + # - default-jobs + - spark-jobs + +- name: Create collection-summary cron job + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh collection-summary-report" + with_dict: "{{ run_collection_summary }}" + tags: + - cronjobs + # - default-jobs + - spark-jobs + +- name: Copy collection-summary ingestion spec + copy: src="collection-summary-ingestion-spec.json" dest={{ analytics.home }}/scripts/ mode=755 owner={{ analytics_user }} group={{ analytics_group }} + tags: + - ed-dataproducts + +- name: Create sourcing-summary cron job + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-dock-job.sh sourcing-summary-report" + with_dict: "{{ run_sourcing_summary }}" + tags: + - cronjobs + # - default-jobs + - spark-jobs + +- name: Create uci-private-exhaust cron job + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh uci-private-exhaust" + with_dict: "{{ run_uci_private_exhaust_job }}" + tags: + - cronjobs + # - default-jobs + - spark-jobs + +- name: Create uci-response-exhaust cron job + cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh uci-response-exhaust" + with_dict: "{{ run_uci_response_exhaust_job }}" + tags: + - cronjobs + # - default-jobs + - spark-jobs + +- name: Copy sourcing-summary ingestion spec + copy: src="sourcing-ingestion-spec.json" dest={{ analytics.home }}/scripts/ mode=755 owner={{ analytics_user }} group={{ analytics_group }} + tags: + - ed-dataproducts - name: Update start jobmanager template: src=start-jobmanager.j2 dest={{ analytics.home }}/scripts/start-jobmanager.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }} tags: - update-jobmanager-config - - dataproducts \ No newline at end of file + - dataproducts + +# Cluster job sumbit tasks +- name: Copy cluster-config.json file + template: src=cluster-config.json.j2 dest={{ analytics_cluster.home }}/cluster-config.json + delegate_to: localhost + tags: + - replay-job + - run-job + - config-update + +- name: Copy submit-script.sh file + template: src=submit-script.j2 dest={{ analytics_cluster.home }}/submit-script.sh mode=755 + delegate_to: localhost + tags: + - replay-job + - run-job + - config-update + +- name: Copy model-config.sh file + template: src=model-config.j2 dest={{ analytics_cluster.home }}/model-config.sh + delegate_to: localhost + tags: + - replay-job + - run-job + - config-update + +- name: Replay Job + shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ job_id }} --mode {{ mode }} --partitions {{ partitions }} --parallelisation {{ parallelisation }} --startDate {{ start_date }} --endDate {{ end_date }} --sparkMaster {{ sparkMaster }} --selectedPartitions {{ selected_partitions }} &" + async: "{{ (pause_min * 60) }}" + poll: 0 + tags: + - replay-job + +- name: Run Job + shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ job_id }} --mode {{ mode }} --partitions {{ partitions }} --parallelisation {{ parallelisation }} --sparkMaster {{ sparkMaster }} --selectedPartitions {{ selected_partitions }} --batch_id {{ batch_id }} &" + async: "{{ (pause_min * 60) }}" + poll: 0 + tags: + - run-job + +- name: Submit jobs + shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ item }} --mode default --sparkMaster yarn &" + with_items: "{{ jobs.split(',')|list }}" + tags: + - job-submit + +# Cluster exhaust parallel jobs sumbit tasks + +- name: Install required python packages + pip: + name: + - psycopg2-binary + - pandas + - IPython + tags: + - parallel-jobs-submit + +- name: Copy python script file + template: src=update-job-requests.py.j2 dest={{ analytics_cluster.home }}/update-job-requests.py + delegate_to: localhost + tags: + - parallel-jobs-submit + +- name: Execute python script to populate batch numbers + shell: | + if echo "{{jobs}}" | grep 'druid' + then + python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} druid {{env}}_report_config + elif echo "{{jobs}}" | grep 'exhaust' + then + python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} exhaust {{env}}_job_request + fi + tags: + - parallel-jobs-submit + register: jobsCountStr + + +- debug: + var: jobsCountStr + tags: + - parallel-jobs-submit + +- name: Get stdout with parallelisation value from python script to tmp file + shell: echo "{{ jobsCountStr.stdout }}" > /tmp/test.txt + tags: + - parallel-jobs-submit + +- name: Extract parallelisation value from tmp file + shell: "cat /tmp/test.txt | tr '\n' ' ' | awk -F': ' '{print $NF}'" + register: jobsCountOut + tags: + - parallel-jobs-submit + +- debug: + var: jobsCountOut + tags: + - parallel-jobs-submit + +# set jobs count variable from python script output +- set_fact: + jobs_count: "{{ jobsCountOut.stdout }}" + tags: + - parallel-jobs-submit + +- name: Submit parallel exhaust jobs + shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ jobs }} --mode parallel-jobs --parallelisation {{ jobs_count }} &" + poll: 30 + tags: + - parallel-jobs-submit + register: submitOutput + +- debug: + var: submitOutput + tags: + - parallel-jobs-submit + +# Execute Exhaust job sanity check script tasks + +- name: Install required python packages + pip: + name: + - requests + tags: + - run-sanity + +- name: Run sanity check python script + shell: python {{ analytics.home }}/scripts/exhaust_sanity_check.py + tags: + - run-sanity + register: SanityCheckStatus \ No newline at end of file diff --git a/ansible/roles/data-products-deploy/templates/bq-scripts.sh.j2 b/ansible/roles/data-products-deploy/templates/bq-scripts.sh.j2 new file mode 100644 index 0000000000..a53a650fad --- /dev/null +++ b/ansible/roles/data-products-deploy/templates/bq-scripts.sh.j2 @@ -0,0 +1,35 @@ +# Set your project ID +PROJECT_ID="{{cloud_storage_project_id}}" + +# Set the dataset name +DATASET_NAME="{{bq_dataset_name}}" + +# cloud storage folder name +STORAGE_FOLDER="{{cloud_storage_file_dest}}" + +# Loop over each table name +for TABLE_NAME in "assessment_detail" "bp_enrolments" "cb_plan" "content" "content_resource" "kcm_content_mapping" "kcm_dictionary" "org_hierarchy" "user_detail" "user_enrolments" "event_details" "event_enrolment_details"; do + + echo "Processing table: $TABLE_NAME" + + # Delete all existing avro files in Cloud Storage + gsutil rm -a gs://$STORAGE_FOLDER/$TABLE_NAME/*.avro + + # Copy avro files to Cloud Storage + gsutil cp /mount/data/analytics/warehouse/$TABLE_NAME/part*.avro gs://$STORAGE_FOLDER/$TABLE_NAME/ + + # delete existing table + bq rm -f $DATASET_NAME.$TABLE_NAME + + # create empty + bq mk --table $DATASET_NAME.$TABLE_NAME + + # Load data into the BigQuery table + bq load \ + --source_format=AVRO \ + $DATASET_NAME.$TABLE_NAME \ + "gs://$STORAGE_FOLDER/$TABLE_NAME/*.avro" + + echo "Completed processing for table: $TABLE_NAME" + +done diff --git a/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 new file mode 100644 index 0000000000..1a26514684 --- /dev/null +++ b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 @@ -0,0 +1,27 @@ +{ + "jars": [ + "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_core_artifact }}", + "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ scruid_artifact }}", + "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}" + ], + "file": "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_batch_module_artifact }}", + "files": [ + "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/application.conf" + ], + "className": "org.ekstep.analytics.job.JobExecutor", + "executorCores": {{ spark_cluster.executor_core }}, + "executorMemory": "{{ spark_cluster.executor_memory }}", + "numExecutors": {{ spark_cluster.num_executors }}, + "conf": { + "spark.sql.autoBroadcastJoinThreshold" : "-1", + "spark.dynamicAllocation.enabled" :"{{ spark_enable_dynamic_allocation }}", + "spark.shuffle.service.enabled" :"{{ spark_enable_dynamic_allocation }}", + "spark.sql.shuffle.partitions" : "{{ spark_sql_shuffle_partitions }}", + "spark.scheduler.mode" : "FAIR", + "spark.cassandra.connection.timeoutMS" : "{{ spark_cassandra_connection_timeout_millis }}", + "spark.cassandra.read.timeoutMS" : "{{ spark_cassandra_query_timeout_millis }}", + "spark.cassandra.input.fetch.sizeInRows": "{{ spark_cassandra_query_max_rows_fetch_count }}", + "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}", + "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}" + } +} diff --git a/ansible/roles/data-products-deploy/templates/common.conf.j2 b/ansible/roles/data-products-deploy/templates/common.conf.j2 index fe271b3e2f..c9c4f46a64 100644 --- a/ansible/roles/data-products-deploy/templates/common.conf.j2 +++ b/ansible/roles/data-products-deploy/templates/common.conf.j2 @@ -9,28 +9,33 @@ spark.cassandra.connection.host="{{groups['dp-cassandra'][0]}}" cassandra.keyspace_prefix="{{ cassandra_keyspace_prefix }}" cassandra.hierarchy_store_prefix="{{ cassandra_hierarchy_store_prefix }}" -# Content to vec configurations -pc_files_cache="s3" -pc_dispatch_params="""{"bucket":"{{dp_dispatch_bucket_name}}"}""" -pc_files_prefix="metrics/" - -metrics.dispatch.to="s3" -metrics.dispatch.params="""{"bucket":"{{dp_dispatch_bucket_name}}"}""" -metrics.consumption.dataset.id="eks-consumption-metrics/" -metrics.creation.dataset.id="eks-creation-metrics/" +storage.key.config="{{ cloud_storage_key }}" +storage.secret.config="""{{ cloud_secret_key }}""" +storage.secret.configfile="{{ analytics.home }}/models-{{ model_version }}/gcp_service_account.json" +reports.storage.key.config="{{ cloud_storage_key }}" +reports.storage.secret.config="""{{ cloud_secret_key }}""" +{% if dp_object_store_type == "azure" %} +cloud_storage_type="azure" +{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3") %} +cloud_storage_type="s3" +cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}" +cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}" +aws_storage_key="{{ s3_storage_key }}" +aws_storage_secret="""{{ s3_storage_secret }}""" +{% endif %} +cloud_storage_type="gcloud" +# NIC quirks +es.index.auto.create=false +es.nodes.wan.only=true +es.nodes.discovery=false lp.contentmodel.versionkey="jd5ECm/o0BXwQCe8PfZY1NoUkB9HN41QjA80p22MKyRIcP5RW4qHw8sZztCzv87M" -# Neo4j -neo4j.bolt.url="" -neo4j.bolt.user="" -neo4j.bolt.password="" - -# Log4j Kafka appender config -log4j.appender.kafka.enable="false" -log4j.appender.kafka.broker_host="{{groups['processing-cluster-kafka'][0]}}:9092" -log4j.appender.kafka.topic="{{ env }}.telemetry.backend" +# Joblog Kafka appender config for cluster execution +log.appender.kafka.enable="false" +log.appender.kafka.broker_host="{{groups['processing-cluster-kafka'][0]}}:9092" +log.appender.kafka.topic="{{ env }}.druid.events.log" # Kafka connection configuration kafka.consumer.brokerlist="{{groups['processing-cluster-kafka'][0]}}:9092" @@ -38,25 +43,11 @@ kafka.consumer.topic="{{ env }}.analytics.job_queue" no_of_jobs=42 # Spark Driver -spark.driver.memory=6g +spark.driver.memory={{ spark.driver.memory }} spark.memory_fraction={{ spark.memory_fraction }} spark.storage_fraction={{ spark.storage_fraction }} -spark.driver_memory="{{ spark.driver_memory }}" - -#Data Exhaust - -data_exhaust { - save_config { - save_type="azure" - bucket="{{data_exhaust_blob_name}}" - prefix="data-exhaust/" - public_s3_url="{{data_exhaust_s3_url}}" - local_path="/mount/data/analytics/{{ env }}/tmp/" - } - delete_source: "true" - package.enable: "true" -} +spark.driver_memory="{{ spark.driver_memory }}" #Monitor Jobs @@ -75,14 +66,6 @@ default.consumption.app.id="no_value" default.channel.id="in.ekstep" default.creation.app.id="no_value" -metrics.creation.es.url="http://{{groups['telemetry-search-cluster-kibana-node'][0]}}:9200" -metrics.creation.es.indexes="compositesearch" - -storage-service.request-signature-version="AWS4-HMAC-SHA256" -s3service.region="ap-south-1" - -cloud_storage_type="azure" - # Media Service Type media_service_type = "azure" @@ -140,33 +123,23 @@ cloud.container.reports="reports" # course metrics container in azure course.metrics.cassandra.sunbirdKeyspace="sunbird" course.metrics.cassandra.sunbirdCoursesKeyspace="sunbird_courses" +course.metrics.cassandra.sunbirdHierarchyStore="{{ cassandra_hierarchy_store_keyspace }}" course.metrics.cloud.objectKey="" -course.metrics.temp.dir="/mount/data/analytics/course-reports" course.metrics.cassandra.input.consistency="QUORUM" es.host="http://{{groups['core-es'][0]}}" es.port="9200" -course.metrics.es.alias="cbatchstats" -course.metrics.es.index.cbatchstats.prefix="cbatchstats-" -course.metrics.es.index.cbatch="course-batch" es.composite.host="{{groups['composite-search-cluster'][0]}}" # State admin user reports # Uses azure only - course.metrics.cloud.provider -admin.reports.cloud.container="reports" admin.metrics.cloud.objectKey="" admin.metrics.temp.dir="/mount/data/analytics/admin-user-reports" #Assessment report config -assessment.metrics.temp.dir="/mount/data/analytics/assessment-report" -assessment.metrics.cassandra.input.consistency="QUORUM" -assessment.metrics.cloud.objectKey="assessment-reports/" -assessment.metrics.content.index="compositesearch" es.scroll.size = 1000 -assessment.metrics.es.alias="cbatch-assessment" -assessment.metrics.es.index.prefix="cbatch-assessment-" -course.upload.reports.enabled=true -course.es.index.enabled=true -assessment.metrics.bestscore.report=true // BestScore or Latst Updated Score + +#BestScore or Latst Updated Score +assessment.metrics.bestscore.report=true assessment.metrics.supported.contenttype="SelfAssess" spark.sql.caseSensitive=true @@ -174,14 +147,14 @@ spark.sql.caseSensitive=true druid.sql.host="http://{{druid_broker_host}}:8082/druid/v2/sql/" druid.unique.content.query="{\"query\":\"SELECT DISTINCT \\\"object_id\\\" AS \\\"Id\\\"\\nFROM \\\"druid\\\".\\\"summary-events\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP '%s' AND TIMESTAMP '%s'\"}" -druid.content.rating.query="{\"query\":\"SELECT \\\"object_id\\\" AS contentId, COUNT(*) AS \\\"totalRatingsCount\\\", SUM(edata_rating) AS \\\"Total Ratings\\\", SUM(edata_rating)/COUNT(*) AS \\\"averageRating\\\" FROM \\\"druid\\\".\\\"telemetry-feedback-events\\\" WHERE \\\"eid\\\" = 'FEEDBACK' GROUP BY \\\"object_id\\\"\"}" +druid.content.rating.query="{\"query\":\"SELECT \\\"object_id\\\" AS contentId, COUNT(*) AS \\\"totalRatingsCount\\\", SUM(edata_rating) AS \\\"Total Ratings\\\", SUM(edata_rating)/COUNT(*) AS \\\"averageRating\\\" FROM \\\"druid\\\".\\\"telemetry-feedback-events\\\" WHERE \\\"eid\\\" = 'FEEDBACK' AND \\\"edata_rating\\\">0 GROUP BY \\\"object_id\\\"\"}" druid.content.consumption.query="{\"query\":\"SELECT COUNT(*) as \\\"play_sessions_count\\\", object_id as \\\"contentId\\\", SUM(total_time_spent) as \\\"total_time_spent\\\", dimensions_pdata_id, object_id\\nFROM \\\"summary-events\\\"\\nWHERE \\\"dimensions_mode\\\" = 'play' AND \\\"dimensions_type\\\" ='content'\\nGROUP BY object_id, dimensions_pdata_id\"}" lp.system.update.base.url="{{lp_url}}/system/v3/content/update" #Experiment Configuration -user.search.api.url="http://{{sunbird_learner_service_url}}/private/user/v1/search" +user.search.api.url="{{sunbird_learner_service_url}}/private/user/v1/search" user.search.limit="10000" # pipeline auditing @@ -209,32 +182,131 @@ druid = { url = "/druid/v2/" datasource = "telemetry-events" response-parsing-timeout = 300000 + client-backend = "ing.wbaa.druid.client.DruidAdvancedHttpClient" + client-config = { + druid-advanced-http-client ={ + queue-size = 32768 + queue-overflow-strategy = "Backpressure" + query-retries = 5 + query-retry-delay = 10 ms + host-connection-pool = { + max-connections = 32 + min-connections = 0 + max-open-requests = 128 + max-connection-lifetime = 20 min + idle-timeout = 15 min + client = { + # The time after which an idle connection will be automatically closed. + # Set to `infinite` to completely disable idle timeouts. + idle-timeout = 10 min + parsing.max-chunk-size = 10m + } + } + } + + } } druid.rollup.host="{{druid_rollup_broker_host}}" druid.rollup.port=8082 -druid.query.wait.time.mins=1 -druid.report.upload.wait.time.mins=1 +druid.query.wait.time.mins=10 +druid.report.upload.wait.time.mins=10 +druid.scan.batch.size=100 +druid.scan.batch.bytes=2000000 +druid.query.batch.buffer=500000 + // Metric event config metric.producer.id="pipeline.monitoring" metric.producer.pid="dataproduct.metrics" push.metrics.kafka=true metric.kafka.broker="{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" -metric.kafka.topic="{{ env }}.telemetry.metrics" +metric.kafka.topic="{{ env }}.prom.monitoring.metrics" //Postgres Config postgres.db="{{postgres.db_name}}" postgres.url="jdbc:postgresql://{{postgres.db_url}}:{{postgres.db_port}}/" postgres.user="{{postgres.db_username}}" postgres.pass="{{postgres.db_password}}" +postgres.program.table="program" +postgres.nomination.table="nomination" +postgres.usertable="\"V_User\"" +postgres.org.table="\"V_User_Org\"" + +druid.ingestion.path="/druid/indexer/v1/task" +druid.segment.path="/druid/coordinator/v1/metadata/datasources/" +druid.deletesegment.path="/druid/coordinator/v1/datasources/" + +postgres.druid.db="{{ druid_report_postgres_db_name }}" +postgres.druid.url="jdbc:postgresql://{{postgres.db_url}}:{{postgres.db_port}}/" +postgres.druid.user="{{ druid_report_postgres_db_username }}" +postgres.druid.pass="{{ dp_vault_druid_postgress_pass }}" location.search.url="https://{{location_search_url}}/v1/location/search" location.search.token="{{ location_search_token }}" +location.search.request="{\"request\": {\"filters\": {\"type\" :[\"state\",\"district\"]},\"limit\" : 10000}}" druid.state.lookup.url = "http://{{groups['raw-coordinator'][0]}}:8081/druid/coordinator/v1/lookups/config/__default/stateSlugLookup" +sunbird_encryption_key="{{ core_vault_sunbird_encryption_key }}" + dcedialcode.filename="DCE_dialcode_data.csv" etbdialcode.filename="ETB_dialcode_data.csv" dcetextbook.filename="DCE_textbook_data.csv" etbtextbook.filename="ETB_textbook_data.csv" + + +druid.report.default.storage="azure" +druid.report.date.format="yyyy-MM-dd" +druid.report.default.container="report-verification" + +## Collection Exhaust Jobs Configuration -- Start ## + +sunbird.user.keyspace="{{ user_table_keyspace }}" +sunbird.courses.keyspace="{{ course_keyspace }}" +sunbird.content.hierarchy.keyspace="{{ cassandra_hierarchy_store_keyspace }}" +sunbird.user.cluster.host="{{ core_cassandra_host }}" +sunbird.courses.cluster.host="{{ core_cassandra_host }}" +sunbird.content.cluster.host="{{ core_cassandra_host }}" +sunbird.report.cluster.host="{{ report_cassandra_cluster_host }}" +sunbird.user.report.keyspace="{{ report_user_table_keyspace }}" +collection.exhaust.store.prefix="" +postgres.table.job_request="{{ job_request_table }}" +postgres.table.dataset_metadata="{{ dataset_metadata_table }}" + +## Collection Exhaust Jobs Configuration -- End ## + +## Exhaust throttling variables +exhaust.batches.limit.per.channel={{ exhaust_batches_limit_per_channel }} +exhaust.file.size.limit.per.channel={{ exhaust_file_size_limit_bytes_per_channel }} + +exhaust.parallel.batch.load.limit={{ exhaust_parallel_batch_load_limit }} +exhaust.user.parallelism={{ exhaust_user_parallelism }} + +data_exhaust.batch.limit.per.request={{ data_exhaust_batch_limit_per_request }} + + + +//START of UCI Postgres Config + +uci.conversation.postgres.db="{{ uci_postgres.conversation_db_name }}" +uci.conversation.postgres.url="jdbc:postgresql://{{uci_postgres.conversation_db_host}}:{{uci_postgres.conversation_db_port}}/" + +uci.fushionauth.postgres.db="{{ uci_postgres.fushionauth_db_name }}" +uci.fushionauth.postgres.url="jdbc:postgresql://{{uci_postgres.fushionauth_db_host}}:{{uci_postgres.fushionauth_db_port}}/" + +uci.postgres.table.conversation="{{ uci_postgres.conversation_table_name }}" +uci.postgres.table.user="{{ uci_postgres.user_table_name }}" +uci.postgres.table.user_registration="{{ uci_postgres.user_registration_table_name }}" +uci.postgres.table.identities="{{ uci_postgres.user_identities_table_name }}" + +uci.conversation.postgres.user="{{ uci_postgres.conversation_db_user }}" +uci.conversation.postgres.pass="{{ uci_postgres.conversation_db_psss }}" + +uci.fushionauth.postgres.user="{{ uci_postgres.fushionauth_db_user }}" +uci.fushionauth.postgres.pass="{{ uci_postgres.fushionauth_db_psss }}" + +uci.exhaust.store.prefix="" +uci.encryption.secret="{{ uci_encryption_secret_key }}" + +// END OF UCI Related Job Configs \ No newline at end of file diff --git a/ansible/roles/data-products-deploy/templates/es-form-data.j2 b/ansible/roles/data-products-deploy/templates/es-form-data.j2 new file mode 100644 index 0000000000..6087244e3d --- /dev/null +++ b/ansible/roles/data-products-deploy/templates/es-form-data.j2 @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# Get the form ID from the argument +FORM_ID=$1 +OUTPUT_FILE="/mount/data/analytics/es-form-data/es_${FORM_ID}.json" +LOG_FILE="/mount/data/analytics/es-form-data/es_${FORM_ID}_log.txt" + +source model-config.sh + +ES_HOST="{{ single_node_es_host }}" +ES_PORT="{{ single_node_es_port }}" +INDEX_NAME="{{ es_form_data_index }}" + +# Elasticsearch URL and index name +ES_URL="http://$ES_HOST:$ES_PORT" + +SCROLL_TIME="1m" # Keep the search context alive + +# Function to log messages +log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +log_message "Fetching data for form ID: $FORM_ID using Scroll API" + +# Initial scroll request +INITIAL_RESPONSE=$(curl -s -X POST "$ES_URL/$INDEX_NAME/_search?scroll=$SCROLL_TIME" -H 'Content-Type: application/json' -d '{ + "query": { + "bool": { + "must": [ + { + "match": { + "formId": '"$FORM_ID"' + } + } + ] + } + }, + "size": 10000 +}') + +# Extract the initial scroll_id from the response +SCROLL_ID=$(echo "$INITIAL_RESPONSE" | jq -r '._scroll_id') + +# Initialize the output file with the hits from the first scroll batch +HITS=$(echo "$INITIAL_RESPONSE" | jq -r '.hits.hits') +echo "$HITS" > "$OUTPUT_FILE" +log_message "Initial batch of hits saved to $OUTPUT_FILE" + +# Check the number of hits in the initial response +NUM_HITS=$(echo "$INITIAL_RESPONSE" | jq '.hits.hits | length') +log_message "Number of hits in initial response: $NUM_HITS" + +# If there are no hits, exit early +if [[ "$NUM_HITS" -eq 0 ]]; then + log_message "No hits found. Exiting." + echo "No hits found. Exiting." + exit 0 +fi + +# Process the remaining scroll responses +while true; do + # Fetch the next batch of data using the scroll ID + NEXT_RESPONSE=$(curl -s -X POST "http://$ES_HOST:$ES_PORT/_search/scroll" -H 'Content-Type: application/json' -d '{ + "scroll": "'"$SCROLL_TIME"'", + "scroll_id": "'"$SCROLL_ID"'" + }') + + # Extract the scroll ID and hits from the response + SCROLL_ID=$(echo "$NEXT_RESPONSE" | jq -r '._scroll_id') + HITS=$(echo "$NEXT_RESPONSE" | jq -r '.hits.hits') + + # Check the number of hits in the current response + NUM_HITS=$(echo "$NEXT_RESPONSE" | jq '.hits.hits | length') + log_message "Number of hits in this response: $NUM_HITS" + + # If no more hits are returned, break the loop + if [[ "$NUM_HITS" -eq 0 ]]; then + log_message "No more hits. Exiting." + echo "No more hits. Exiting." + break + fi + + # Append the hits to the output file + echo "$HITS" >> "$OUTPUT_FILE" + log_message "Appended batch of hits to $OUTPUT_FILE" +done + +# Print the results +log_message "Data has been saved to $OUTPUT_FILE" +echo "Data has been saved to $OUTPUT_FILE" diff --git a/ansible/roles/data-products-deploy/templates/exhaust_sanity_check.py.j2 b/ansible/roles/data-products-deploy/templates/exhaust_sanity_check.py.j2 new file mode 100644 index 0000000000..3f6ba98d9d --- /dev/null +++ b/ansible/roles/data-products-deploy/templates/exhaust_sanity_check.py.j2 @@ -0,0 +1,58 @@ +import requests +from requests.auth import HTTPBasicAuth +import json +from kafka import KafkaConsumer +from json import loads +import sys + +def checkClusterStatus(): + try: + res = requests.get('https://{{ spark_cluster_name }}.azurehdinsight.net/api/v1/clusters/{{ spark_cluster_name }}/alerts?format=summary', auth = HTTPBasicAuth("{{ admin_name }}" ,"{{ admin_password }}")) + if(res.status_code == 200): + resJson = json.loads(res.text) + warningCount = resJson["alerts_summary"]["WARNING"]["count"] + criticalCount = resJson["alerts_summary"]["CRITICAL"]["count"] + unknownCount = resJson["alerts_summary"]["UNKNOWN"]["count"] + if((warningCount + criticalCount + unknownCount) == 0): + print("Cluster is up & running fine. With these - WARNING:{0}, CRITICAL:{1}, UNKNOWN:{2}".format(warningCount, criticalCount, unknownCount)) + return "SUCCESS" + else: + return "FAILED. Cluster is not running properly. Found these - WARNING:{0}, CRITICAL:{1}, UNKNOWN:{2}".format(warningCount, criticalCount, unknownCount) + else: + return "FAILED. Cluster failed to provide response. Resulted in {0} response".format(res.status_code) + except Exception as e: + return "FAILED with {0}".format(str(e)) + +def checkCassandraMigratorStatus(): + try: + ## from joblog file + migratorENDEvent = "" + with open ('{{ analytics.home }}/scripts/logs/joblog.log', 'rt') as logs: + for log in logs: + if (log.count("JOB_END") == 1 & log.count("{{ cassandra_migrator_job_name }}") == 1): + migratorENDEvent = log + logJson = json.loads(migratorENDEvent) + jobStatus = logJson["edata"]["status"] + if (jobStatus == "SUCCESS"): + print("Cassandra Migrator Completed successfully!") + return "SUCCESS" + else: + return "Cassandra Migrator failed" + except Exception as e: + return "FAILED with {0}".format(str(e)) + + +def main(): + finalSuccessMessage="All checks are successful" + ## check Cassandra Migrator status + cassandraMigratorState=checkCassandraMigratorStatus() + ## check spark cluster status + clusterState=checkClusterStatus() + + if(cassandraMigratorState == "SUCCESS" and clusterState == "SUCCESS"): + return finalSuccessMessage + else: + raise Exception("Required checks failed. Job Status: {0} and Cluster status: {1}".format(cassandraMigratorState, clusterState)) + +result=main() +print(result) \ No newline at end of file diff --git a/ansible/roles/data-products-deploy/templates/generate-metrics.j2 b/ansible/roles/data-products-deploy/templates/generate-metrics.j2 deleted file mode 100644 index d87a454597..0000000000 --- a/ansible/roles/data-products-deploy/templates/generate-metrics.j2 +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -set_env="{{business_metrics.env}}" -cd {{ analytics.home }}/scripts/BusinessMetrics -#python template_usage_by_content.py $set_env >> $log_file_path -python template_usage_by_items.py $set_env -echo "template_usage_by_items completed." -python concept_coverage_across_items.py $set_env -echo "concept_coverage_across_items completed." -python concept_coverage_across_content.py $set_env -echo "concept_coverage_across_content completed." -python asset_usage_in_content.py $set_env -echo "asset_usage_in_content completed." -aws s3 cp metrics/ {{business_metrics.s3_path}} --recursive --region ap-south-1 -echo "pushing metrics to s3 completed." -rm -rf metrics/ -echo "Metrics generated and pushed to S3 successfully." diff --git a/ansible/roles/data-products-deploy/templates/model-config.j2 b/ansible/roles/data-products-deploy/templates/model-config.j2 index 717caad340..d191e8aa8f 100644 --- a/ansible/roles/data-products-deploy/templates/model-config.j2 +++ b/ansible/roles/data-products-deploy/templates/model-config.j2 @@ -1,125 +1,117 @@ #!/usr/bin/env bash config() { - bucket={{ bucket }} - datasetRawBucket={{ datasetRawBucket }} - dataExhaustBucket={{ dataExhaustBucket }} - dataExhaustPrefix={{ dataExhaustPrefix }} - consumptionRawPrefix={{ consumptionRawPrefix }} + bucket={{ cloud_storage_bucket_secor }} brokerList={{ brokerlist }} - zookeeper={{ zookeeper }} - job_topic={{ analytics_job_queue_topic }} topic={{ topic }} - analyticsMetricsTopic={{ analytics_metrics_topic }} - learning_topic={{ learning_topic }} - sinkTopic={{ sink_topic }} - metricsTopic={{ metrics_topic }} - currentDate=$(date "+%Y-%m-%d"); - analytics_home={{ analytics.home }} - temp_folder={{ job_manager_tmp_dir }} - sparkCassandraConnectionHost="{{groups['lp-cassandra'][0]}}" - sunbirdPlatformCassandraHost="{{ core_cassandra_host }}" - sunbirdPlatformElasticsearchHost="{{ sunbird_es_host }}" - jobManagerJobsCount="{{ analytics_jobs_count }}" - dataExhaustChannel="{{ data_exhaust_Channel }}" - pipelineAuditSlackUser="Pipeline Audit" - druidBrokerUrl="http://{{druid_broker_host}}:8082/" - producerEnv="{{ producer_env }}" - baseScriptPath="{{ spark_output_temp_dir }}" - reportPostContainer="{{ reports_container }}" - + sparkCassandraConnectionHost="{{ lp_cassandra_host }}" + reportMDOIDs="{{ report_mdo_ids }}" + reportSolutionIDs="{{ report_solution_ids }}" + anonymousAssessmentNonLoggedInUserAssessmentIDs="{{ anonymous_assessment_non_loggedIn_user_assessment_ids }}" + anonymousAssessmentLoggedInUserContentIDs="{{ anonymous_assessment_loggedIn_user_content_ids }}" + if [ -z "$2" ]; then endDate=$(date --date yesterday "+%Y-%m-%d"); else endDate=$2; fi - if [ ! -z "$3" ]; then inputBucket=$3; fi - if [ ! -z "$4" ]; then sinkTopic=$4; fi + case "$1" in - "cmu") - if [ -z "$2" ]; then queryDate=$currentDate; else queryDate=$endDate; fi - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.updater.ContentModelUpdater","modelParams":{"date":"'$queryDate'","contentTypes":{"Content":["Resource", "Collection", "TextBook", "LessonPlan", "Course", "Template", "Asset", "Plugin", "LessonPlanUnit","CourseUnit", "TextBookUnit"]}},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$learning_topic'"}}],"parallelization":10,"appName":"Content Model Updater","deviceMapping":false}' - ;; - "data-exhaust") - #echo '{"search":{"type":"azure"},"model":"org.ekstep.analytics.model.DataExhaustJobModel","parallelization":8,"appName":"Data Exhaust","deviceMapping":false,"modelParams":{"shouldDelay":true,"delayInMilis":1800000},"exhaustConfig":{"eks-consumption-raw":{"events":["eks-consumption-raw"],"eventConfig":{"eks-consumption-raw":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"unique/"}},"filterMapping":{"tags":{"name":"tags","operator":"IN"},"channel":{"name":"context.channel","operator":"EQ"},"app_id":{"name":"context.pdata.id","operator":"EQ"}}}}},"eks-consumption-summary":{"events":["ME_SESSION_SUMMARY","ME_CONTENT_USAGE_SUMMARY","ME_GENIE_LAUNCH_SUMMARY","ME_ITEM_SUMMARY","ME_GENIE_USAGE_SUMMARY","ME_ITEM_USAGE_SUMMARY","ME_ASSET_SNAPSHOT_SUMMARY","ME_CONTENT_SNAPSHOT_SUMMARY","ME_DEVICE_USAGE_SUMMARY","ME_GENIE_SESSION_SUMMARY"],"eventConfig":{"ME_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"ss/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.timeSpent":{"to":"Time spent on the content"},"edata.eks.screenSummary":{"hidden":true},"edata.eks.currentLevel":{"to":"Current Domain Level"},"edata.eks.syncDate":{"mapFunc":"timestampToDateTime","to":"Sync Date(IST)"},"edata.eks.timeDiff":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Content Start Time(IST)"},"uid":{"to":"Genie User ID"},"ets":{"hidden":true},"dimensions.group_user":{"to":"Is Group User"},"edata.eks.activitySummary":{"to":"Activity Summary"},"edata.eks.end_time":{"hidden":true},"edata.eks.eventsSummary":{"hidden":true},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.levels":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.itemResponses":{"to":"edata.eks.itemResponses"},"dimensions.gdata.id":{"to":"Content ID"},"edata.eks.noOfLevelTransitions":{"to":"Number of Level Transitions"},"edata.eks.noOfAttempts":{"to":"Number of Attempts"},"tags":{"to":"Genie Tags"},"edata.eks.interruptTime":{"to":"Total interrupt time"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time stamp(IST)"},"edata.eks.mimeType":{"hidden":true},"edata.eks.telemetryVersion":{"to":"Telemetry Version"},"context.pdata.ver":{"hidden":true},"dimensions.gdata.ver":{"hidden":true},"edata.eks.interactEventsPerMin":{"to":"Number of interactions per minute"},"edata.eks.contentType":{"to":"Content Type"},"context.pdata.model":{"hidden":true},"dimensions.anonymous_user":{"to":"Logged in User"},"dimensions.did":{"to":"Device ID"},"dimensions.loc":{"to":"Lat / Long"},"mid":{"to":"Session ID"},"edata.eks.noOfInteractEvents":{"to":"Number of Interact events in the content session"},"context.date_range.to":{"hidden":true}}}},"ME_CONTENT_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"cus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_GENIE_LAUNCH_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"gls/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_ITEM_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"is/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_GENIE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"genie-launch-summ/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"uid":{"to":"UIDs"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"edata.eks.total_ts":{"to":"Total time (secs) per tag for the period"},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.contents":{"to":"Update col header\tContent IDs of all contents used per tag for the period"},"context.pdata.id":{"hidden":true},"edata.eks.avg_ts_session":{"to":"Average time spent per Genie session"},"edata.eks.device_ids":{"to":"Device IDs per tag for the period"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.tag":{"to":"Tag name"},"edata.eks.total_sessions":{"to":"Total number of sessions per tag for the period"},"mid":{"to":"Unique Genie Session ID"},"context.date_range.to":{"hidden":true},"dimensions.period":{"to":"period"}}}},"ME_ITEM_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"item-usage-summ/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_ASSET_SNAPSHOT_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"asset-snapshot/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.used_activities_count":{"to":"No. of activities that are part of lessons"},"edata.eks.used_templates_count":{"to":"No. of templates that are part of lessons"},"context.date_range.from":{"hidden":true},"uid":{"to":"uid"},"edata.eks.total_questions_count":{"to":"Total number of questions"},"ets":{"mapFunc":"timestampToDateTime","to":"Generation Time(IST)"},"edata.eks.used_audio_count":{"to":"No. of audio files that are being used"},"edata.eks.used_images_count":{"to":"No. of images that are being used"},"edata.eks.used_questions_count":{"to":"No. of questions that are being used"},"edata.eks.total_activities_count":{"to":"Total number of activities"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"edata.eks.total_templates_count":{"to":"Total no. of templates"},"dimensions.partner_id":{"to":"Partner ID"},"context.pdata.ver":{"hidden":true},"edata.eks.total_images_count":{"to":"Total no. of images"},"context.pdata.model":{"hidden":true},"edata.eks.total_audio_count":{"to":"Total no. of audio files"},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_CONTENT_SNAPSHOT_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"content-snapshot/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.active_user_count":{"to":"No. of active authors on the portal"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"dimensions.author_id":{"to":"Author ID"},"edata.eks.total_user_count":{"to":"Total number of registered authors on the portal"},"edata.eks.live_content_count":{"to":"Total number of content pieces live"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"edata.eks.total_content_count":{"to":"Total number of content pieces"},"edata.eks.review_content_count":{"to":"Total number of content pieces pending review"},"dimensions.partner_id":{"to":"Partner ID"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_DEVICE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"dus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.total_play_time":{"to":"Total Play time on device"},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Time stamp of first Genie installation on device(IST)"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"edata.eks.avg_num_launches":{"to":"Average num. of Genie launches per day"},"edata.eks.num_contents":{"to":"No. of content pieces available on device"},"edata.eks.play_start_time":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device first started game play(IST)"},"edata.eks.end_time":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device was last used(IST)"},"eid":{"hidden":true},"edata.eks.num_days":{"to":"Number of days since Genie installed before the timestamp indicating the last use"},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.mean_play_time":{"to":"Average play time on device"},"edata.eks.mean_play_time_interval":{"to":"Average time interval between game play on the device"},"edata.eks.last_played_on":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device was last used for game play(IST)"},"edata.eks.avg_time":{"to":"Average time on Genie per day"},"edata.eks.last_played_content":{"to":"Last played content on device"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.did":{"to":"Device ID"},"edata.eks.num_sessions":{"to":"Total number of sessions from the device"},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_GENIE_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"cus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.content":{"to":"Content IDs of content used in Genie Session"},"edata.eks.timeSpent":{"to":"Time spent on Genie Session\t"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"dimensions.group_user":{"to":"Is Group user?"},"edata.eks.contentCount":{"to":"Number of content pieces used"},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.time_stamp":{"mapFunc":"timestampToDateTime","to":"Session End time stamp(IST)"},"context.pdata.id":{"hidden":true},"tags":{"to":"Genie tags"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.anonymous_user":{"to":"Logged in user?"},"dimensions.did":{"to":"Device ID"},"mid":{"to":"Unique Genie Session ID"},"context.date_range.to":{"hidden":true}}}}}},"eks-consumption-metrics":{"events":["ME_CONTENT_USAGE_METRICS","ME_ITEM_USAGE_METRICS","ME_GENIE_USAGE_METRICS","ME_CONTENT_SNAPSHOT_METRICS","ME_CONCEPT_SNAPSHOT_METRICS","ME_ASSET_SNAPSHOT_METRICS"],"eventConfig":{"ME_CONTENT_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_content_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.content_id":{"to":"Content ID"},"dimensions.tag":{"to":"tag"},"edata.eks.m_publish_date":{"to":"Date when the content is published"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_total_sessions":{"to":"Total number of sessions"},"edata.eks.m_avg_ts_session":{"to":"Average timespent per session"},"edata.eks.m_total_interactions":{"to":"Total interactions count"},"edata.eks.m_avg_interactions_min":{"to":"Average interactions per minute"},"edata.eks.m_total_devices":{"to":"Total number of devices"},"edata.eks.m_avg_sess_device":{"to":"Average sessions per device"}}}},"ME_ITEM_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_item_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.tag":{"to":"tag"},"dimensions.content_id":{"to":"Content ID"},"dimensions.item_id":{"to":"item id"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_total_count":{"to":"Total response count"},"edata.eks.m_correct_res_count":{"to":"Correct response count"},"edata.eks.m_inc_res_count":{"to":"Incorrect response count"},"edata.eks.m_correct_res":{"hidden":true},"edata.eks.m_incorrect_res":{"hidden":true},"edata.eks.m_top5_incorrect_res":{"to":"Top 5 Incorrect response"},"edata.eks.m_avg_ts":{"to":"Average time spent"},"edata.eks.m_top5_misconception_res":{"to":"Top 5 Misconception response"}}}},"ME_GENIE_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_genie_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.tag":{"to":"Device Tag"},"edata.eks.m_total_sessions":{"to":"Total number of sessions"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_avg_ts_session":{"to":"Average sessions per device"},"edata.eks.m_contents":{"hidden":true},"edata.eks.m_total_devices":{"to":"Total number of devices"}}}},"ME_CONTENT_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_content_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_CONCEPT_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_concept_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_ASSET_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_asset_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}}}},"eks-creation-raw":{"events":["eks-creation-raw"],"eventConfig":{"eks-creation-raw":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"unique/"}},"filterMapping":{"channel":{"name":"context.channel","operator":"EQ"},"app_id":{"name":"context.pdata.id","operator":"EQ"}}}}},"eks-creation-summary":{"events":["ME_APP_SESSION_SUMMARY","ME_CE_SESSION_SUMMARY","ME_TEXTBOOK_SESSION_SUMMARY","ME_APP_USAGE_SUMMARY","ME_CE_USAGE_SUMMARY","ME_TEXTBOOK_USAGE_SUMMARY","ME_AUTHOR_USAGE_SUMMARY","ME_PUBLISH_PIPELINE_SUMMARY"],"eventConfig":{"ME_APP_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"app-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.interact_events_per_min":{"to":"Average ineractions per minute"},"edata.eks.events_summary":{"to":"edata.eks.events_summary"},"dimensions.app_id":{"to":"Event Source"},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Start Time(IST)"},"uid":{"to":"User ID"},"edata.eks.page_summary":{"to":"Pages visited Summary"},"ets":{"hidden":true},"edata.eks.page_views_count":{"to":"Total pageviews in Session"},"edata.eks.end_time":{"hidden":true},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.ce_visits":{"to":"Number of Authoring tool visits"},"dimensions.sid":{"to":"Session ID"},"edata.eks.time_diff":{"to":"Session length "},"edata.eks.interact_events_count":{"to":"Number of Interactions"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.time_spent":{"to":"Session length"},"dimensions.anonymous_user":{"to":"User Logged In"},"mid":{"to":"mid"},"edata.eks.env_summary":{"to":"Usage summary"},"edata.eks.first_visit":{"to":"First Time Visitor"},"context.date_range.to":{"hidden":true}}}},"ME_CE_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"ce-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_TEXTBOOK_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"textbook-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_APP_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"app-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"edata.eks.avg_ts_session":{"to":"Average session duration per day"},"context.granularity":{"to":"Granularity"},"edata.eks.anon_total_sessions":{"to":"Time spent by non-registered users"},"edata.eks.unique_users":{"to":"Unique users list"},"dimensions.app_id":{"to":"Source"},"edata.eks.unique_users_count":{"to":"Unique users count"},"context.date_range.from":{"hidden":true},"edata.eks.total_pageviews_count":{"to":"Total Page Views per day"},"uid":{"to":"uid"},"edata.eks.ce_total_sessions":{"to":"Number of sessions where creation happened"},"edata.eks.anon_avg_ts_session":{"to":"Avg. session duration for non-registered users per day"},"ets":{"mapFunc":"timestampToDateTime","to":"Generation Time(IST)"},"edata.eks.total_ts":{"to":"Total time spent by visitors on Portal"},"edata.eks.ce_percent_sessions":{"to":"Percentage of visitors creating content"},"dimensions.author_id":{"to":"Author ID"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.new_user_count":{"to":"Number of new users"},"syncts":{"to":"Event sync time(IST)","mapFunc":"timestampToDateTime"},"edata.eks.percent_new_users_count":{"to":"Percentage of new visitors per day"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.avg_pageviews":{"to":"Avg. pages per visit per day"},"edata.eks.total_sessions":{"to":"Total number of sessions"},"edata.eks.anon_total_ts":{"to":"Total time spent by non-registered users on Portal"},"mid":{"hidden":true},"dimensions.period":{"to":"dimensions.period"},"context.date_range.to":{"hidden":true}}}},"ME_CE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"ce-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_TEXTBOOK_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"textbook-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_AUTHOR_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"author-usage-summary/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"uid":{"to":"Author ID"},"mid":{"hidden":true},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"eid":{"hidden":true},"ver":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.id":{"hidden":true},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.avg_ts_session":{"to":"Total number of seesion per day"},"edata.eks.total_ts":{"to":"Total time spent by author"},"edata.eks.ce_total_ts":{"to":"Total time spent on content editor by author"},"edata.eks.total_sessions":{"to":"No of sessions per Author"},"context.granularity":{"hidden":true},"edata.eks.ce_percent_ts":{"to":"Percentage of time spent browsing vs creating"},"edata.eks.ce_percent_sessions":{"to":"Percentage of sessions in which users log into content editor"},"edata.eks.ce_total_visits":{"to":"No of Content Editor visits per author"},"context.date_range.from":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_PUBLISH_PIPELINE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"publish-pipeline-summ/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"uid":{"hidden":true},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.publish_pipeline_summary":{"to":"edata.eks.publish_pipeline_summary"},"syncts":{"to":"Event sync time(IST)","mapFunc":"timestampToDateTime"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"mid":{"hidden":true},"dimensions.period":{"to":"Date"},"context.date_range.to":{"hidden":true}}}}}},"eks-creation-metrics":{"events":["ME_APP_USAGE_METRICS","ME_CE_USAGE_METRICS","ME_TEXTBOOK_CREATION_METRICS","ME_TEXTBOOK_SNAPSHOT_METRICS","ME_AUTHOR_USAGE_METRICS"],"eventConfig":{"ME_APP_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_app_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.author_id":{"to":"Author ID / All"},"dimensions.app_id":{"to":"App (EkStep portal currently)"},"edata.eks.anon_total_sessions":{"to":"Total anonymous sessions on portal"},"edata.eks.anon_total_ts":{"to":"Tota time spent (Anonymous) on portal"},"edata.eks.total_sessions":{"to":"Total number of portal sessions"},"edata.eks.total_ts":{"to":"Total time spent on portal"},"edata.eks.ce_total_sessions":{"to":"Total content editor sessions"},"edata.eks.ce_percent_sessions":{"to":"% of total sessions that are content editor sessions"},"edata.eks.total_pageviews_count":{"to":"Total pageviews on portal"},"edata.eks.unique_users":{"hidden":true},"edata.eks.unique_users_count":{"to":"Count of unique users on portal"},"edata.eks.avg_pageviews":{"to":"Avg. number of pageviews per session for specified period"},"edata.eks.avg_ts_session":{"to":"Avg. session length (time spent) per session for specified period"},"edata.eks.anon_avg_ts_session":{"to":"Avg. Anonymous session length (time spent) per session for specified period"},"edata.eks.new_user_count":{"to":"No. of new users on the app (portal) for the given time period"},"edata.eks.percent_new_users_count":{"to":"% of new users for the given time period"}}}},"ME_CE_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_ce_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.content_id":{"to":"Content ID"},"edata.eks.unique_users_count":{"to":"Number of users who accessed the content piece via the editor"},"edata.eks.total_sessions":{"to":"Total number of sessions that accessed the content piece via the editor"},"edata.eks.total_ts":{"to":"Time spent on the content piece in the content editor"},"edata.eks.avg_ts_session":{"to":"Average session time on the editor"},"edata.eks.update_date":{"to":"Last updated timestamp of the content piece"}}}},"ME_TEXTBOOK_CREATION_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_textbook_creation_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_TEXTBOOK_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_textbook_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_AUTHOR_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_author_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":" Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.author_id":{"to":"Author ID /All"},"edata.eks.total_sessions":{"to":"Total number of portal sessions by author"},"edata.eks.total_ts":{"to":"Total time spent on portal by author"},"edata.eks.total_ce_ts":{"to":"Total time spent on content editor by author"},"edata.eks.total_ce_visit":{"to":"Number of content editor visits"},"edata.eks.percent_ce_sessions":{"to":"% of sessions that were spent on the editor"},"edata.eks.avg_ts_session":{"to":"Average time spent by author on portal per session"},"edata.eks.percent_ce_ts":{"to":"% time spent by author on content editor"}}}}}}}}' - echo '{"search":{"type":"azure"},"model":"org.ekstep.analytics.model.DataExhaustJobModel","parallelization":8,"appName":"Data Exhaust","deviceMapping":false,"modelParams":{"shouldDelay":true,"delayInMilis":1800000},"exhaustConfig":{"eks-consumption-raw":{"events":["eks-consumption-raw"],"eventConfig":{"eks-consumption-raw":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"channel/"}},"filterMapping":{"tags":{"name":"tags","operator":"IN"},"channel":{"name":"context.channel","operator":"EQ"},"app_id":{"name":"context.pdata.id","operator":"EQ"}}}}},"eks-consumption-summary":{"events":["ME_SESSION_SUMMARY","ME_CONTENT_USAGE_SUMMARY","ME_GENIE_LAUNCH_SUMMARY","ME_ITEM_SUMMARY","ME_GENIE_USAGE_SUMMARY","ME_ITEM_USAGE_SUMMARY","ME_ASSET_SNAPSHOT_SUMMARY","ME_CONTENT_SNAPSHOT_SUMMARY","ME_DEVICE_USAGE_SUMMARY","ME_GENIE_SESSION_SUMMARY","ME_WORKFLOW_SUMMARY"],"eventConfig":{"ME_WORKFLOW_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/wfs/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"dimensions.channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/ss/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.timeSpent":{"to":"Time spent on the content"},"edata.eks.screenSummary":{"hidden":true},"edata.eks.currentLevel":{"to":"Current Domain Level"},"edata.eks.syncDate":{"mapFunc":"timestampToDateTime","to":"Sync Date(IST)"},"edata.eks.timeDiff":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Content Start Time(IST)"},"uid":{"to":"Genie User ID"},"ets":{"hidden":true},"dimensions.group_user":{"to":"Is Group User"},"edata.eks.activitySummary":{"to":"Activity Summary"},"edata.eks.end_time":{"hidden":true},"edata.eks.eventsSummary":{"hidden":true},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.levels":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.itemResponses":{"to":"edata.eks.itemResponses"},"dimensions.gdata.id":{"to":"Content ID"},"edata.eks.noOfLevelTransitions":{"to":"Number of Level Transitions"},"edata.eks.noOfAttempts":{"to":"Number of Attempts"},"tags":{"to":"Genie Tags"},"edata.eks.interruptTime":{"to":"Total interrupt time"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time stamp(IST)"},"edata.eks.mimeType":{"hidden":true},"edata.eks.telemetryVersion":{"to":"Telemetry Version"},"context.pdata.ver":{"hidden":true},"dimensions.gdata.ver":{"hidden":true},"edata.eks.interactEventsPerMin":{"to":"Number of interactions per minute"},"edata.eks.contentType":{"to":"Content Type"},"context.pdata.model":{"hidden":true},"dimensions.anonymous_user":{"to":"Logged in User"},"dimensions.did":{"to":"Device ID"},"dimensions.loc":{"to":"Lat / Long"},"mid":{"to":"Session ID"},"edata.eks.noOfInteractEvents":{"to":"Number of Interact events in the content session"},"context.date_range.to":{"hidden":true}}}},"ME_CONTENT_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/cus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_GENIE_LAUNCH_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/gls/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_ITEM_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/is/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_GENIE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/genie-launch-summ/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"uid":{"to":"UIDs"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"edata.eks.total_ts":{"to":"Total time (secs) per tag for the period"},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.contents":{"to":"Update col header\tContent IDs of all contents used per tag for the period"},"context.pdata.id":{"hidden":true},"edata.eks.avg_ts_session":{"to":"Average time spent per Genie session"},"edata.eks.device_ids":{"to":"Device IDs per tag for the period"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.tag":{"to":"Tag name"},"edata.eks.total_sessions":{"to":"Total number of sessions per tag for the period"},"mid":{"to":"Unique Genie Session ID"},"context.date_range.to":{"hidden":true},"dimensions.period":{"to":"period"}}}},"ME_ITEM_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/item-usage-summ/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_ASSET_SNAPSHOT_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/asset-snapshot/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.used_activities_count":{"to":"No. of activities that are part of lessons"},"edata.eks.used_templates_count":{"to":"No. of templates that are part of lessons"},"context.date_range.from":{"hidden":true},"uid":{"to":"uid"},"edata.eks.total_questions_count":{"to":"Total number of questions"},"ets":{"mapFunc":"timestampToDateTime","to":"Generation Time(IST)"},"edata.eks.used_audio_count":{"to":"No. of audio files that are being used"},"edata.eks.used_images_count":{"to":"No. of images that are being used"},"edata.eks.used_questions_count":{"to":"No. of questions that are being used"},"edata.eks.total_activities_count":{"to":"Total number of activities"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"edata.eks.total_templates_count":{"to":"Total no. of templates"},"dimensions.partner_id":{"to":"Partner ID"},"context.pdata.ver":{"hidden":true},"edata.eks.total_images_count":{"to":"Total no. of images"},"context.pdata.model":{"hidden":true},"edata.eks.total_audio_count":{"to":"Total no. of audio files"},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_CONTENT_SNAPSHOT_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/content-snapshot/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.active_user_count":{"to":"No. of active authors on the portal"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"dimensions.author_id":{"to":"Author ID"},"edata.eks.total_user_count":{"to":"Total number of registered authors on the portal"},"edata.eks.live_content_count":{"to":"Total number of content pieces live"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"edata.eks.total_content_count":{"to":"Total number of content pieces"},"edata.eks.review_content_count":{"to":"Total number of content pieces pending review"},"dimensions.partner_id":{"to":"Partner ID"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_DEVICE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/dus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.total_play_time":{"to":"Total Play time on device"},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Time stamp of first Genie installation on device(IST)"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"edata.eks.avg_num_launches":{"to":"Average num. of Genie launches per day"},"edata.eks.num_contents":{"to":"No. of content pieces available on device"},"edata.eks.play_start_time":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device first started game play(IST)"},"edata.eks.end_time":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device was last used(IST)"},"eid":{"hidden":true},"edata.eks.num_days":{"to":"Number of days since Genie installed before the timestamp indicating the last use"},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.mean_play_time":{"to":"Average play time on device"},"edata.eks.mean_play_time_interval":{"to":"Average time interval between game play on the device"},"edata.eks.last_played_on":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device was last used for game play(IST)"},"edata.eks.avg_time":{"to":"Average time on Genie per day"},"edata.eks.last_played_content":{"to":"Last played content on device"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.did":{"to":"Device ID"},"edata.eks.num_sessions":{"to":"Total number of sessions from the device"},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_GENIE_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/cus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.content":{"to":"Content IDs of content used in Genie Session"},"edata.eks.timeSpent":{"to":"Time spent on Genie Session\t"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"dimensions.group_user":{"to":"Is Group user?"},"edata.eks.contentCount":{"to":"Number of content pieces used"},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.time_stamp":{"mapFunc":"timestampToDateTime","to":"Session End time stamp(IST)"},"context.pdata.id":{"hidden":true},"tags":{"to":"Genie tags"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.anonymous_user":{"to":"Logged in user?"},"dimensions.did":{"to":"Device ID"},"mid":{"to":"Unique Genie Session ID"},"context.date_range.to":{"hidden":true}}}}}},"eks-consumption-metrics":{"events":["ME_CONTENT_USAGE_METRICS","ME_ITEM_USAGE_METRICS","ME_GENIE_USAGE_METRICS","ME_CONTENT_SNAPSHOT_METRICS","ME_CONCEPT_SNAPSHOT_METRICS","ME_ASSET_SNAPSHOT_METRICS"],"eventConfig":{"ME_CONTENT_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_content_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.content_id":{"to":"Content ID"},"dimensions.tag":{"to":"tag"},"edata.eks.m_publish_date":{"to":"Date when the content is published"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_total_sessions":{"to":"Total number of sessions"},"edata.eks.m_avg_ts_session":{"to":"Average timespent per session"},"edata.eks.m_total_interactions":{"to":"Total interactions count"},"edata.eks.m_avg_interactions_min":{"to":"Average interactions per minute"},"edata.eks.m_total_devices":{"to":"Total number of devices"},"edata.eks.m_avg_sess_device":{"to":"Average sessions per device"}}}},"ME_ITEM_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_item_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.tag":{"to":"tag"},"dimensions.content_id":{"to":"Content ID"},"dimensions.item_id":{"to":"item id"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_total_count":{"to":"Total response count"},"edata.eks.m_correct_res_count":{"to":"Correct response count"},"edata.eks.m_inc_res_count":{"to":"Incorrect response count"},"edata.eks.m_correct_res":{"hidden":true},"edata.eks.m_incorrect_res":{"hidden":true},"edata.eks.m_top5_incorrect_res":{"to":"Top 5 Incorrect response"},"edata.eks.m_avg_ts":{"to":"Average time spent"},"edata.eks.m_top5_misconception_res":{"to":"Top 5 Misconception response"}}}},"ME_GENIE_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_genie_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.tag":{"to":"Device Tag"},"edata.eks.m_total_sessions":{"to":"Total number of sessions"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_avg_ts_session":{"to":"Average sessions per device"},"edata.eks.m_contents":{"hidden":true},"edata.eks.m_total_devices":{"to":"Total number of devices"}}}},"ME_CONTENT_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_content_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_CONCEPT_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_concept_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_ASSET_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_asset_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}}}},"eks-creation-raw":{"events":["eks-creation-raw"],"eventConfig":{"eks-creation-raw":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"unique/"}},"filterMapping":{"channel":{"name":"context.channel","operator":"EQ"},"app_id":{"name":"context.pdata.id","operator":"EQ"}}}}},"eks-creation-summary":{"events":["ME_APP_SESSION_SUMMARY","ME_CE_SESSION_SUMMARY","ME_TEXTBOOK_SESSION_SUMMARY","ME_APP_USAGE_SUMMARY","ME_CE_USAGE_SUMMARY","ME_TEXTBOOK_USAGE_SUMMARY","ME_AUTHOR_USAGE_SUMMARY","ME_PUBLISH_PIPELINE_SUMMARY"],"eventConfig":{"ME_APP_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/app-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.interact_events_per_min":{"to":"Average ineractions per minute"},"edata.eks.events_summary":{"to":"edata.eks.events_summary"},"dimensions.app_id":{"to":"Event Source"},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Start Time(IST)"},"uid":{"to":"User ID"},"edata.eks.page_summary":{"to":"Pages visited Summary"},"ets":{"hidden":true},"edata.eks.page_views_count":{"to":"Total pageviews in Session"},"edata.eks.end_time":{"hidden":true},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.ce_visits":{"to":"Number of Authoring tool visits"},"dimensions.sid":{"to":"Session ID"},"edata.eks.time_diff":{"to":"Session length "},"edata.eks.interact_events_count":{"to":"Number of Interactions"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.time_spent":{"to":"Session length"},"dimensions.anonymous_user":{"to":"User Logged In"},"mid":{"to":"mid"},"edata.eks.env_summary":{"to":"Usage summary"},"edata.eks.first_visit":{"to":"First Time Visitor"},"context.date_range.to":{"hidden":true}}}},"ME_CE_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/ce-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_TEXTBOOK_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/textbook-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_APP_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/app-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"edata.eks.avg_ts_session":{"to":"Average session duration per day"},"context.granularity":{"to":"Granularity"},"edata.eks.anon_total_sessions":{"to":"Time spent by non-registered users"},"edata.eks.unique_users":{"to":"Unique users list"},"dimensions.app_id":{"to":"Source"},"edata.eks.unique_users_count":{"to":"Unique users count"},"context.date_range.from":{"hidden":true},"edata.eks.total_pageviews_count":{"to":"Total Page Views per day"},"uid":{"to":"uid"},"edata.eks.ce_total_sessions":{"to":"Number of sessions where creation happened"},"edata.eks.anon_avg_ts_session":{"to":"Avg. session duration for non-registered users per day"},"ets":{"mapFunc":"timestampToDateTime","to":"Generation Time(IST)"},"edata.eks.total_ts":{"to":"Total time spent by visitors on Portal"},"edata.eks.ce_percent_sessions":{"to":"Percentage of visitors creating content"},"dimensions.author_id":{"to":"Author ID"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.new_user_count":{"to":"Number of new users"},"syncts":{"to":"Event sync time(IST)","mapFunc":"timestampToDateTime"},"edata.eks.percent_new_users_count":{"to":"Percentage of new visitors per day"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.avg_pageviews":{"to":"Avg. pages per visit per day"},"edata.eks.total_sessions":{"to":"Total number of sessions"},"edata.eks.anon_total_ts":{"to":"Total time spent by non-registered users on Portal"},"mid":{"hidden":true},"dimensions.period":{"to":"dimensions.period"},"context.date_range.to":{"hidden":true}}}},"ME_CE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/ce-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_TEXTBOOK_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/textbook-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_AUTHOR_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/author-usage-summary/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"uid":{"to":"Author ID"},"mid":{"hidden":true},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"eid":{"hidden":true},"ver":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.id":{"hidden":true},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.avg_ts_session":{"to":"Total number of seesion per day"},"edata.eks.total_ts":{"to":"Total time spent by author"},"edata.eks.ce_total_ts":{"to":"Total time spent on content editor by author"},"edata.eks.total_sessions":{"to":"No of sessions per Author"},"context.granularity":{"hidden":true},"edata.eks.ce_percent_ts":{"to":"Percentage of time spent browsing vs creating"},"edata.eks.ce_percent_sessions":{"to":"Percentage of sessions in which users log into content editor"},"edata.eks.ce_total_visits":{"to":"No of Content Editor visits per author"},"context.date_range.from":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_PUBLISH_PIPELINE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/publish-pipeline-summ/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"uid":{"hidden":true},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.publish_pipeline_summary":{"to":"edata.eks.publish_pipeline_summary"},"syncts":{"to":"Event sync time(IST)","mapFunc":"timestampToDateTime"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"mid":{"hidden":true},"dimensions.period":{"to":"Date"},"context.date_range.to":{"hidden":true}}}}}},"eks-creation-metrics":{"events":["ME_APP_USAGE_METRICS","ME_CE_USAGE_METRICS","ME_TEXTBOOK_CREATION_METRICS","ME_TEXTBOOK_SNAPSHOT_METRICS","ME_AUTHOR_USAGE_METRICS"],"eventConfig":{"ME_APP_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_app_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.author_id":{"to":"Author ID / All"},"dimensions.app_id":{"to":"App (EkStep portal currently)"},"edata.eks.anon_total_sessions":{"to":"Total anonymous sessions on portal"},"edata.eks.anon_total_ts":{"to":"Tota time spent (Anonymous) on portal"},"edata.eks.total_sessions":{"to":"Total number of portal sessions"},"edata.eks.total_ts":{"to":"Total time spent on portal"},"edata.eks.ce_total_sessions":{"to":"Total content editor sessions"},"edata.eks.ce_percent_sessions":{"to":"% of total sessions that are content editor sessions"},"edata.eks.total_pageviews_count":{"to":"Total pageviews on portal"},"edata.eks.unique_users":{"hidden":true},"edata.eks.unique_users_count":{"to":"Count of unique users on portal"},"edata.eks.avg_pageviews":{"to":"Avg. number of pageviews per session for specified period"},"edata.eks.avg_ts_session":{"to":"Avg. session length (time spent) per session for specified period"},"edata.eks.anon_avg_ts_session":{"to":"Avg. Anonymous session length (time spent) per session for specified period"},"edata.eks.new_user_count":{"to":"No. of new users on the app (portal) for the given time period"},"edata.eks.percent_new_users_count":{"to":"% of new users for the given time period"}}}},"ME_CE_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_ce_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.content_id":{"to":"Content ID"},"edata.eks.unique_users_count":{"to":"Number of users who accessed the content piece via the editor"},"edata.eks.total_sessions":{"to":"Total number of sessions that accessed the content piece via the editor"},"edata.eks.total_ts":{"to":"Time spent on the content piece in the content editor"},"edata.eks.avg_ts_session":{"to":"Average session time on the editor"},"edata.eks.update_date":{"to":"Last updated timestamp of the content piece"}}}},"ME_TEXTBOOK_CREATION_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_textbook_creation_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_TEXTBOOK_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_textbook_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_AUTHOR_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_author_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":" Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.author_id":{"to":"Author ID /All"},"edata.eks.total_sessions":{"to":"Total number of portal sessions by author"},"edata.eks.total_ts":{"to":"Total time spent on portal by author"},"edata.eks.total_ce_ts":{"to":"Total time spent on content editor by author"},"edata.eks.total_ce_visit":{"to":"Number of content editor visits"},"edata.eks.percent_ce_sessions":{"to":"% of sessions that were spent on the editor"},"edata.eks.avg_ts_session":{"to":"Average time spent by author on portal per session"},"edata.eks.percent_ce_ts":{"to":"% time spent by author on content editor"}}}}}}}}' - ;; - "monitor-job-summ") - #echo '{"search":{"type":"local","queries":[{"file":"/mnt/data/analytics/scripts/logs/joblog.log"}]},"model":"org.ekstep.analytics.model.MonitorSummaryModel","modelParams":{"model":[{"model":"ItemSummaryModel","category":"consumption","input_dependency":"LearnerSessionSummaryModel"},{"model":"GenieUsageSummaryModel","category":"consumption","input_dependency":"GenieLaunchSummaryModel"},{"model":"GenieStageSummaryModel","category":"consumption","input_dependency":"GenieLaunchSummaryModel"},{"model":"ItemUsageSummaryModel","category":"consumption","input_dependency":"ItemSummaryModel"},{"model":"DeviceContentUsageSummaryModel","category":"consumption","input_dependency":"LearnerSessionSummaryModel"},{"model":"DeviceUsageSummaryModel","category":"consumption","input_dependency":"GenieLaunchSummaryModel"},{"model":"UpdateGenieUsageDB","category":"consumption","input_dependency":"GenieUsageSummaryModel"},{"model":"UpdateItemSummaryDB","category":"consumption","input_dependency":"ItemUsageSummaryModel"},{"model":"UpdateContentUsageDB","category":"consumption","input_dependency":"ContentUsageSummaryModel"},{"model":"UpdateContentPopularityDB","category":"consumption","input_dependency":"ContentPopularitySummaryModel"},{"model":"ContentUsageSummaryModel","category":"consumption","input_dependency":"LearnerSessionSummaryModel"},{"model":"UpdateLearnerProfileDB","category":"consumption","input_dependency":"None"},{"model":"UpdateDeviceSpecificationDB","category":"consumption","input_dependency":"None"},{"model":"LearnerSessionSummaryModel","category":"consumption","input_dependency":"None"},{"model":"GenieSessionSummaryModel","category":"consumption","input_dependency":"None"},{"model":"GenieLaunchSummaryModel","category":"consumption","input_dependency":"None"},{"model":"ContentPopularitySummaryModel","category":"consumption","input_dependency":"None"},{"model":"EOCRecommendationFunnelModel","category":"consumption","input_dependency":"None"},{"model":"StageSummaryModel","category":"consumption","input_dependency":"None"},{"model":"GenieFunnelAggregatorModel","category":"consumption","input_dependency":"None"},{"model":"ContentSideloadingSummaryModel","category":"consumption","input_dependency":"None"},{"model":"GenieFunnelModel","category":"consumption","input_dependency":"None"},{"model":"UpdateContentModel","category":"consumption","input_dependency":"None"},{"model":"ConsumptionMetricsUpdater","category":"consumption","input_dependency":"None"},{"model":"PrecomputedViews","category":"consumption","input_dependency":"None"},{"model":"DataExhaustJob","category":"consumption","input_dependency":"None"},{"model":"AppUsageSummaryModel","category":"creation","input_dependency":"AppSessionSummaryModel"},{"model":"ContentEditorUsageSummaryModel","category":"creation","input_dependency":"ContentEditorSessionSummaryModel"},{"model":"TextbookUsageSummaryModel","category":"creation","input_dependency":"TextbookSessionSummaryModel"},{"model":"UpdateAppUsageDB","category":"creation","input_dependency":"AppUsageSummaryModel"},{"model":"UpdateContentEditorUsageDB","category":"creation","input_dependency":"ContentEditorUsageSummaryModel"},{"model":"UpdateTextbookUsageDB","category":"creation","input_dependency":"TextbookUsageSummaryModel"},{"model":"UpdateAuthorSummaryDB","category":"creation","input_dependency":"AuthorUsageSummaryModel"},{"model":"UpdatePublishPipelineSummarycreation","category":"creation","input_dependency":"PublishPipelineSummaryModel"},{"model":"AppSessionSummaryModel","category":"creation","input_dependency":"None"},{"model":"ContentEditorSessionSummaryModel","category":"creation","input_dependency":"None"},{"model":"PublishPipelineSummaryModel","category":"creation","input_dependency":"None"},{"model":"UpdateObjectLifecycleDB","category":"creation","input_dependency":"None"},{"model":"UpdateContentCreationMetricsDB","category":"creation","input_dependency":"None"},{"model":"AuthorUsageSummaryModel","category":"creation","input_dependency":"None"},{"model":"TextbookSessionSummaryModel","category":"creation","input_dependency":"None"},{"model":"UpdateCreationMetricsDB","category":"creation","input_dependency":"None"},{"model":"UpdateConceptSnapshotDB","category":"creation","input_dependency":"None"},{"model":"ContentSnapshotSummaryModel","category":"creation","input_dependency":"None"},{"model":"ConceptSnapshotSummaryModel","category":"creation","input_dependency":"None"},{"model":"UpdateContentSnapshotDB","category":"creation","input_dependency":"None"},{"model":"AssetSnapshotSummaryModel","category":"creation","input_dependency":"None"},{"model":"UpdateTextbookSnapshotDB","category":"creation","input_dependency":"None"},{"model":"UpdateAssetSnapshotDB","category":"creation","input_dependency":"None"},{"model":"ContentLanguageRelationModel","category":"creation","input_dependency":"None"},{"model":"ConceptLanguageRelationModel","category":"creation","input_dependency":"None"},{"model":"AuthorRelationsModel","category":"creation","input_dependency":"None"},{"model":"CreationRecommendationEnrichmentModel","category":"creation","input_dependency":"None"},{"model":"ContentAssetRelationModel","category":"creation","input_dependency":"None"},{"model":"DeviceRecommendationTrainingModel","category":"recommendation","input_dependency":"None"},{"model":"DeviceRecommendationScoringModel","category":"recommendation","input_dependency":"None"},{"model":"ContentVectorsModel","category":"recommendation","input_dependency":"None"},{"model":"EndOfContentRecommendationModel","category":"recommendation","input_dependency":"None"},{"model":"CreationRecommendationModel","category":"recommendation","input_dependency":"None"},{"model":"UpdatePluginSnapshotDB","category":"creation","input_dependency":"None"},{"model":"UpdateTemplateSnapshotDB","category":"creation","input_dependency":"None"},{"model":"AppUsageMetricCreationModel","category":"creation","input_dependency":"None"},{"model":"AssetSnapshotMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"CEUsageMetricCreationModel","category":"creation","input_dependency":"None"},{"model":"ConceptSnapshotMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"ContentSnapshotMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"ContentUsageMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"GenieUsageMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"ItemUsageMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"TextbookSnapshotMetricCreationModel","category":"creation","input_dependency":"None"},{"model":"TextbookUsageMetricCreationModel","category":"creation","input_dependency":"None"},{"model":"AuthorUsageMetricCreationModel","category":"creation","input_dependency":"None"}]},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"appName":"TestMonitorSummarizer","deviceMapping":true}' - echo '{"search":{"type":"local","queries":[{"file":"'$analytics_home'/scripts/logs/joblog.log"}]},"model":"org.ekstep.analytics.model.MonitorSummaryModel","modelParams":{"pushMetrics":true,"brokerList":"'$brokerList'","topic":"'$analyticsMetricsTopic'","model":[{"model":"DeviceSummaryModel","category":"consumption","input_dependency":"None"},{"model":"UpdateDeviceProfileDB","category":"consumption","input_dependency":"DeviceSummaryModel"},{"model":"DataExhaustJob","category":"consumption","input_dependency":"None"},{"model":"UpdateWorkFlowUsageDB","category":"consumption","input_dependency":"WorkFlowUsageSummaryModel"},{"model":"WorkFlowSummaryModel","category":"consumption","input_dependency":"None"},{"model":"WorkFlowUsageSummaryModel","category":"consumption","input_dependency":"WorkFlowSummaryModel"},{"model":"UpdatePortalMetrics","category":"consumption","input_dependency":"None"},{"model":"UpdateWorkFlowUsageMetricsModel","category":"consumption","input_dependency":"None"},{"model":"CourseMetricsJob","category":"consumption","input_dependency":"None"},{"model":"UpdateContentRating","category":"consumption","input_dependency":"None"},{"model":"DruidQueryProcessingModel","category":"consumption","input_dependency":"None"},{"model":"DistrictMonthlyJob","category":"consumption","input_dependency":"None"},{"model":"DesktopConsumptionDailyMetricsJob","category":"consumption","input_dependency":"None"},{"model":"DistrictWeeklyJob","category":"consumption","input_dependency":"None"},{"model":"ConsumptionDailyMetricsJob","category":"consumption","input_dependency":"None"},{"model":"MetricsAuditJob","category":"consumption","input_dependency":"None"},{"model":"AssessmentMetricsJob","category":"consumption","input_dependency":"None"},{"model":"StateAdminReportJob","category":"consumption","input_dependency":"None"},{"model":"StateAdminGeoReportJob","category":"consumption","input_dependency":"None"},{"model": "CourseEnrollmentJob","category": "consumption","input_dependency": "None"}]},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"appName":"TestMonitorSummarizer","deviceMapping":true}' - ;; - "job-manager") - echo '{"jobsCount":'$jobManagerJobsCount',"topic":"'$job_topic'","bootStrapServer":"'$brokerList'","zookeeperConnect":"'$zookeeper'","consumerGroup":"jobmanager","slackChannel":"#test_channel","slackUserName":"JobManager","tempBucket":"'$bucket'","tempFolder":"'$temp_folder'"}' - ;; "wfs") - echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"apiVersion":"v2", "parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}' - #echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":8,"appName":"Workflow Summarizer","deviceMapping":true}' - ;; - "wfus") - echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowUsageSummary","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":8,"appName":"Workflow Usage Summarizer","deviceMapping":false}' - ;; - "wfu") - echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfus/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.updater.UpdateWorkFlowUsageDB","output":[{"to":"console","params":{"printEvent":false}}],"parallelization":10,"appName":"Workflow Usage Updater","deviceMapping":false}' - ;; - "ds") - echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0},{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.DeviceSummaryModel","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":12,"appName":"Device Summarizer","deviceMapping":false}' - ;; - "dpu") - echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/device-summary/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.UpdateDeviceProfileDB","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}}],"parallelization":8,"appName":"Device Profile Updater","deviceMapping":false}' - ;; - "video-streaming") - echo '{"search":{"type":"azure"},"model":"org.ekstep.analytics.job.VideoStreamingJob","modelParams":{"maxIterations":10},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Video Streaming Job","deviceMapping":false}' + echo '{"search":{"type":"gcloud","queries":[{"bucket":"'$bucket'","prefix":"{{ dp_raw_telemetry_backup_location }}","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "apiVersion":"v2", "parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'","compression":"{{ dashboards_broker_compression }}"} }],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}' ;; - "course-dashboard-metrics") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CourseMetricsJob","modelParams":{"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"channel","aliasName":"channel"}],"filters":[{"type":"equals","dimension":"contentType","value":"Course"},{"type":"isnotnull","dimension":"identifier","value":""},{"type":"isnotnull","dimension":"channel","value":""}],"descending":"false"},"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Course Dashboard Metrics","deviceMapping":false}' - ;; - "assessment-dashboard-metrics") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.AssessmentMetricsJob","modelParams":{"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"channel","aliasName":"channel"}],"filters":[{"type":"equals","dimension":"contentType","value":"Course"},{"type":"isnotnull","dimension":"identifier","value":""},{"type":"isnotnull","dimension":"channel","value":""}],"descending":"false"},"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Assessment Dashboard Metrics","deviceMapping":false}' - ;; - "admin-user-reports") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminReportJob","modelParams":{"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}' - ;; - "admin-geo-reports") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminGeoReportJob","modelParams":{"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin Geo Reports","deviceMapping":false}' - ;; - "telemetry-replay") - echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"'$inputBucket'","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"TelemetryReplayJob","deviceMapping":false}' - ;; - "summary-replay") - echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"SummaryReplayJob","deviceMapping":false}' - ;; - "content-rating-updater") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.updater.UpdateContentRating","modelParams":{"startDate":"'$endDate'","endDate":"'$endDate'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Content Rating Updater","deviceMapping":false}' - ;; - "experiment") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.ExperimentDefinitionModel","modelParams":{"sparkElasticsearchConnectionHost":"{{ lp_composite_search_host }}"},"output":[{"to":"elasticsearch","params":{"index":"experiment"}}],"parallelization":8,"appName":"Experiment-Definition","deviceMapping":false}' - ;; - "district-monthly") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"reportConfig":{"id":"district_level_report","queryType":"groupBy","dateRange":{"staticInterval":"LastMonth","granularity":"all"},"metrics":[{"metric":"totalUniqueDevices","label":"Total Unique Devices","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastMonth","aggregations":[{"name":"total_unique_devices","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.portal","'$producerEnv'.app"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}}],"labels":{"state":"State","district":"District","total_unique_devices":"Number of Unique Devices"},"output":[{"type":"csv","metrics":["total_unique_devices"],"dims":["state","date"],"fileParameters":["id","dims"],"locationMapping":true}],"mergeConfig":{"frequency":"MONTH","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"district_report.csv","postContainer":"'$reportPostContainer'"}},"store":"azure","container":"'$bucket'","key":"druid-reports/"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Druid Query Processor","deviceMapping":false}' - ;; - "district-weekly") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"reportConfig":{"id":"district_weekly_report","queryType":"groupBy","dateRange":{"staticInterval":"LastWeek","granularity":"all"},"metrics":[{"metric":"totalUniqueDevices","label":"Total Unique Devices","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastWeek","aggregations":[{"name":"total_unique_devices","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.portal","'$producerEnv'.app"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalUniqueDevicesOnPortal","label":"Total Unique Devices on Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastWeek","aggregations":[{"name":"total_unique_devices_on_portal","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalUniqueDevicesOnApp","label":"Total Unique Devices on App","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastWeek","aggregations":[{"name":"total_unique_devices_on_app","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalQRScans","label":"Total QR Scans","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_scans","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"context_pdata_id","values":["'$producerEnv'.portal","'$producerEnv'.app"]},{"type":"equals","dimension":"eid","value":"SEARCH"},{"type":"in","dimension":"object_type","values":["qr","Qr","DialCode","dialcode"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalQRScansOnPortal","label":"Total QR Scans on Portal","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_scans_on_portal","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"context_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"eid","value":"SEARCH"},{"type":"in","dimension":"object_type","values":["qr","Qr","DialCode","dialcode"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalQRScansOnApp","label":"Total QR Scans on App","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_scans_on_app","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"context_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"eid","value":"SEARCH"},{"type":"in","dimension":"object_type","values":["qr","Qr","DialCode","dialcode"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalContentPlays","label":"Total Content Plays","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_content_plays","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.portal","'$producerEnv'.app"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalContentPlaysOnPortal","label":"Total Content Plays on Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_content_plays_on_portal","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalContentPlaysOnApp","label":"Total Content Plays on App","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_content_plays_on_app","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}}],"labels":{"state":"State","district":"District","total_unique_devices":"Total Unique Devices","total_unique_devices_on_portal":"Total Unique Devices on Portal","total_unique_devices_on_app":"Total Unique Devices on App","total_scans":"Total QR Scans","total_scans_on_portal":"Total QR Scans on Portal","total_scans_on_app":"Total QR Scans on App","total_content_plays":"Total Content Plays","total_content_plays_on_portal":"Total Content Plays on Portal","total_content_plays_on_app":"Total Content Plays on App"},"output":[{"type":"csv","metrics":["total_unique_devices_on_app","total_unique_devices_on_portal","total_unique_devices","total_scans_on_app","total_scans_on_portal","total_scans","total_content_plays_on_app","total_content_plays_on_portal","total_content_plays"],"dims":["state","date"],"fileParameters":["id","dims"],"locationMapping":true}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":1,"rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":1,"reportPath":"district_weekly.csv","postContainer":"'$reportPostContainer'"}},"store":"azure","container":"'$bucket'","key":"druid-reports/"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Druid Query Processor","deviceMapping":false}' - ;; - "etb-metrics") - echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.model.report.ETBMetricsJob","modelParams": {"reportConfig": {"id": "etb_metrics","metrics": [],"labels": {"date": "Date","identifier": "TextBook ID","name": "TextBook Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","totalQRCodes": "Total number of QR codes","contentLinkedQR": "Number of QR codes with atleast 1 linked content","withoutContentQR": "Number of QR codes with no linked content","withoutContentT1": "Term 1 QR Codes with no linked content","withoutContentT2": "Term 2 QR Codes with no linked content","status": "Status","totalContentLinked": "Total content linked","totalQRLinked": "Total QR codes linked to content","totalQRNotLinked": "Total number of QR codes with no linked content","leafNodesCount": "Total number of leaf nodes","leafNodeUnlinked": "Number of leaf nodes with no content","l1Name": "Level 1 Name","l2Name": "Level 2 Name","l3Name": "Level 3 Name","l4Name": "Level 4 Name","l5Name": "Level 5 Name","dialcode": "QR Code","sum(scans)": "Total Scans","noOfContent": "Number of contents","nodeType": "Type of Node","term": "Term"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}],"mergeConfig": {"frequency": "WEEK","basePath": "'$baseScriptPath'","rollup": 0,"reportPath": "dialcode_counts.csv","postContainer":"'$reportPostContainer'"}},"dialcodeReportConfig": {"id": "etb_metrics","metrics": [],"labels": {},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}],"mergeConfig": {"frequency": "WEEK","basePath": "'$baseScriptPath'","rollup": 1,"reportPath": "dialcode_counts.csv","rollupAge": "ACADEMIC_YEAR","rollupCol": "Date","rollupRange": 10,"postContainer":"'$reportPostContainer'"}},"etbFileConfig": {"bucket": "'$reportPostContainer'","file": "dialcode_scans/dialcode_counts.csv"},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "equals","dimension": "contentType","value": "TextBook"}, {"type": "in","dimension": "status","values": ["Live", "Draft", "Review"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"tenantConfig": {"tenantId": "","slugName": ""},"store": "azure","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$bucket'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "ETB Metrics Model","deviceMapping": false}' - ;; - "daily-metrics") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"reportConfig":{"id":"ETB-Consumption-Daily-Reports","queryType":"groupBy","dateRange":{"staticInterval":"LastDay","granularity":"all"},"metrics":[{"metric":"totalPercentFailedScans","label":"Total Percent Failed QR Scans","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-rollup-syncts","intervals":"LastDay","aggregations":[{"type":"filtered","filterAggType":"longSum","fieldName":"total_count","filterFieldName":"edata_size","filterValue":0,"name":"total_failed_scans"},{"type":"filtered","filterAggType":"longSum","fieldName":"total_count","filterFieldName":"edata_size","filterValue":1,"name":"total_successful_scans"},{"name":"total_scans","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"dialcode_channel","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"object_type","values":["DialCode","dialcode","qr","Qr"]},{"type":"equals","dimension":"eid","value":"SEARCH"}],"postAggregation":[{"type":"javascript","name":"total_percent_failed_scans","fields":{"leftField":"total_failed_scans","rightField":"total_scans","rightFieldType":"FieldAccess"},"fn":"function(total_failed_scans, total_scans) { return (total_scans > 0) ? (total_failed_scans/total_scans) * 100 : 0 }"}],"descending":"false"}},{"metric":"totalContentDownload","label":"Total Content Download","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-events-syncts","intervals":"LastDay","aggregations":[{"name":"total_content_download","type":"count","fieldName":""}],"dimensions":[{"fieldName":"content_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"edata_subtype","value":"ContentDownload-Success"},{"type":"equals","dimension":"eid","value":"INTERACT"},{"type":"in","dimension":"context_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"object_type","value":"Resource"}],"descending":"false"}},{"metric":"totalContentPlayed","label":"Total Content Played","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"total_content_plays","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalContentPlayedOnApp","label":"Total Content Played On App","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"total_content_plays_on_app","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalContentPlayedOnPortal","label":"Total Content Played On Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"total_content_plays_on_portal","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalUniqueDevices","label":"Total Unique Devices","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"total_unique_devices","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"app"}],"descending":"false"}},{"metric":"totalUniqueDevicesOnApp","label":"Total Unique Devices On App","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"total_unique_devices_on_app","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"app"}],"descending":"false"}},{"metric":"totalUniqueDevicesOnPortal","label":"Total Unique Devices On Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"total_unique_devices_on_portal","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"app"}],"descending":"false"}},{"metric":"totalDevicesPlayingContent","label":"Total Devices Playing Content","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"total_devices_playing_content","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalDevicesPlayingContentOnApp","label":"Total Devices Playing Content On App","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"devices_playing_content_on_app","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalDevicesPlayingContentOnPortal","label":"Total Devices Playing Content On Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"devices_playing_content_on_portal","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalContentPlayedInHour","label":"Total Content Played In Hour","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"total_time_spent"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"postAggregation":[{"type":"javascript","name":"total_time_spent_in_hours","fields":{"leftField":"sum__edata_time_spent","rightField":"sum__edata_time_spent","rightFieldType":"FieldAccess"},"fn":"function(delta, time_spent) { return Math.round(delta/3600) }"}],"descending":"false"}},{"metric":"totalContentPlayedInHourOnApp","label":"Total Content Played In Hour On App","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"total_time_spent"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"postAggregation":[{"type":"javascript","name":"total_time_spent_in_hours_on_app","fields":{"leftField":"sum__edata_time_spent","rightField":"sum__edata_time_spent","rightFieldType":"FieldAccess"},"fn":"function(delta, time_spent) { return Math.round(delta/3600) }"}],"descending":"false"}},{"metric":"totalContentPlayedInHourOnPortal","label":"Total Content Played In Hour On Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"total_time_spent"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"postAggregation":[{"type":"javascript","name":"total_time_spent_in_hours_on_portal","fields":{"leftField":"sum__edata_time_spent","rightField":"sum__edata_time_spent","rightFieldType":"FieldAccess"},"fn":"function(delta, time_spent) { return Math.round(delta/3600) }"}],"descending":"false"}}],"labels":{"state":"State","total_content_download":"Total Content Downloads","total_successful_scans":"Number of successful QR Scans","total_percent_failed_scans":"Total Percent Failed Scans","total_unique_devices":"Total Unique Devices","total_unique_devices_on_app":"Total Unique Devices On App","total_unique_devices_on_portal":"Total Unique Devices On Portal","total_time_spent_in_hours":"Total Content Play Time (in hours)","total_time_spent_in_hours_on_app":"Total Content Play Time on App (in hours)","total_time_spent_in_hours_on_portal":"Total Content Play Time on Portal (in hours)","total_failed_scans":"Number of failed QR Scans","total_scans":"Number of QR Scans","total_content_plays":"Total Content Plays","total_content_plays_on_app":"Total Content Plays On App","total_content_plays_on_portal":"Total Content Plays On Portal","total_app_sessions":"Total App Sesions","time_spent_on_app_in_hours":"Total Time on App (in hours)","devices_playing_content_on_app":"Total Devices that Played Content on App","devices_playing_content_on_portal":"Total Devices that Played Content on Portal","total_devices_playing_content":"Total Devices that Played Content"},"output":[{"type":"csv","metrics":["total_scans","total_successful_scans","total_failed_scans","total_percent_failed_scans","total_content_download","total_content_plays_on_app","devices_playing_content_on_app","total_time_spent_in_hours_on_app","total_content_plays_on_portal","devices_playing_content_on_portal","total_time_spent_in_hours_on_portal","total_content_plays","total_devices_playing_content","total_time_spent_in_hours","total_unique_devices_on_app","total_unique_devices_on_portal","total_unique_devices"],"dims":["state","date"],"fileParameters":["id","dims"],"locationMapping":false}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":1,"rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":1,"reportPath":"daily_metrics.csv","postContainer":"'$reportPostContainer'"}},"store":"azure","container":"'$bucket'","key":"druid-reports/"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Druid Query Processor","deviceMapping":false}' - ;; - "desktop-consumption-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"reportConfig":{"id":"Desktop-Consumption-Daily-Reports","queryType":"groupBy","dateRange":{"staticInterval":"LastDay","granularity":"day"},"metrics":[{"metric":"totalContentDownloadDesktop","label":"Total Content Download","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-events-syncts","intervals":"LastDay","granularity":"all","aggregations":[{"name":"total_content_download_on_desktop","type":"count","fieldName":"mid"}],"dimensions":[{"fieldName":"content_board","aliasName":"state"}],"filters":[{"type":"equals","dimension":"context_env","value":"downloadManager"},{"type":"equals","dimension":"edata_state","value":"COMPLETED"},{"type":"equals","dimension":"context_pdata_id","value":"'$producerEnv'.desktop"},{"type":"equals","dimension":"eid","value":"AUDIT"}],"descending":"false"}},{"metric":"totalContentPlayedDesktop","label":"Total time spent in hours","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","granularity":"all","aggregations":[{"name":"total_content_plays_on_desktop","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"collection_board","aliasName":"state"}],"filters":[{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.desktop"}],"descending":"false"}},{"metric":"totalContentPlayedInHourOnDesktop","label":"Total Content Download","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","granularity":"all","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"edata_time_spent"}],"dimensions":[{"fieldName":"collection_board","aliasName":"state"}],"filters":[{"type":"equals","dimension":"eid","value":"ME_WORKFLOW_SUMMARY"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.desktop"}],"postAggregation":[{"type":"arithmetic","name":"total_time_spent_in_hours_on_desktop","fields":{"leftField":"sum__edata_time_spent","rightField":3600,"rightFieldType":"constant"},"fn":"/"}],"descending":"false"}},{"metric":"totalUniqueDevicesPlayedContentOnDesktop","label":"Total Unique Devices On Desktop that played content","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","granularity":"all","aggregations":[{"name":"total_unique_devices_on_desktop_played_content","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_board","aliasName":"state"}],"filters":[{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.desktop"}],"descending":"false"}}],"labels":{"state":"State","total_content_plays_on_desktop":"Total Content Played","total_content_download_on_desktop":"Total Content Downloads","total_time_spent_in_hours_on_desktop":"Total time spent in hours","total_unique_devices_on_desktop_played_content":"Total Unique Devices On Desktop that played content"},"output":[{"type":"csv","label":"desktop","metrics":["total_content_download_on_desktop","total_time_spent_in_hours_on_desktop","total_content_plays_on_desktop","total_unique_devices_on_desktop_played_content"],"dims":["state","date"],"fileParameters":["dims"],"locationMapping":false}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":1,"rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":1,"reportPath":"desktop_consumption_report.csv","postContainer":"'$reportPostContainer'"}},"store":"azure","container":"'$bucket'","key":"druid-reports/"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Druid Query Processor","deviceMapping":false}' - ;; - "course-enrollment-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"azure","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}' - ;; - "course-consumption-report") - echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.model.report.CourseConsumptionJob","modelParams": {"esConfig": {"request": {"filters": {"objectType": ["Content"],"contentType": ["Course"],"identifier": [],"status": ["Live"]}}},"reportConfig": {"id": "tpd_metrics","labels": {"date": "Date","status": "Batch Status","timespent": "Timespent in mins","courseName": "Course Name","batchName": "Batch Name"},"dateRange": {"staticInterval": "LastDay","granularity": "all"},"metrics": [{"metric": "totalCoursePlays","label": "Total Course Plays (in mins)","druidQuery": {"queryType": "groupBy","dataSource": "summary-events","intervals":"LastDay","aggregations": [{"name": "sum__edata_time_spent","type": "doubleSum","fieldName": "edata_time_spent"}],"dimensions": [{"fieldName": "object_rollup_l1","aliasName": "courseId"}, {"fieldName": "uid","aliasName": "userId"}, {"fieldName": "context_cdata_id","aliasName": "batchId"}],"filters": [{"type": "equals","dimension": "eid","value": "ME_WORKFLOW_SUMMARY"}, {"type": "in","dimension": "dimensions_pdata_id","values": ["'$producerEnv'.app", "'$producerEnv'.portal"]}, {"type": "equals","dimension": "dimensions_type","value": "content"}, {"type": "equals","dimension": "dimensions_mode","value": "play"}, {"type": "equals","dimension": "context_cdata_type","value": "batch"}],"postAggregation": [{"type": "arithmetic","name": "timespent","fields": {"leftField": "sum__edata_time_spent","rightField": 60,"rightFieldType": "constant"},"fn": "/"}],"descending": "false"}}],"output": [{"type": "csv","metrics": ["timespent"],"dims": []}],"queryType": "groupBy"},"store": "azure","format":"csv","key": "druid-reports/","filePath": "druid-reports/","container":"'$bucket'","folderPrefix": ["slug", "reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "TPD Course Consumption Metrics Model","deviceMapping": false}' - ;; - "textbook-progress-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.TextBookProgressModel","modelParams":{"reportConfig":{"id":"content_progress_metrics","metrics":[],"labels":{"board":"Board","medium":"Medium","gradeLevel":"Grade","subject":"Subject","resourceType":"Content Type","totalContent": "Total Contents","live":"Live","review":"Review","draft":"Draft","unlisted":"Limited Sharing","application_ecml":"Created on Diksha","video_youtube":"YouTube Content","video_mp4":"Uploaded Videos","application_pdf":"Text Content","application_html":"Uploaded Interactive Content","identifier":"Content ID","creator":"Created By","createdOn":"Creation Date","lastPublishDate":"Last Publish Date","status":"Status","pkgVersion":"Number of times Published","lastPublishedOn":"Pending in current status since","pendingInCurrentStatus":"Pending in current status since"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"content_progress_metrics.csv","postContainer":"'$reportPostContainer'"}},"filter":{"tenantId":"","slugName":""},"store":"azure","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Progress Metrics Model","deviceMapping":false}' - ;; - "audit-metrics-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.MetricsAuditJob","modelParams":{"auditConfig":[{"name":"denorm","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"telemetry-denormalized/raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]},"filters":[{"name":"flags.user_data_retrieved","operator":"EQ","value":true},{"name":"flags.content_data_retrieved","operator":"EQ","value":true},{"name":"flags.device_data_retrieved","operator":"EQ","value":true},{"name":"flags.dialcode_data_retrieved","operator":"EQ","value":true},{"name":"flags.collection_data_retrieved","operator":"EQ","value":true},{"name":"flags.derived_location_retrieved","operator":"EQ","value":true}]},{"name":"failed","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"failed/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"unique","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"raw","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"channel-raw","search":{"type":"azure","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/raw/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"channel-summary","search":{"type":"azure","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/summary/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"derived","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"telemetry-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"telemetry-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}},{"name":"summary-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}}]},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$metricsTopic'"}}],"parallelization":8,"appName":"Metrics Audit"}' - ;; - "*") + "survey-nps") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.survey.nps.NpsJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserFeedKeyspace":"{{ dashboards_cassandra_user_feed_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraUserFeedTable":"{{ dashboards_cassandra_user_feed_table }}","mongoDatabase":"{{ dashboards_mongo_discussions_db }}","mongoDBCollection":"{{ dashboards_mongo_discussions_db_collection }}","platformRatingSurveyId":"{{ dashboards_platform_rating_survey_id }}","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Survey Nps","deviceMapping":false}' + ;; + "ministry-learner-leaderboard") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.acbp.MinistryLearnerLeaderboardJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraAcbpTable":"{{ dashboards_cassandra_acbp_table }}","cassandraKarmaPointsLookupTable":"{{ dashboards_cassandra_karma_points_lookup_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraHallOfFameTable":"{{ dashboards_cassandra_mdo_karma_points_table }}","cassandraMDOLearnerLeaderboardTable":"{{ dashboards_cassandra_mdo_learner_leaderboard_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","commsConsoleReportPath":"{{ comms_console_report_path }}","acbpReportPath":"{{ acbp_report_path }}","acbpMdoEnrolmentReportPath":"{{ acbp_mdo_enrolment_report_path }}","acbpMdoSummaryReportPath":"{{ acbp_mdo_summary_report_path }}","commsConsolePrarambhEmailSuffix":"{{ comms_console_prarambh_email_suffix }}","commsConsoleNumDaysToConsider":"{{ comms_console_num_days_to_consider }}","commsConsoleNumTopLearnersToConsider":"{{ comms_console_num_top_learners_to_consider }}","commsConsolePrarambhTags":"{{ comms_console_prarambh_tags }}","commsConsolePrarambhCbpIds":"{{ comms_console_prarambh_cbp_ids }}","commsConsolePrarambhNCount":"{{ comms_console_prarambh_n_count }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}","acbpEnrolment":"{{ dashboards_acbp_enrolment_topic }}"}}},"output":[],"parallelization":16,"appName":"Ministry Learner Leaderboard Job","deviceMapping":false}' + ;; + "ministry-metrics") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.acbp.MinistryMetricsJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraAcbpTable":"{{ dashboards_cassandra_acbp_table }}","cassandraKarmaPointsLookupTable":"{{ dashboards_cassandra_karma_points_lookup_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraHallOfFameTable":"{{ dashboards_cassandra_mdo_karma_points_table }}","cassandraMDOLearnerLeaderboardTable":"{{ dashboards_cassandra_mdo_learner_leaderboard_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","commsConsoleReportPath":"{{ comms_console_report_path }}","acbpReportPath":"{{ acbp_report_path }}","acbpMdoEnrolmentReportPath":"{{ acbp_mdo_enrolment_report_path }}","acbpMdoSummaryReportPath":"{{ acbp_mdo_summary_report_path }}","commsConsolePrarambhEmailSuffix":"{{ comms_console_prarambh_email_suffix }}","commsConsoleNumDaysToConsider":"{{ comms_console_num_days_to_consider }}","commsConsoleNumTopLearnersToConsider":"{{ comms_console_num_top_learners_to_consider }}","commsConsolePrarambhTags":"{{ comms_console_prarambh_tags }}","commsConsolePrarambhCbpIds":"{{ comms_console_prarambh_cbp_ids }}","commsConsolePrarambhNCount":"{{ comms_console_prarambh_n_count }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}","acbpEnrolment":"{{ dashboards_acbp_enrolment_topic }}"}}},"output":[],"parallelization":16,"appName":"Ministry Metrics Job","deviceMapping":false}' + ;; + "data-exhaust") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.exhaust.DataExhaustJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraOldAssesmentTable":"{{ dashboards_cassandra_old_assessment_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraUserEntityEnrolmentTable":"{{ dashboards_cassandra_user_entity_enrolment_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraFrameworkHierarchyTable":"{{ dashboards_cassandra_framework_hierarchy_table }}","cassandraLearnerLeaderBoardLookupTable":"{{ dashboards_cassandra_learner_leaderboard_lookup_table }}","cassandraLearnerLeaderBoardTable":"{{ dashboards_cassandra_learner_leaderboard_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraAcbpTable":"{{ dashboards_cassandra_acbp_table }}","cassandraKarmaPointsSummaryTable":"{{ dashboards_cassandra_karma_points_summary_table}}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Data Exhaust","deviceMapping":false}' + ;; + "dashboard-sync") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.DashboardSyncJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","platformRatingSurveyId":"{{ dashboards_platform_rating_survey_id }}","nationalLearningWeekStart":"{{ national_learning_week_start }}","nationalLearningWeekEnd":"{{ national_learning_week_end }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraPublicUserAssessmentDataTable":"{{ dashboards_cassandra_public_userassessmenttable }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","anonymousAssessmentLoggedInUserContentIDs":"'$anonymousAssessmentLoggedInUserContentIDs'","anonymousAssessmentNonLoggedInUserAssessmentIDs":"'$anonymousAssessmentNonLoggedInUserAssessmentIDs'","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Dashboard Sync","deviceMapping":false}' + ;; + "summary-redis-sync") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.telemetry.SummaryRedisSyncJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Summary Redis Sync Job","deviceMapping":false}' + ;; + "assessment-metrics") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.assess.UserAssessmentJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Assessment Metrics","deviceMapping":false}' + ;; + "user-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.user.UserReportJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"User Report","deviceMapping":false}' + ;; + "user-enrolment-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.enrolment.UserEnrolmentJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","acbpReportPath":"{{ acbp_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":8,"appName":"User Enrolment Report","deviceMapping":false}' + ;; + "user-activity") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.activity.user.UserActivityJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","acbpReportPath":"{{ acbp_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":8,"appName":"User Enrolment Report","deviceMapping":false}' + ;; + "course-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.course.CourseReportJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwContentResourceTable":"content_resource","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Course Report","deviceMapping":false}' + ;; + "course-based-assessment-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.cba.CourseBasedAssessmentJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Course Based Assessment Report","deviceMapping":false}' + ;; + "blended-program-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.blended.BlendedProgramReportJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Blended Program Report","deviceMapping":false}' + ;; + "data-warehouse-sync") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.warehouse.DataWarehouseJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEventsTable":"events","dwEventsEnrolmentTable":"events_enrolment","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","dwContentResourceTable":"content_resource","dwKcmDictionaryTable":"kcm_dictionary","dwKcmContentTable":"kcm_content_mapping","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","kcmReportPath":"{{ kcm_report_path }}","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","acbpReportPath":"{{ acbp_report_path }}","dwCBPlanTable":"cb_plan","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Data Warehouse Sync","deviceMapping":false}' + ;; + "weekly-claps") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.weekly.claps.WeeklyClapsJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Weekly Claps","deviceMapping":false}' + ;; + "comms-console") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.commsconsole.CommsReportJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","commsConsoleReportPath":"{{ comms_console_report_path }}","commsConsolePrarambhEmailSuffix":"{{ comms_console_prarambh_email_suffix }}","commsConsoleNumDaysToConsider":"{{ comms_console_num_days_to_consider }}","commsConsoleNumTopLearnersToConsider":"{{ comms_console_num_top_learners_to_consider }}","commsConsolePrarambhTags":"{{ comms_console_prarambh_tags }}","commsConsolePrarambhCbpIds":"{{ comms_console_prarambh_cbp_ids }}","commsConsolePrarambhNCount":"{{ comms_console_prarambh_n_count }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Comms Console Report","deviceMapping":false}' + ;; + "karma-points") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.karma.points.KarmaPointsJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraAcbpTable":"{{ dashboards_cassandra_acbp_table }}","cassandraKarmaPointsLookupTable":"{{ dashboards_cassandra_karma_points_lookup_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraHallOfFameTable":"{{ dashboards_cassandra_mdo_karma_points_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","commsConsoleReportPath":"{{ comms_console_report_path }}","acbpReportPath":"{{ acbp_report_path }}","acbpMdoEnrolmentReportPath":"{{ acbp_mdo_enrolment_report_path }}","acbpMdoSummaryReportPath":"{{ acbp_mdo_summary_report_path }}","commsConsolePrarambhEmailSuffix":"{{ comms_console_prarambh_email_suffix }}","commsConsoleNumDaysToConsider":"{{ comms_console_num_days_to_consider }}","commsConsoleNumTopLearnersToConsider":"{{ comms_console_num_top_learners_to_consider }}","commsConsolePrarambhTags":"{{ comms_console_prarambh_tags }}","commsConsolePrarambhCbpIds":"{{ comms_console_prarambh_cbp_ids }}","commsConsolePrarambhNCount":"{{ comms_console_prarambh_n_count }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}","acbpEnrolment":"{{ dashboards_acbp_enrolment_topic }}"}}},"output":[],"parallelization":16,"appName":"Karma Points Job","deviceMapping":false}' + ;; + "acbp-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.acbp.UserACBPReportJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraAcbpTable":"{{ dashboards_cassandra_acbp_table }}","cassandraKarmaPointsLookupTable":"{{ dashboards_cassandra_karma_points_lookup_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraHallOfFameTable":"{{ dashboards_cassandra_mdo_karma_points_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwCBPlanTable":"cb_plan","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","commsConsoleReportPath":"{{ comms_console_report_path }}","acbpReportPath":"{{ acbp_report_path }}","acbpMdoEnrolmentReportPath":"{{ acbp_mdo_enrolment_report_path }}","acbpMdoSummaryReportPath":"{{ acbp_mdo_summary_report_path }}","commsConsolePrarambhEmailSuffix":"{{ comms_console_prarambh_email_suffix }}","commsConsoleNumDaysToConsider":"{{ comms_console_num_days_to_consider }}","commsConsoleNumTopLearnersToConsider":"{{ comms_console_num_top_learners_to_consider }}","commsConsolePrarambhTags":"{{ comms_console_prarambh_tags }}","commsConsolePrarambhCbpIds":"{{ comms_console_prarambh_cbp_ids }}","commsConsolePrarambhNCount":"{{ comms_console_prarambh_n_count }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}","acbpEnrolment":"{{ dashboards_acbp_enrolment_topic }}"}}},"output":[],"parallelization":16,"appName":"ACBP Report Job","deviceMapping":false}' + ;; + "kcm-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.kcm.KCMJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","postgresCompetencyTable":"data_node","postgresCompetencyHierarchyTable":"node_mapping","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwKcmDictionaryTable":"kcm_dictionary","dwKcmContentTable":"kcm_content_mapping","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","cutoffTime":"60.0","kcmReportPath":"{{ kcm_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Data Warehouse Sync","deviceMapping":false}' + ;; + "survey-question-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.survey.question.QuestionReportJob","modelParams":{"debug":"false","validation":"false","solutionIDs":"'$reportSolutionIDs'","gracePeriod":"{{ report_grace_period }}","baseUrlForEvidences":"{{ report_base_url_for_evidences }}","mlReportPath":"{{ ml_report_path }}","includeExpiredSolutionIDs":"{{ report_include_expired_solution_ids }}","mlSparkDruidRouterHost":"{{ ml_druid_router_host }}","mlSparkMongoConnectionHost":"{{ ml_mongo_host }}","mlMongoDatabase":"{{ dashboards_mongo_survey_observation_db }}","surveyCollection":"{{ dashboards_mongo_survey_collection }}","reportConfigCollection":"{{ dashboards_mongo_report_config_collection }}","reportSyncEnable":"{{ operational_sl_report_sync_enable }}","SurveyQuestionReportBatchSize":"{{ survey_question_report_batch_size }}","SurveyStatusReportBatchSize":"{{ survey_status_report_batch_size }}","ObservationQuestionReportBatchSize":"{{ observation_question_report_batch_size }}","ObservationStatusReportBatchSize":"{{ observation_status_report_batch_size }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Survey Question Report","deviceMapping":false}' + ;; + "survey-status-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.survey.status.StatusReportJob","modelParams":{"debug":"false","validation":"false","solutionIDs":"'$reportSolutionIDs'","gracePeriod":"{{ report_grace_period }}","baseUrlForEvidences":"{{ report_base_url_for_evidences }}","mlReportPath":"{{ ml_report_path }}","includeExpiredSolutionIDs":"{{ report_include_expired_solution_ids }}","mlSparkDruidRouterHost":"{{ ml_druid_router_host }}","mlSparkMongoConnectionHost":"{{ ml_mongo_host }}","mlMongoDatabase":"{{ dashboards_mongo_survey_observation_db }}","surveyCollection":"{{ dashboards_mongo_survey_collection }}","reportConfigCollection":"{{ dashboards_mongo_report_config_collection }}","reportSyncEnable":"{{ operational_sl_report_sync_enable }}","SurveyQuestionReportBatchSize":"{{ survey_question_report_batch_size }}","SurveyStatusReportBatchSize":"{{ survey_status_report_batch_size }}","ObservationQuestionReportBatchSize":"{{ observation_question_report_batch_size }}","ObservationStatusReportBatchSize":"{{ observation_status_report_batch_size }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Survey Status Report","deviceMapping":false}' + ;; + "observation-question-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.observation.question.ObservationQuestionReportJob","modelParams":{"debug":"false","validation":"false","solutionIDs":"'$reportSolutionIDs'","gracePeriod":"{{ report_grace_period }}","baseUrlForEvidences":"{{ report_base_url_for_evidences }}","mlReportPath":"{{ ml_report_path }}","includeExpiredSolutionIDs":"{{ report_include_expired_solution_ids }}","mlSparkDruidRouterHost":"{{ ml_druid_router_host }}","mlSparkMongoConnectionHost":"{{ ml_mongo_host }}","mlMongoDatabase":"{{ dashboards_mongo_survey_observation_db }}","surveyCollection":"{{ dashboards_mongo_survey_collection }}","reportConfigCollection":"{{ dashboards_mongo_report_config_collection }}","reportSyncEnable":"{{ operational_sl_report_sync_enable }}","SurveyQuestionReportBatchSize":"{{ survey_question_report_batch_size }}","SurveyStatusReportBatchSize":"{{ survey_status_report_batch_size }}","ObservationQuestionReportBatchSize":"{{ observation_question_report_batch_size }}","ObservationStatusReportBatchSize":"{{ observation_status_report_batch_size }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Observation Question Report","deviceMapping":false}' + ;; + "observation-status-report") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.observation.status.ObservationStatusReportJob","modelParams":{"debug":"false","validation":"false","solutionIDs":"'$reportSolutionIDs'","gracePeriod":"{{ report_grace_period }}","baseUrlForEvidences":"{{ report_base_url_for_evidences }}","mlReportPath":"{{ ml_report_path }}","includeExpiredSolutionIDs":"{{ report_include_expired_solution_ids }}","mlSparkDruidRouterHost":"{{ ml_druid_router_host }}","mlSparkMongoConnectionHost":"{{ ml_mongo_host }}","mlMongoDatabase":"{{ dashboards_mongo_survey_observation_db }}","surveyCollection":"{{ dashboards_mongo_survey_collection }}","reportConfigCollection":"{{ dashboards_mongo_report_config_collection }}","reportSyncEnable":"{{ operational_sl_report_sync_enable }}","SurveyQuestionReportBatchSize":"{{ survey_question_report_batch_size }}","SurveyStatusReportBatchSize":"{{ survey_status_report_batch_size }}","ObservationQuestionReportBatchSize":"{{ observation_question_report_batch_size }}","ObservationStatusReportBatchSize":"{{ observation_status_report_batch_size }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Observation Status Report","deviceMapping":false}' + ;; + "in-app-review-feed") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.survey.inappreview.InAppReviewJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserFeedKeyspace":"{{ dashboards_cassandra_user_feed_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraUserFeedTable":"{{ dashboards_cassandra_user_feed_table }}","mongoDatabase":"{{ dashboards_mongo_discussions_db }}","mongoDBCollection":"{{ dashboards_mongo_discussions_db_collection }}","platformRatingSurveyId":"{{ dashboards_platform_rating_survey_id }}","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"In App Review Trigger Job","deviceMapping":false}' + ;; + "zip-upload") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.zipreports.ZipReportsWithSecurityJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_zip_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","postgresCompetencyTable":"data_node","postgresCompetencyHierarchyTable":"node_mapping","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEventsTable":"events","dwEventsEnrolmentTable":"events_enrolment","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","dwContentResourceTable":"content_resource","dwKcmDictionaryTable":"kcm_dictionary","dwKcmContentTable":"kcm_content_mapping","dwCBPlanTable":"cb_plan","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","cutoffTime":"60.0","prefixDirectoryPath":"{{ zip_reports_prefix_directory_path }}","destinationDirectoryPath":"{{ zip_reports_destination_path }}","directoriesToSelect":"{{ directories_to_select }}","password":"{{ zip_reports_password }}","kcmReportPath":"{{ kcm_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Zip Upload Job","deviceMapping":false}' + ;; + "learner-leaderboard") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.leaderboard.LearnerLeaderBoardJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraLearnerLeaderBoardLookupTable":"{{ dashboards_cassandra_learner_leaderboard_lookup_table }}","cassandraLearnerLeaderBoardTable":"{{ dashboards_cassandra_learner_leaderboard_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraAcbpTable":"{{ dashboards_cassandra_acbp_table }}","cassandraKarmaPointsLookupTable":"{{ dashboards_cassandra_karma_points_lookup_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraHallOfFameTable":"{{ dashboards_cassandra_mdo_karma_points_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","commsConsoleReportPath":"{{ comms_console_report_path }}","acbpReportPath":"{{ acbp_report_path }}","acbpMdoEnrolmentReportPath":"{{ acbp_mdo_enrolment_report_path }}","acbpMdoSummaryReportPath":"{{ acbp_mdo_summary_report_path }}","commsConsolePrarambhEmailSuffix":"{{ comms_console_prarambh_email_suffix }}","commsConsoleNumDaysToConsider":"{{ comms_console_num_days_to_consider }}","commsConsoleNumTopLearnersToConsider":"{{ comms_console_num_top_learners_to_consider }}","commsConsolePrarambhTags":"{{ comms_console_prarambh_tags }}","commsConsolePrarambhCbpIds":"{{ comms_console_prarambh_cbp_ids }}","commsConsolePrarambhNCount":"{{ comms_console_prarambh_n_count }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}","acbpEnrolment":"{{ dashboards_acbp_enrolment_topic }}"}}},"output":[],"parallelization":16,"appName":"Learner Leaderboard Job","deviceMapping":false}' + ;; + "national-learning-week") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.karma.points.KarmaPointsJob","modelParams":{"debug":"false","validation":"false", "stateLearningWeekStart":"{{ state_learning_week_start }}","stateLearningWeekEnd":"{{ state_learning_week_end }}","cassandraSLWMdoLeaderboardTable":"{{ dashboards_cassandra_slw_mdo_leaderboard_table }}","cassandraSLWMdoTopLearnerTable":"{{ dashboards_cassandra_slw_mdo_top_learners_table }}","nationalLearningWeekStart":"{{ national_learning_week_start }}","nationalLearningWeekEnd":"{{ national_learning_week_end }}","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraAcbpTable":"{{ dashboards_cassandra_acbp_table }}","cassandraKarmaPointsLookupTable":"{{ dashboards_cassandra_karma_points_lookup_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraNLWMdoLeaderboardTable":"{{ dashboards_cassandra_nlw_mdo_table }}", "cassandraNLWUserLeaderboardTable":"{{ dashboards_cassandra_nlw_user_table }}","cassandraHallOfFameTable":"{{ dashboards_cassandra_mdo_karma_points_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","commsConsoleReportPath":"{{ comms_console_report_path }}","acbpReportPath":"{{ acbp_report_path }}","acbpMdoEnrolmentReportPath":"{{ acbp_mdo_enrolment_report_path }}","acbpMdoSummaryReportPath":"{{ acbp_mdo_summary_report_path }}","commsConsolePrarambhEmailSuffix":"{{ comms_console_prarambh_email_suffix }}","commsConsoleNumDaysToConsider":"{{ comms_console_num_days_to_consider }}","commsConsoleNumTopLearnersToConsider":"{{ comms_console_num_top_learners_to_consider }}","commsConsolePrarambhTags":"{{ comms_console_prarambh_tags }}","commsConsolePrarambhCbpIds":"{{ comms_console_prarambh_cbp_ids }}","commsConsolePrarambhNCount":"{{ comms_console_prarambh_n_count }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}","acbpEnrolment":"{{ dashboards_acbp_enrolment_topic }}"}}},"output":[],"parallelization":16,"appName":"Karma Points Job","deviceMapping":false}' + ;; + "monthly-requests") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard..report.monthly.requests.MonthlyRequestJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraPublicUserAssessmentDataTable":"{{ dashboards_cassandra_public_userassessmenttable }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","anonymousAssessmentLoggedInUserContentIDs":"'$anonymousAssessmentLoggedInUserContentIDs'","anonymousAssessmentNonLoggedInUserAssessmentIDs":"'$anonymousAssessmentNonLoggedInUserAssessmentIDs'","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Monthly Requests","deviceMapping":false}' + ;; + "bq-data-push") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.bq.BqDataModel","modelParams":{"debug":"false","validation":"false","nationalLearningWeekStart":"{{ national_learning_week_start }}","nationalLearningWeekEnd":"{{ national_learning_week_end }}","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraAcbpTable":"{{ dashboards_cassandra_acbp_table }}","cassandraKarmaPointsLookupTable":"{{ dashboards_cassandra_karma_points_lookup_table }}","cassandraKarmaPointsTable":"{{ dashboards_cassandra_karma_points_table }}","cassandraNLWMdoLeaderboardTable":"{{ dashboards_cassandra_nlw_mdo_table }}","cassandraNLWUserLeaderboardTable":"{{ dashboards_cassandra_nlw_user_table }}","cassandraHallOfFameTable":"{{ dashboards_cassandra_mdo_karma_points_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","commsConsoleReportPath":"{{ comms_console_report_path }}","acbpReportPath":"{{ acbp_report_path }}","acbpMdoEnrolmentReportPath":"{{ acbp_mdo_enrolment_report_path }}","acbpMdoSummaryReportPath":"{{ acbp_mdo_summary_report_path }}","commsConsolePrarambhEmailSuffix":"{{ comms_console_prarambh_email_suffix }}","commsConsoleNumDaysToConsider":"{{ comms_console_num_days_to_consider }}","commsConsoleNumTopLearnersToConsider":"{{ comms_console_num_top_learners_to_consider }}","commsConsolePrarambhTags":"{{ comms_console_prarambh_tags }}","commsConsolePrarambhCbpIds":"{{ comms_console_prarambh_cbp_ids }}","commsConsolePrarambhNCount":"{{ comms_console_prarambh_n_count }}","bqScriptPath":"{{ bq_script_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}","acbpEnrolment":"{{ dashboards_acbp_enrolment_topic }}"}}},"output":[],"parallelization":16,"appName":"Karma Points Job","deviceMapping":false}' + ;; + "odcs-recomendation") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.odcs.OdcsRecomendationJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraFrameworkHierarchyTable":"{{ dashboards_cassandra_framework_hierarchy_table }}","cassandraGroupDesignationTable":"{{ dashboards_cassandra_group_designation_table }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEventsTable":"events","dwEventsEnrolmentTable":"events_enrolment","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","dwContentResourceTable":"content_resource","dwKcmDictionaryTable":"kcm_dictionary","dwKcmContentTable":"kcm_content_mapping","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","kcmReportPath":"{{ kcm_report_path }}","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","acbpReportPath":"{{ acbp_report_path }}","dwCBPlanTable":"cb_plan","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Data Warehouse Sync","deviceMapping":false}' + ;; + "es-form-data") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.esformdata.EsFormDataJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","sparkElasticsearchConnectionPort":"{{ single_node_es_port }}","esFormDataIndex":"{{ es_form_data_index }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraFrameworkHierarchyTable":"{{ dashboards_cassandra_framework_hierarchy_table }}","cassandraGroupDesignationTable":"{{ dashboards_cassandra_group_designation_table }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","appOrgHierarchyTable":"org_hierarchy_v4","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwUserTable":"user_detail","dwCourseTable":"content","dwEventsTable":"events","dwEventsEnrolmentTable":"events_enrolment","dwEnrollmentsTable":"user_enrolments","dwOrgTable":"org_hierarchy","dwAssessmentTable":"assessment_detail","dwBPEnrollmentsTable":"bp_enrolments","dwContentResourceTable":"content_resource","dwKcmDictionaryTable":"kcm_dictionary","dwKcmContentTable":"kcm_content_mapping","key":"storage.key.config","secret":"storage.secret.config","store":"{{ report_storage_type }}","container":"{{ cloud_storage_bucket_reports }}","mdoIDs":"'$reportMDOIDs'","cutoffTime":"60.0","kcmReportPath":"{{ kcm_report_path }}","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","blendedReportPath":"{{ blended_report_path }}","orgHierarchyReportPath":"{{ org_hierarchy_report_path }}","acbpReportPath":"{{ acbp_report_path }}","esFormDataIds":"{{ es_form_data_ids }}","dwCBPlanTable":"cb_plan","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"ES Form Data Job","deviceMapping":false}' + ;; + "*") echo "Unknown model code" exit 1 # Command to come out of the program with status 1 ;; esac -} +} \ No newline at end of file diff --git a/ansible/roles/data-products-deploy/templates/model-config.json.j2 b/ansible/roles/data-products-deploy/templates/model-config.json.j2 index 09c7312231..4594a1978a 100644 --- a/ansible/roles/data-products-deploy/templates/model-config.json.j2 +++ b/ansible/roles/data-products-deploy/templates/model-config.json.j2 @@ -1,1860 +1,11 @@ { - "data-exhaust": { - "search": { - "type": "azure" - }, - "model": "org.ekstep.analytics.model.DataExhaustJobModel", - "parallelization": 8, - "appName": "Data Exhaust", - "deviceMapping": false, - "modelParams": { - "shouldDelay": true, - "delayInMilis": 1800000 - }, - "exhaustConfig": { - "eks-consumption-raw": { - "events": [ - "eks-consumption-raw" - ], - "eventConfig": { - "eks-consumption-raw": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "channel/" - } - }, - "filterMapping": { - "tags": { - "name": "tags", - "operator": "IN" - }, - "channel": { - "name": "context.channel", - "operator": "EQ" - }, - "app_id": { - "name": "context.pdata.id", - "operator": "EQ" - } - } - } - } - }, - "eks-consumption-summary": { - "events": [ - "ME_SESSION_SUMMARY", - "ME_CONTENT_USAGE_SUMMARY", - "ME_GENIE_LAUNCH_SUMMARY", - "ME_ITEM_SUMMARY", - "ME_GENIE_USAGE_SUMMARY", - "ME_ITEM_USAGE_SUMMARY", - "ME_ASSET_SNAPSHOT_SUMMARY", - "ME_CONTENT_SNAPSHOT_SUMMARY", - "ME_DEVICE_USAGE_SUMMARY", - "ME_GENIE_SESSION_SUMMARY", - "ME_WORKFLOW_SUMMARY" - ], - "eventConfig": { - "ME_WORKFLOW_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/wfs/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "dimensions.channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - } - }, - "ME_SESSION_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/ss/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "context.granularity": { - "hidden": true - }, - "edata.eks.timeSpent": { - "to": "Time spent on the content" - }, - "edata.eks.screenSummary": { - "hidden": true - }, - "edata.eks.currentLevel": { - "to": "Current Domain Level" - }, - "edata.eks.syncDate": { - "mapFunc": "timestampToDateTime", - "to": "Sync Date(IST)" - }, - "edata.eks.timeDiff": { - "hidden": true - }, - "context.date_range.from": { - "hidden": true - }, - "edata.eks.start_time": { - "mapFunc": "timestampToDateTime", - "to": "Content Start Time(IST)" - }, - "uid": { - "to": "Genie User ID" - }, - "ets": { - "hidden": true - }, - "dimensions.group_user": { - "to": "Is Group User" - }, - "edata.eks.activitySummary": { - "to": "Activity Summary" - }, - "edata.eks.end_time": { - "hidden": true - }, - "edata.eks.eventsSummary": { - "hidden": true - }, - "eid": { - "hidden": true - }, - "ver": { - "hidden": true - }, - "edata.eks.levels": { - "hidden": true - }, - "context.pdata.id": { - "hidden": true - }, - "edata.eks.itemResponses": { - "to": "edata.eks.itemResponses" - }, - "dimensions.gdata.id": { - "to": "Content ID" - }, - "edata.eks.noOfLevelTransitions": { - "to": "Number of Level Transitions" - }, - "edata.eks.noOfAttempts": { - "to": "Number of Attempts" - }, - "tags": { - "to": "Genie Tags" - }, - "edata.eks.interruptTime": { - "to": "Total interrupt time" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Sync time stamp(IST)" - }, - "edata.eks.mimeType": { - "hidden": true - }, - "edata.eks.telemetryVersion": { - "to": "Telemetry Version" - }, - "context.pdata.ver": { - "hidden": true - }, - "dimensions.gdata.ver": { - "hidden": true - }, - "edata.eks.interactEventsPerMin": { - "to": "Number of interactions per minute" - }, - "edata.eks.contentType": { - "to": "Content Type" - }, - "context.pdata.model": { - "hidden": true - }, - "dimensions.anonymous_user": { - "to": "Logged in User" - }, - "dimensions.did": { - "to": "Device ID" - }, - "dimensions.loc": { - "to": "Lat / Long" - }, - "mid": { - "to": "Session ID" - }, - "edata.eks.noOfInteractEvents": { - "to": "Number of Interact events in the content session" - }, - "context.date_range.to": { - "hidden": true - } - } - } - }, - "ME_CONTENT_USAGE_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/cus/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": false, - "columnMappings": {} - } - }, - "ME_GENIE_LAUNCH_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/gls/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": false, - "columnMappings": {} - } - }, - "ME_ITEM_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/is/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": false, - "columnMappings": {} - } - }, - "ME_GENIE_USAGE_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/genie-launch-summ/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "context.granularity": { - "hidden": true - }, - "context.date_range.from": { - "hidden": true - }, - "uid": { - "to": "UIDs" - }, - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation time(IST)" - }, - "edata.eks.total_ts": { - "to": "Total time (secs) per tag for the period" - }, - "eid": { - "hidden": true - }, - "ver": { - "hidden": true - }, - "edata.eks.contents": { - "to": "Update col header\tContent IDs of all contents used per tag for the period" - }, - "context.pdata.id": { - "hidden": true - }, - "edata.eks.avg_ts_session": { - "to": "Average time spent per Genie session" - }, - "edata.eks.device_ids": { - "to": "Device IDs per tag for the period" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Sync time(IST)" - }, - "context.pdata.ver": { - "hidden": true - }, - "context.pdata.model": { - "hidden": true - }, - "dimensions.tag": { - "to": "Tag name" - }, - "edata.eks.total_sessions": { - "to": "Total number of sessions per tag for the period" - }, - "mid": { - "to": "Unique Genie Session ID" - }, - "context.date_range.to": { - "hidden": true - }, - "dimensions.period": { - "to": "period" - } - } - } - }, - "ME_ITEM_USAGE_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/item-usage-summ/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": false, - "columnMappings": {} - } - }, - "ME_ASSET_SNAPSHOT_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/asset-snapshot/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "context.granularity": { - "hidden": true - }, - "edata.eks.used_activities_count": { - "to": "No. of activities that are part of lessons" - }, - "edata.eks.used_templates_count": { - "to": "No. of templates that are part of lessons" - }, - "context.date_range.from": { - "hidden": true - }, - "uid": { - "to": "uid" - }, - "edata.eks.total_questions_count": { - "to": "Total number of questions" - }, - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Generation Time(IST)" - }, - "edata.eks.used_audio_count": { - "to": "No. of audio files that are being used" - }, - "edata.eks.used_images_count": { - "to": "No. of images that are being used" - }, - "edata.eks.used_questions_count": { - "to": "No. of questions that are being used" - }, - "edata.eks.total_activities_count": { - "to": "Total number of activities" - }, - "eid": { - "hidden": true - }, - "ver": { - "hidden": true - }, - "context.pdata.id": { - "hidden": true - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Sync time(IST)" - }, - "edata.eks.total_templates_count": { - "to": "Total no. of templates" - }, - "dimensions.partner_id": { - "to": "Partner ID" - }, - "context.pdata.ver": { - "hidden": true - }, - "edata.eks.total_images_count": { - "to": "Total no. of images" - }, - "context.pdata.model": { - "hidden": true - }, - "edata.eks.total_audio_count": { - "to": "Total no. of audio files" - }, - "mid": { - "hidden": true - }, - "context.date_range.to": { - "hidden": true - } - } - } - }, - "ME_CONTENT_SNAPSHOT_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/content-snapshot/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "context.granularity": { - "hidden": true - }, - "context.date_range.from": { - "hidden": true - }, - "edata.eks.active_user_count": { - "to": "No. of active authors on the portal" - }, - "uid": { - "to": "uid" - }, - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation time(IST)" - }, - "dimensions.author_id": { - "to": "Author ID" - }, - "edata.eks.total_user_count": { - "to": "Total number of registered authors on the portal" - }, - "edata.eks.live_content_count": { - "to": "Total number of content pieces live" - }, - "eid": { - "hidden": true - }, - "ver": { - "hidden": true - }, - "context.pdata.id": { - "hidden": true - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Sync time(IST)" - }, - "edata.eks.total_content_count": { - "to": "Total number of content pieces" - }, - "edata.eks.review_content_count": { - "to": "Total number of content pieces pending review" - }, - "dimensions.partner_id": { - "to": "Partner ID" - }, - "context.pdata.ver": { - "hidden": true - }, - "context.pdata.model": { - "hidden": true - }, - "mid": { - "hidden": true - }, - "context.date_range.to": { - "hidden": true - } - } - } - }, - "ME_DEVICE_USAGE_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/dus/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "context.granularity": { - "hidden": true - }, - "edata.eks.total_play_time": { - "to": "Total Play time on device" - }, - "context.date_range.from": { - "hidden": true - }, - "edata.eks.start_time": { - "mapFunc": "timestampToDateTime", - "to": "Time stamp of first Genie installation on device(IST)" - }, - "uid": { - "to": "uid" - }, - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation time(IST)" - }, - "edata.eks.avg_num_launches": { - "to": "Average num. of Genie launches per day" - }, - "edata.eks.num_contents": { - "to": "No. of content pieces available on device" - }, - "edata.eks.play_start_time": { - "mapFunc": "timestampToDateTime", - "to": "Time stamp when the device first started game play(IST)" - }, - "edata.eks.end_time": { - "mapFunc": "timestampToDateTime", - "to": "Time stamp when the device was last used(IST)" - }, - "eid": { - "hidden": true - }, - "edata.eks.num_days": { - "to": "Number of days since Genie installed before the timestamp indicating the last use" - }, - "ver": { - "hidden": true - }, - "context.pdata.id": { - "hidden": true - }, - "edata.eks.mean_play_time": { - "to": "Average play time on device" - }, - "edata.eks.mean_play_time_interval": { - "to": "Average time interval between game play on the device" - }, - "edata.eks.last_played_on": { - "mapFunc": "timestampToDateTime", - "to": "Time stamp when the device was last used for game play(IST)" - }, - "edata.eks.avg_time": { - "to": "Average time on Genie per day" - }, - "edata.eks.last_played_content": { - "to": "Last played content on device" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Sync time(IST)" - }, - "context.pdata.ver": { - "hidden": true - }, - "context.pdata.model": { - "hidden": true - }, - "dimensions.did": { - "to": "Device ID" - }, - "edata.eks.num_sessions": { - "to": "Total number of sessions from the device" - }, - "mid": { - "hidden": true - }, - "context.date_range.to": { - "hidden": true - } - } - } - }, - "ME_GENIE_SESSION_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/cus/" - } - }, - "filterMapping": { - "tags": { - "name": "genieTag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "context.granularity": { - "hidden": true - }, - "context.date_range.from": { - "hidden": true - }, - "edata.eks.content": { - "to": "Content IDs of content used in Genie Session" - }, - "edata.eks.timeSpent": { - "to": "Time spent on Genie Session\t" - }, - "uid": { - "to": "uid" - }, - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation time(IST)" - }, - "dimensions.group_user": { - "to": "Is Group user?" - }, - "edata.eks.contentCount": { - "to": "Number of content pieces used" - }, - "eid": { - "hidden": true - }, - "ver": { - "hidden": true - }, - "edata.eks.time_stamp": { - "mapFunc": "timestampToDateTime", - "to": "Session End time stamp(IST)" - }, - "context.pdata.id": { - "hidden": true - }, - "tags": { - "to": "Genie tags" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Sync time(IST)" - }, - "context.pdata.ver": { - "hidden": true - }, - "context.pdata.model": { - "hidden": true - }, - "dimensions.anonymous_user": { - "to": "Logged in user?" - }, - "dimensions.did": { - "to": "Device ID" - }, - "mid": { - "to": "Unique Genie Session ID" - }, - "context.date_range.to": { - "hidden": true - } - } - } - } - } - }, - "eks-consumption-metrics": { - "events": [ - "ME_CONTENT_USAGE_METRICS", - "ME_ITEM_USAGE_METRICS", - "ME_GENIE_USAGE_METRICS", - "ME_CONTENT_SNAPSHOT_METRICS", - "ME_CONCEPT_SNAPSHOT_METRICS", - "ME_ASSET_SNAPSHOT_METRICS" - ], - "eventConfig": { - "ME_CONTENT_USAGE_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-consumption-metrics/me_content_usage_metrics/" - } - }, - "filterMapping": { - "tags": { - "name": "dimensions.tag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation Time(IST)" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Event sync time(IST)" - }, - "eid": { - "hidden": true - }, - "dimensions.period": { - "to": "Time period of the summary. For ex: Day or Week or Month represented by a number" - }, - "dimensions.content_id": { - "to": "Content ID" - }, - "dimensions.tag": { - "to": "tag" - }, - "edata.eks.m_publish_date": { - "to": "Date when the content is published" - }, - "edata.eks.m_total_ts": { - "to": "Total time spent" - }, - "edata.eks.m_total_sessions": { - "to": "Total number of sessions" - }, - "edata.eks.m_avg_ts_session": { - "to": "Average timespent per session" - }, - "edata.eks.m_total_interactions": { - "to": "Total interactions count" - }, - "edata.eks.m_avg_interactions_min": { - "to": "Average interactions per minute" - }, - "edata.eks.m_total_devices": { - "to": "Total number of devices" - }, - "edata.eks.m_avg_sess_device": { - "to": "Average sessions per device" - } - } - } - }, - "ME_ITEM_USAGE_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-consumption-metrics/me_item_usage_metrics/" - } - }, - "filterMapping": { - "tags": { - "name": "dimensions.tag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation Time(IST)" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Event sync time(IST)" - }, - "eid": { - "hidden": true - }, - "dimensions.period": { - "to": "Time period of the summary. For ex: Day or Week or Month represented by a number" - }, - "dimensions.tag": { - "to": "tag" - }, - "dimensions.content_id": { - "to": "Content ID" - }, - "dimensions.item_id": { - "to": "item id" - }, - "edata.eks.m_total_ts": { - "to": "Total time spent" - }, - "edata.eks.m_total_count": { - "to": "Total response count" - }, - "edata.eks.m_correct_res_count": { - "to": "Correct response count" - }, - "edata.eks.m_inc_res_count": { - "to": "Incorrect response count" - }, - "edata.eks.m_correct_res": { - "hidden": true - }, - "edata.eks.m_incorrect_res": { - "hidden": true - }, - "edata.eks.m_top5_incorrect_res": { - "to": "Top 5 Incorrect response" - }, - "edata.eks.m_avg_ts": { - "to": "Average time spent" - }, - "edata.eks.m_top5_misconception_res": { - "to": "Top 5 Misconception response" - } - } - } - }, - "ME_GENIE_USAGE_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-consumption-metrics/me_genie_usage_metrics/" - } - }, - "filterMapping": { - "tags": { - "name": "dimensions.tag", - "operator": "IN" - }, - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation Time(IST)" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Event sync time(IST)" - }, - "eid": { - "hidden": true - }, - "dimensions.period": { - "to": "Time period of the summary. For ex: Day or Week or Month represented by a number" - }, - "dimensions.tag": { - "to": "Device Tag" - }, - "edata.eks.m_total_sessions": { - "to": "Total number of sessions" - }, - "edata.eks.m_total_ts": { - "to": "Total time spent" - }, - "edata.eks.m_avg_ts_session": { - "to": "Average sessions per device" - }, - "edata.eks.m_contents": { - "hidden": true - }, - "edata.eks.m_total_devices": { - "to": "Total number of devices" - } - } - } - }, - "ME_CONTENT_SNAPSHOT_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-consumption-metrics/me_content_snapshot_metrics/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - } - }, - "ME_CONCEPT_SNAPSHOT_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-consumption-metrics/me_concept_snapshot_metrics/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - } - }, - "ME_ASSET_SNAPSHOT_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-consumption-metrics/me_asset_snapshot_metrics/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - } - } - } - }, - "eks-creation-raw": { - "events": [ - "eks-creation-raw" - ], - "eventConfig": { - "eks-creation-raw": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "unique/" - } - }, - "filterMapping": { - "channel": { - "name": "context.channel", - "operator": "EQ" - }, - "app_id": { - "name": "context.pdata.id", - "operator": "EQ" - } - } - } - } - }, - "eks-creation-summary": { - "events": [ - "ME_APP_SESSION_SUMMARY", - "ME_CE_SESSION_SUMMARY", - "ME_TEXTBOOK_SESSION_SUMMARY", - "ME_APP_USAGE_SUMMARY", - "ME_CE_USAGE_SUMMARY", - "ME_TEXTBOOK_USAGE_SUMMARY", - "ME_AUTHOR_USAGE_SUMMARY", - "ME_PUBLISH_PIPELINE_SUMMARY" - ], - "eventConfig": { - "ME_APP_SESSION_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/app-ss/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "context.granularity": { - "hidden": true - }, - "edata.eks.interact_events_per_min": { - "to": "Average ineractions per minute" - }, - "edata.eks.events_summary": { - "to": "edata.eks.events_summary" - }, - "dimensions.app_id": { - "to": "Event Source" - }, - "context.date_range.from": { - "hidden": true - }, - "edata.eks.start_time": { - "mapFunc": "timestampToDateTime", - "to": "Start Time(IST)" - }, - "uid": { - "to": "User ID" - }, - "edata.eks.page_summary": { - "to": "Pages visited Summary" - }, - "ets": { - "hidden": true - }, - "edata.eks.page_views_count": { - "to": "Total pageviews in Session" - }, - "edata.eks.end_time": { - "hidden": true - }, - "eid": { - "hidden": true - }, - "ver": { - "hidden": true - }, - "context.pdata.id": { - "hidden": true - }, - "edata.eks.ce_visits": { - "to": "Number of Authoring tool visits" - }, - "dimensions.sid": { - "to": "Session ID" - }, - "edata.eks.time_diff": { - "to": "Session length " - }, - "edata.eks.interact_events_count": { - "to": "Number of Interactions" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Sync time(IST)" - }, - "context.pdata.ver": { - "hidden": true - }, - "context.pdata.model": { - "hidden": true - }, - "edata.eks.time_spent": { - "to": "Session length" - }, - "dimensions.anonymous_user": { - "to": "User Logged In" - }, - "mid": { - "to": "mid" - }, - "edata.eks.env_summary": { - "to": "Usage summary" - }, - "edata.eks.first_visit": { - "to": "First Time Visitor" - }, - "context.date_range.to": { - "hidden": true - } - } - } - }, - "ME_CE_SESSION_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/ce-ss/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": false, - "columnMappings": {} - } - }, - "ME_TEXTBOOK_SESSION_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/textbook-ss/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": false, - "columnMappings": {} - } - }, - "ME_APP_USAGE_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/app-usage/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "edata.eks.avg_ts_session": { - "to": "Average session duration per day" - }, - "context.granularity": { - "to": "Granularity" - }, - "edata.eks.anon_total_sessions": { - "to": "Time spent by non-registered users" - }, - "edata.eks.unique_users": { - "to": "Unique users list" - }, - "dimensions.app_id": { - "to": "Source" - }, - "edata.eks.unique_users_count": { - "to": "Unique users count" - }, - "context.date_range.from": { - "hidden": true - }, - "edata.eks.total_pageviews_count": { - "to": "Total Page Views per day" - }, - "uid": { - "to": "uid" - }, - "edata.eks.ce_total_sessions": { - "to": "Number of sessions where creation happened" - }, - "edata.eks.anon_avg_ts_session": { - "to": "Avg. session duration for non-registered users per day" - }, - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Generation Time(IST)" - }, - "edata.eks.total_ts": { - "to": "Total time spent by visitors on Portal" - }, - "edata.eks.ce_percent_sessions": { - "to": "Percentage of visitors creating content" - }, - "dimensions.author_id": { - "to": "Author ID" - }, - "eid": { - "hidden": true - }, - "ver": { - "hidden": true - }, - "context.pdata.id": { - "hidden": true - }, - "edata.eks.new_user_count": { - "to": "Number of new users" - }, - "syncts": { - "to": "Event sync time(IST)", - "mapFunc": "timestampToDateTime" - }, - "edata.eks.percent_new_users_count": { - "to": "Percentage of new visitors per day" - }, - "context.pdata.ver": { - "hidden": true - }, - "context.pdata.model": { - "hidden": true - }, - "edata.eks.avg_pageviews": { - "to": "Avg. pages per visit per day" - }, - "edata.eks.total_sessions": { - "to": "Total number of sessions" - }, - "edata.eks.anon_total_ts": { - "to": "Total time spent by non-registered users on Portal" - }, - "mid": { - "hidden": true - }, - "dimensions.period": { - "to": "dimensions.period" - }, - "context.date_range.to": { - "hidden": true - } - } - } - }, - "ME_CE_USAGE_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/ce-usage/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": false, - "columnMappings": {} - } - }, - "ME_TEXTBOOK_USAGE_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/textbook-usage/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": false, - "columnMappings": {} - } - }, - "ME_AUTHOR_USAGE_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/author-usage-summary/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "uid": { - "to": "Author ID" - }, - "mid": { - "hidden": true - }, - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation time(IST)" - }, - "eid": { - "hidden": true - }, - "ver": { - "hidden": true - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Sync time(IST)" - }, - "context.pdata.id": { - "hidden": true - }, - "context.pdata.ver": { - "hidden": true - }, - "context.pdata.model": { - "hidden": true - }, - "edata.eks.avg_ts_session": { - "to": "Total number of seesion per day" - }, - "edata.eks.total_ts": { - "to": "Total time spent by author" - }, - "edata.eks.ce_total_ts": { - "to": "Total time spent on content editor by author" - }, - "edata.eks.total_sessions": { - "to": "No of sessions per Author" - }, - "context.granularity": { - "hidden": true - }, - "edata.eks.ce_percent_ts": { - "to": "Percentage of time spent browsing vs creating" - }, - "edata.eks.ce_percent_sessions": { - "to": "Percentage of sessions in which users log into content editor" - }, - "edata.eks.ce_total_visits": { - "to": "No of Content Editor visits per author" - }, - "context.date_range.from": { - "hidden": true - }, - "context.date_range.to": { - "hidden": true - } - } - } - }, - "ME_PUBLISH_PIPELINE_SUMMARY": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "derived/publish-pipeline-summ/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "context.granularity": { - "hidden": true - }, - "context.date_range.from": { - "hidden": true - }, - "uid": { - "hidden": true - }, - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation Time(IST)" - }, - "eid": { - "hidden": true - }, - "ver": { - "hidden": true - }, - "context.pdata.id": { - "hidden": true - }, - "edata.eks.publish_pipeline_summary": { - "to": "edata.eks.publish_pipeline_summary" - }, - "syncts": { - "to": "Event sync time(IST)", - "mapFunc": "timestampToDateTime" - }, - "context.pdata.ver": { - "hidden": true - }, - "context.pdata.model": { - "hidden": true - }, - "mid": { - "hidden": true - }, - "dimensions.period": { - "to": "Date" - }, - "context.date_range.to": { - "hidden": true - } - } - } - } - } - }, - "eks-creation-metrics": { - "events": [ - "ME_APP_USAGE_METRICS", - "ME_CE_USAGE_METRICS", - "ME_TEXTBOOK_CREATION_METRICS", - "ME_TEXTBOOK_SNAPSHOT_METRICS", - "ME_AUTHOR_USAGE_METRICS" - ], - "eventConfig": { - "ME_APP_USAGE_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-creation-metrics/me_app_usage_metrics/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation Time(IST)" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Event sync time(IST)" - }, - "eid": { - "hidden": true - }, - "dimensions.period": { - "to": "Time period of the summary. For ex: Day or Week or Month represented by a number" - }, - "dimensions.author_id": { - "to": "Author ID / All" - }, - "dimensions.app_id": { - "to": "App (EkStep portal currently)" - }, - "edata.eks.anon_total_sessions": { - "to": "Total anonymous sessions on portal" - }, - "edata.eks.anon_total_ts": { - "to": "Tota time spent (Anonymous) on portal" - }, - "edata.eks.total_sessions": { - "to": "Total number of portal sessions" - }, - "edata.eks.total_ts": { - "to": "Total time spent on portal" - }, - "edata.eks.ce_total_sessions": { - "to": "Total content editor sessions" - }, - "edata.eks.ce_percent_sessions": { - "to": "% of total sessions that are content editor sessions" - }, - "edata.eks.total_pageviews_count": { - "to": "Total pageviews on portal" - }, - "edata.eks.unique_users": { - "hidden": true - }, - "edata.eks.unique_users_count": { - "to": "Count of unique users on portal" - }, - "edata.eks.avg_pageviews": { - "to": "Avg. number of pageviews per session for specified period" - }, - "edata.eks.avg_ts_session": { - "to": "Avg. session length (time spent) per session for specified period" - }, - "edata.eks.anon_avg_ts_session": { - "to": "Avg. Anonymous session length (time spent) per session for specified period" - }, - "edata.eks.new_user_count": { - "to": "No. of new users on the app (portal) for the given time period" - }, - "edata.eks.percent_new_users_count": { - "to": "% of new users for the given time period" - } - } - } - }, - "ME_CE_USAGE_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-creation-metrics/me_ce_usage_metrics/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation Time(IST)" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Event sync time(IST)" - }, - "eid": { - "hidden": true - }, - "dimensions.period": { - "to": "Time period of the summary. For ex: Day or Week or Month represented by a number" - }, - "dimensions.content_id": { - "to": "Content ID" - }, - "edata.eks.unique_users_count": { - "to": "Number of users who accessed the content piece via the editor" - }, - "edata.eks.total_sessions": { - "to": "Total number of sessions that accessed the content piece via the editor" - }, - "edata.eks.total_ts": { - "to": "Time spent on the content piece in the content editor" - }, - "edata.eks.avg_ts_session": { - "to": "Average session time on the editor" - }, - "edata.eks.update_date": { - "to": "Last updated timestamp of the content piece" - } - } - } - }, - "ME_TEXTBOOK_CREATION_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-creation-metrics/me_textbook_creation_metrics/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - } - }, - "ME_TEXTBOOK_SNAPSHOT_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-creation-metrics/me_textbook_snapshot_metrics/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - } - }, - "ME_AUTHOR_USAGE_METRICS": { - "searchType": "azure", - "fetchConfig": { - "params": { - "bucket": "{{ bucket }}", - "prefix": "eks-creation-metrics/me_author_usage_metrics/" - } - }, - "filterMapping": { - "channel": { - "name": "channel", - "operator": "EQ" - }, - "app_id": { - "name": "dimensions.pdata.id", - "operator": "EQ" - } - }, - "csvConfig": { - "auto_extract_column_names": true, - "columnMappings": { - "ets": { - "mapFunc": "timestampToDateTime", - "to": "Event generation Time(IST)" - }, - "syncts": { - "mapFunc": "timestampToDateTime", - "to": "Event sync time(IST)" - }, - "eid": { - "hidden": true - }, - "dimensions.period": { - "to": " Time period of the summary. For ex: Day or Week or Month represented by a number" - }, - "dimensions.author_id": { - "to": "Author ID /All" - }, - "edata.eks.total_sessions": { - "to": "Total number of portal sessions by author" - }, - "edata.eks.total_ts": { - "to": "Total time spent on portal by author" - }, - "edata.eks.total_ce_ts": { - "to": "Total time spent on content editor by author" - }, - "edata.eks.total_ce_visit": { - "to": "Number of content editor visits" - }, - "edata.eks.percent_ce_sessions": { - "to": "% of sessions that were spent on the editor" - }, - "edata.eks.avg_ts_session": { - "to": "Average time spent by author on portal per session" - }, - "edata.eks.percent_ce_ts": { - "to": "% time spent by author on content editor" - } - } - } - } - } - } - } - }, "wfs": { "search": { "type": "azure", "queries": [ { "bucket": "{{ bucket }}", - "prefix": "unique/", + "prefix": "unique/raw/", "endDate": "$(date --date yesterday '+%Y-%m-%d')", "delta": 0 } @@ -1891,167 +42,6 @@ "appName": "Workflow Summarizer", "deviceMapping": true }, - "wfus": { - "search": { - "type": "azure", - "queries": [ - { - "bucket": "{{ bucket }}", - "prefix": "{{ job_manager_tmp_dir }}/wfs/", - "endDate": "$(date --date yesterday '+%Y-%m-%d')", - "delta": 0, - "folder": "true" - } - ] - }, - "model": "org.ekstep.analytics.model.WorkflowUsageSummary", - "modelParams": { - "apiVersion": "v2" - }, - "output": [ - { - "to": "azure", - "params": { - "bucket": "{{ bucket }}", - "key": "{{ job_manager_tmp_dir }}/wfus/$(date --date yesterday '+%Y-%m-%d')" - } - }, - { - "to": "kafka", - "params": { - "brokerList": "{{ brokerlist }}", - "topic": "{{ topic }}" - } - } - ], - "parallelization": 8, - "appName": "Workflow Usage Summarizer", - "deviceMapping": true - }, - "wfu": { - "search": { - "type": "azure", - "queries": [ - { - "bucket": "{{ bucket }}", - "prefix": "{{ job_manager_tmp_dir }}/wfus/", - "endDate": "$(date --date yesterday '+%Y-%m-%d')", - "delta": 0, - "folder": "true" - } - ] - }, - "appName": "Workflow Usage Updater", - "parallelization": 8, - "model": "org.ekstep.analytics.updater.UpdateWorkFlowUsageDB", - "modelParams": {}, - "output": [ - { - "to": "console", - "params": { - "printEvent": false - } - } - ] - }, - "ds": { - "search": { - "type": "azure", - "queries": [ - { - "bucket": "{{ bucket }}", - "prefix": "unique/", - "endDate": "$(date --date yesterday '+%Y-%m-%d')", - "delta": 0 - }, - { - "bucket": "{{ bucket }}", - "prefix": "{{ job_manager_tmp_dir }}/wfs/", - "endDate": "$(date --date yesterday '+%Y-%m-%d')", - "delta": 0, - "folder": "true" - } - ] - }, - "model": "org.ekstep.analytics.model.DeviceSummaryModel", - "modelParams": { - "apiVersion": "v2" - }, - "output": [ - { - "to": "azure", - "params": { - "bucket": "{{ bucket }}", - "key": "{{ job_manager_tmp_dir }}/device-summary/$(date --date yesterday '+%Y-%m-%d')" - } - }, - { - "to": "kafka", - "params": { - "brokerList": "{{ brokerlist }}", - "topic": "{{ topic }}" - } - } - ], - "parallelization": 12, - "appName": "Device Summarizer", - "deviceMapping": false - }, - "dpu": { - "search": { - "type": "azure", - "queries": [ - { - "bucket": "{{ bucket }}", - "prefix": "{{ job_manager_tmp_dir }}/device-summary/", - "endDate": "$(date --date yesterday '+%Y-%m-%d')", - "delta": 0, - "folder": "true" - } - ] - }, - "model": "org.ekstep.analytics.model.UpdateDeviceProfileDB", - "modelParams": { - "apiVersion": "v2" - }, - "output": [ - { - "to": "console", - "params": { - "printEvent": false - } - } - ], - "parallelization": 8, - "appName": "Device Profile Updater", - "deviceMapping": false - }, - "workflow-usage-metrics": { - "search": { - "type": "azure", - "queries": [ - { - "bucket": "{{ bucket }}", - "prefix": "{{ job_manager_tmp_dir }}/wfus/", - "endDate": "$(date --date yesterday '+%Y-%m-%d')", - "delta": 0, - "folder": "true" - } - ] - }, - "model": "org.ekstep.analytics.updater.UpdateWorkFlowUsageMetricsMode", - "output": [ - { - "to": "console", - "params": { - "printEvent": false - } - } - ], - "parallelization": 10, - "appName": "Workflow Usage Metrics Updater", - "deviceMapping": false - }, "video-streaming": { "search": { "type": "azure" @@ -2072,134 +62,6 @@ "appName": "Video Streaming Job", "deviceMapping": false }, - "course-dashboard-metrics": { - "search": { - "type": "none" - }, - "model": "org.ekstep.analytics.updater.UpdateCourseMetrics", - "modelParams": { - "druidConfig": { - "queryType": "groupBy", - "dataSource": "content-model-snapshot", - "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00", - "granularity": "all", - "aggregations": [ - { - "name": "count", - "type": "count", - "fieldName": "count" - } - ], - "dimensions": [ - { - "fieldName": "identifier", - "aliasName": "identifier" - }, - { - "fieldName": "channel", - "aliasName": "channel" - } - ], - "filters": [ - { - "type": "equals", - "dimension": "contentType", - "value": "Course" - }, - { - "type": "isnotnull", - "dimension": "identifier", - "value": "" - }, - { - "type": "isnotnull", - "dimension": "channel", - "value": "" - } - ], - "descending": "false" - }, - "fromDate": "$(date --date yesterday '+%Y-%m-%d')", - "toDate": "$(date --date yesterday '+%Y-%m-%d')", - "sparkCassandraConnectionHost": "{{core_cassandra_host}}", - "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}" - }, - "output": [ - { - "to": "console", - "params": { - "printEvent": false - } - } - ], - "parallelization": 8, - "appName": "Course Dashboard Metrics", - "deviceMapping": false - }, - "assessment-dashboard-metrics": { - "search": { - "type": "none" - }, - "model": "org.sunbird.analytics.job.report.AssessmentMetricsJob", - "modelParams": { - "druidConfig": { - "queryType": "groupBy", - "dataSource": "content-model-snapshot", - "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00", - "granularity": "all", - "aggregations": [ - { - "name": "count", - "type": "count", - "fieldName": "count" - } - ], - "dimensions": [ - { - "fieldName": "identifier", - "aliasName": "identifier" - }, - { - "fieldName": "channel", - "aliasName": "channel" - } - ], - "filters": [ - { - "type": "equals", - "dimension": "contentType", - "value": "Course" - }, - { - "type": "isnotnull", - "dimension": "identifier", - "value": "" - }, - { - "type": "isnotnull", - "dimension": "channel", - "value": "" - } - ], - "descending": "false" - }, - "fromDate": "$(date --date yesterday '+%Y-%m-%d')", - "toDate": "$(date --date yesterday '+%Y-%m-%d')", - "sparkCassandraConnectionHost": "{{core_cassandra_host}}", - "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}" - }, - "output": [ - { - "to": "console", - "params": { - "printEvent": false - } - } - ], - "parallelization": 8, - "appName": "Assessment Dashboard Metrics", - "deviceMapping": false - }, "admin-user-reports": { "search": { "type": "none" @@ -2278,51 +140,11 @@ "brokerList": "{{ brokerlist }}", "topic": "{{ analytics_metrics_topic }}", "model": [ - { - "model": "DeviceSummaryModel", - "category": "consumption", - "input_dependency": "None" - }, - { - "model": "UpdateDeviceProfileDB", - "category": "consumption", - "input_dependency": "DeviceSummaryModel" - }, - { - "model": "DataExhaustJob", - "category": "consumption", - "input_dependency": "None" - }, - { - "model": "UpdateWorkFlowUsageDB", - "category": "consumption", - "input_dependency": "WorkFlowUsageSummaryModel" - }, { "model": "WorkFlowSummaryModel", "category": "consumption", "input_dependency": "None" }, - { - "model": "WorkFlowUsageSummaryModel", - "category": "consumption", - "input_dependency": "WorkFlowSummaryModel" - }, - { - "model": "UpdatePortalMetrics", - "category": "consumption", - "input_dependency": "None" - }, - { - "model": "UpdateWorkFlowUsageMetricsModel", - "category": "consumption", - "input_dependency": "None" - }, - { - "model": "CourseMetricsJob", - "category": "consumption", - "input_dependency": "None" - }, { "model": "UpdateContentRating", "category": "consumption", @@ -2333,36 +155,11 @@ "category": "consumption", "input_dependency": "None" }, - { - "model": "DistrictMonthlyJob", - "category": "consumption", - "input_dependency": "None" - }, - { - "model": "DesktopConsumptionDailyMetricsJob", - "category": "consumption", - "input_dependency": "None" - }, - { - "model": "DistrictWeeklyJob", - "category": "consumption", - "input_dependency": "None" - }, - { - "model": "ConsumptionDailyMetricsJob", - "category": "consumption", - "input_dependency": "None" - }, { "model": "MetricsAuditJob", "category": "consumption", "input_dependency": "None" }, - { - "model": "AssessmentMetricsJob", - "category": "consumption", - "input_dependency": "None" - }, { "model": "StateAdminReportJob", "category": "consumption", @@ -2679,79 +476,6 @@ "appName": "TPD Course Consumption Metrics Model", "deviceMapping": false }, - "textbook-progress-report": { - "search": { - "type": "none" - }, - "model": "org.ekstep.analytics.model.report.TextBookProgressModel", - "modelParams": { - "reportConfig": { - "id": "content_progress_metrics", - "metrics": [], - "labels": { - "board": "Board", - "medium": "Medium", - "gradeLevel": "Grade", - "subject": "Subject", - "resourceType": "Content Type", - "totalContent": "Total Contents", - "live": "Live", - "review": "Review", - "draft": "Draft", - "unlisted": "Limited Sharing", - "application_ecml": "Created on Diksha", - "video_youtube": "YouTube Content", - "video_mp4": "Uploaded Videos", - "application_pdf": "Text Content", - "application_html": "Uploaded Interactive Content", - "identifier": "Content ID", - "creator": "Created By", - "createdOn": "Creation Date", - "lastPublishDate": "Last Publish Date", - "status": "Status", - "pkgVersion": "Number of times Published", - "lastPublishedOn": "Pending in current status since", - "pendingInCurrentStatus": "Pending in current status since" - }, - "output": [{ - "type": "csv", - "dims": [] - }], - "mergeConfig": { - "frequency": "WEEK", - "basePath": "{{ spark_output_temp_dir }}", - "rollup": 0, - "reportPath": "content_progress_metrics.csv", - "postContainer":"{{ reports_container }}" - } - }, - "filter": { - "tenantId": "", - "slugName": "" - }, - "store": "azure", - "format": "csv", - "key": "druid-reports/", - "filePath": "druid-reports/", - "container": "{{ bucket }}", - "folderPrefix": [ - "slug", - "reportName" - ], - "sparkCassandraConnectionHost": "{{core_cassandra_host}}" - }, - "output": [ - { - "to": "console", - "params": { - "printEvent": false - } - } - ], - "parallelization": 8, - "appName": "Textbook Progress Metrics Model", - "deviceMapping": false - }, "audit-metrics-report": { "search": { "type": "none" diff --git a/ansible/roles/data-products-deploy/templates/model-dock-config.j2 b/ansible/roles/data-products-deploy/templates/model-dock-config.j2 new file mode 100644 index 0000000000..20d82dbfb5 --- /dev/null +++ b/ansible/roles/data-products-deploy/templates/model-dock-config.j2 @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +config() { + bucket={{ bucket }} + brokerList={{ brokerlist }} + zookeeper={{ zookeeper }} + job_topic={{ analytics_job_queue_topic }} + topic={{ topic }} + sparkCassandraConnectionHost="{{ lp_cassandra_host }}" + sunbirdPlatformCassandraHost="{{ core_cassandra_host }}" + reportPostContainer="{{ reports_container }}" + druidRollupHost="{{ druid_rollup_cluster_ingestion_task_url }}" + + if [ -z "$2" ]; then endDate=$(date --date yesterday "+%Y-%m-%d"); else endDate=$2; fi + if [ ! -z "$3" ]; then inputBucket=$3; fi + case "$1" in + "content-details") + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.ContentDetailsReport","modelParams":{"tenantId":"","slug":"","reportConfig":{"id":"content_report","metrics":[],"labels":{"programName":"Project Name","programId":"Project ID","contentId":"Content/Question ID","contentName":"Content/Question Name","mimeType":"MimeType","chapterId":"Folder ID","contentStatus":"Content/Question Status","creator":"Creator Name","createdBy":"CreatedBy ID","date":"Date","identifier":"Collection/Question Set ID","name":"Collection/Question Set Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","board":"Board","grade":"Grade","chapters":"Chapter Name","status":"Textbook Status","objectType":"Object Type","primaryCategory":"Primary category","topic":"Topic","learningOutcome":"Learning Outcome","addedFromLibrary":"Added from library","contentType":"Content Type"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"contentQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"unitIdentifiers","aliasName":"unitIdentifiers"},{"fieldName":"collectionId","aliasName":"collectionId"},{"fieldName":"createdBy","aliasName":"createdBy"},{"fieldName":"creator","aliasName":"creator"},{"fieldName":"mimeType","aliasName":"mimeType"},{"fieldName":"topic","aliasName":"topic"},{"fieldName":"learningOutcome","aliasName":"learningOutcome"},{"fieldName":"primaryCategory","aliasName":"contentType"}],"filters":[{"type":"notequals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live"]},{"type":"isnotnull","dimension":"collectionId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"textbookQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"programId","aliasName":"programId"},{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"},{"fieldName":"acceptedContents","aliasName":"acceptedContents"},{"fieldName":"acceptedContributions","aliasName":"acceptedContributions"},{"fieldName":"rejectedContents","aliasName":"rejectedContents"},{"fieldName":"rejectedContributions","aliasName":"rejectedContributions"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"objectType","aliasName":"objectType"},{"fieldName":"reusedContributions","aliasName":"reusedContributions"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper","Exam Question Set","Practice Set","Demo Practice Question Set"]},{"type":"isnotnull","dimension":"programId"},{"type":"in","dimension":"status","values":["Draft"]},{"type":"equals","dimension":"channel","value":"channelId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"azure","storageKeyConfig":"azure_storage_key","storageSecretConfig":"azure_storage_secret","storageContainer":"'$reportPostContainer'","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Content Report Job","deviceMapping":false}' + ;; + "sourcing-summary-report") + echo '{"search": {"type": "none"}, "model": "org.ekstep.analytics.job.report.SourcingSummaryReport", "modelParams": {"storageKeyConfig":"druid_storage_account_key", "storageSecretConfig":"druid_storage_account_secret", "dataSource": "sourcing-summary-snapshot", "druidHost": "'$druidRollupHost'", "druidSegmentUrl":"'$druidRollupHost'/druid/coordinator/v1/metadata/datasources/sourcing-model-snapshot/segments", "deleteSegmentUrl": "'$druidRollupHost'/druid/coordinator/v1/datasources/sourcing-model-snapshot/segments/", "druidIngestionUrl": "'$druidRollupHost'/druid/indexer/v1/task", "specPath": "/mount/data/analytics/scripts/sourcing-ingestion-spec.json", "dbName": "opensaberdb", "tables": {"programTable": "program", "nominationTable": "nomination"}, "druidQuery": {"queryType": "groupBy", "dataSource": "vdn-content-model-snapshot", "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00", "aggregations": [{"name": "count", "type": "count"}], "dimensions": [{"fieldName": "primaryCategory", "aliasName": "primaryCategory"}, {"fieldName": "createdBy", "aliasName": "createdBy"}], "filters": [{"type": "equals", "dimension": "objectType", "value": "Content"}, {"type": "equals", "dimension": "sampleContent", "value": "false"}], "postAggregation": [], "descending": "false", "limitSpec": {"type": "default", "limit": 1000000, "columns": [{"dimension": "count", "direction": "descending"}]}}, "reportConfig": {"id": "sourcing", "metrics": [], "labels": {}, "output": [{"type": "json", "dims": ["identifier", "channel", "name"], "fileParameters": ["id", "dims"]}]}, "store": "azure", "format": "json", "folderPrefix": ["slug", "reportName"]}, "output": [{"to": "console", "params": {"printEvent": false}}], "parallelization": 8, "appName": "Sourcing Summary Report Job", "deviceMapping": false}' + ;; + "funnel-report") + echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.job.report.FunnelReport","modelParams": {"contributionConfig": {"contentRequest": {"request": {"filters": {"programId": "programIdentifier","objectType": "content","status": ["Draft", "Live", "Review"],"mimeType": "application/vnd.ekstep.content-collection"},"fields": ["acceptedContents", "rejectedContents"],"limit": 10000}},"correctionsPendingRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": "Draft","prevStatus": "Live","programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}},"contributionRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": ["Live"],"programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}}},"reportConfig": {"id": "funnel_report","metrics": [],"labels": {"reportDate": "Report generation date","visitors": "No. of users opening the project","projectName": "Project Name","initiatedNominations": "No. of initiated nominations","rejectedNominations": "No. of rejected nominations","pendingNominations": "No. of nominations pending review","acceptedNominations": "No. of accepted nominations to the project","noOfContributors": "No. of contributors to the project","noOfContributions": "No. of contributions to the project","pendingContributions": "No. of contributions pending review","approvedContributions": "No. of approved contributions"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"store": "azure","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","folderPrefix": ["slug", "reportName"]},"sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","druidConfig": {"queryType": "timeseries","dataSource": "telemetry-events-syncts","intervals": "startdate/enddate","aggregations": [{"name": "visitors","type": "count","fieldName": "actor_id"}],"filters": [{"type": "equals","dimension": "context_cdata_id","value": "program_id"}, {"type": "equals","dimension": "edata_pageid","value": "contribution_project_contributions"}, {"type": "equals","dimension": "context_pdata_pid","value": "creation-portal.programs"}, {"type": "equals","dimension": "context_cdata_type","value": "project"}, {"type": "equals","dimension": "context_env","value": "creation-portal"}, {"type": "equals","dimension": "eid","value": "IMPRESSION"}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Funnel Report Job","deviceMapping": false}' + ;; + "sourcing-metrics") + echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.sourcing.SourcingMetrics","modelParams": {"reportConfig": {"id": "textbook_report","metrics": [],"labels": {"date": "Date","identifier": "Textbook ID","name": "Textbook Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","reportDate": "Report generation date","board": "Board","grade": "Grade","chapters": "Chapter Name","totalChapters": "Total number of chapters (first level sections of ToC)","status": "Textbook Status"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "equals","dimension": "contentType","value": "TextBook"}, {"type": "in","dimension": "status","values": ["Live"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"store": "azure","storageContainer": "'$reportPostContainer'","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Textbook Report Job","deviceMapping": false}' + ;; + "*") + echo "Unknown model code" + exit 1 # Command to come out of the program with status 1 + ;; + esac +} diff --git a/ansible/roles/data-products-deploy/templates/monitor-dp.j2 b/ansible/roles/data-products-deploy/templates/monitor-dp.j2 deleted file mode 100644 index a111aa4649..0000000000 --- a/ansible/roles/data-products-deploy/templates/monitor-dp.j2 +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash - -log_file=$1 - -total_jobs=`grep "BE_JOB_END" $log_file` -comp_jobs=`grep "status\":\"SUCCESS" <<< "$total_jobs"` -failed_jobs=`grep "status\":\"FAILED" <<< "$total_jobs"` - -today=$(date "+%Y-%m-%d") - -comp_num=0 -failed_num=0 -total_events=0 -total_time=0 - -file_content="Model,Job Status,Input Events Count, Output Events Count,Time Taken(in Seconds),Date\n" -job_status="COMPLETED" -# Declaring Arrays -declare -A input -declare -A output -warnings="" - -if [ "$comp_jobs" != "" ]; then - comp_num=`echo "$comp_jobs" | wc -l | bc` - while read -r line - do - model=`sed 's/.*model":"\(.*\)","ver.*/\1/' <<< "$line" | sed -e "s/org.ekstep.analytics.model.//g" | sed -e "s/org.ekstep.analytics.updater.//g"` - #message=`sed 's/.*message":"\(.*\)","class.*/\1/' <<< "$line"` - - eventsIn=`sed 's/.*","inputEvents":\(.*\),"outputEvents":.*/\1/' <<< "$line"` - sub="BE_JOB_END" - if [ "${eventsIn/$sub}" != "$eventsIn" ] ; then - eventsIn=0 - fi - events=`sed 's/.*"outputEvents":\(.*\),"timeTaken".*/\1/' <<< "$line"` - if [ "${events/$sub}" != "$events" ] ; then - events=0 - fi - total_events=$((total_events + events)) - time_taken=`sed 's/.*"timeTaken":\(.*\)},".*/\1/' <<< "$line"` - total_time=`echo $total_time + $time_taken | bc` - event_date=`sed 's/.*{"date":"\(.*\)","inputEvents":.*/\1/' <<< "$line"` - if [ "${event_date/$sub}" != "$event_date" ] ; then - event_date="" - fi - input+=( [$model]=$eventsIn ) - output+=( [$model]=$events ) - file_content+="$model,$job_status,$eventsIn,$events,$time_taken,$event_date\n" - done <<< "$comp_jobs" -fi - -events="0" -time_taken="0" -job_status="FAILED" -event_date="" - -if [ "$failed_jobs" != "" ]; then - failed_num=`echo "$failed_jobs" | wc -l | bc` - while read -r line - do - model=`sed 's/.*model":"\(.*\)","ver.*/\1/' <<< "$line" | sed -e "s/org.ekstep.analytics.model.//g" | sed -e "s/org.ekstep.analytics.updater.//g"` - #message=`grep -Po '(?<="localizedMessage":")[^"]*' <<< "$line"` - sub="BE_JOB_END" - eventsIn=`sed 's/.*","inputEvents":\(.*\)},".*/\1/' <<< "$line"` - if [ "${eventsIn/$sub}" != "$eventsIn" ] ; then - eventsIn=0 - fi - event_date=`sed 's/.*"data":{"date":"\(.*\)","inputEvents".*/\1/' <<< "$line"` - if [ "${event_date/$sub}" != "$event_date" ] ; then - event_date="" - fi - echo "$event_date" - input+=( [$model]=$eventsIn ) - output+=( [$model]=$events ) - file_content+="$model,$job_status,$eventsIn,$events,$time_taken,$event_date\n" - done <<< "$failed_jobs" -fi - -if [[ ${output["LearnerSessionSummaryModel"]} != ${input["ItemUsageSummaryModel"]} ]]; then - warnings+="Output of LearnerSessionSummaryModel NOT EQUALS to Input of ItemUsageSummaryModel\n" -fi - -if [[ ${output["GenieLaunchSummaryModel"]} != ${input["GenieUsageSummaryModel"]} ]]; then - warnings+="Output of GenieLaunchSummaryModel NOT EQUALS to Input of GenieUsageSummaryModel\n" -fi - -if [[ ${output["GenieFunnelModel"]} != ${input["GenieFunnelAggregatorModel"]} ]]; then - warnings+="Output of GenieFunnelModel NOT EQUALS to Input of GenieFunnelAggregatorModel\n" -fi - -if [[ ${output["GenieLaunchSummaryModel"]} != ${input["GenieStageSummaryModel"]} ]]; then - warnings+="Output of GenieLaunchSummaryModel NOT EQUALS to Input of GenieStageSummaryModel\n" -fi - -if [[ ${output["ItemUsageSummaryModel"]} != ${input["ItemSummaryModel"]} ]]; then - warnings+="Output of ItemUsageSummaryModel NOT EQUALS to Input of ItemSummaryModel\n" -fi - -if [[ ${output["LearnerSessionSummaryModel"]} != ${input["DeviceContentUsageSummaryModel"]} ]]; then - warnings+="Output of LearnerSessionSummaryModel NOT EQUALS to Input of DeviceContentUsageSummaryModel\n" -fi - -if [[ ${output["GenieLaunchSummaryModel"]} != ${input["DeviceUsageSummaryModel"]} ]]; then - warnings+="Output of GenieLaunchSummaryModel NOT EQUALS to Input of DeviceUsageSummaryModel\n" -fi - -if [[ ${output["GenieUsageSummaryModel"]} != ${input["UpdateGenieUsageDB"]} ]]; then - warnings+="Output of GenieUsageSummaryModel NOT EQUALS to Input of UpdateGenieUsageDB\n" -fi - -if [[ ${output["ItemSummaryModel"]} != ${input["UpdateItemSummaryDB"]} ]]; then - warnings+="Output of ItemSummaryModel NOT EQUALS to Input of UpdateItemSummaryDB\n" -fi - -if [[ ${output["ContentPopularitySummaryModel"]} != ${input["UpdateContentPopularityDB"]} ]]; then - warnings+="Output of ContentPopularitySummaryModel NOT EQUALS to Input of UpdateContentPopularityDB\n" -fi - -if [[ ${output["ContentUsageSummaryModel"]} != ${input["UpdateContentUsageDB"]} ]]; then - warnings+="Output of ContentUsageSummaryModel NOT EQUALS to Input of UpdateContentUsageDB\n" -fi - -echo -e $file_content > {{ analytics.home }}/scripts/monitor-data/dp-monitor-$today.csv - - -echo "-------- Status Report --------" -echo "Number of Completed Jobs: $comp_num" -echo "Number of failed Jobs: $failed_num" -echo "Total time taken: $total_time" -echo "Total events generated: $total_events" - - -if [[("$failed_num" > 0 && "$warnings" == "") ]]; then - -data='{"channel": "{{ channel }}", "username": "{{ dp_username }}", "text":"*Jobs | Monitoring Report | '$today'*\nNumber of Completed Jobs: `'$comp_num'` \nNumber of failed Jobs: `'$failed_num'` \nTotal time taken: `'$total_time'`\nTotal events generated: `'$total_events'`\n\nDetailed Report:\n```'$file_content'```\n\nError: ```'"Job Failed"'```", "icon_emoji": ":ghost:"}' - -elif [[("$warnings" == "" && "$failed_num" == 0)]]; then - -data='{"channel": "{{ channel }}", "username": "{{ dp_username }}", "text":"*Jobs | Monitoring Report | '$today'*\nNumber of Completed Jobs: `'$comp_num'` \nNumber of failed Jobs: `'$failed_num'` \nTotal time taken: `'$total_time'`\nTotal events generated: `'$total_events'`\n\nDetailed Report:\n```'$file_content'```\n\nMessage: ```'"Job Run Completed Successfully"'```", "icon_emoji": ":ghost:"}' - -elif [[("$warnings" != "" && "$failed_num" == 0)]]; then - -data='{"channel": "{{ channel }}", "username": "{{ dp_username }}", "text":"*Jobs | Monitoring Report | '$today'*\nNumber of Completed Jobs: `'$comp_num'` \nNumber of failed Jobs: `'$failed_num'` \nTotal time taken: `'$total_time'`\nTotal events generated: `'$total_events'`\n\nDetailed Report:\n```'$file_content'```\n\nWarnings: ```'$warnings'``` ", "icon_emoji": ":ghost:"}' - -else - -data='{"channel": "{{ channel }}", "username": "{{ dp_username }}", "text":"*Jobs | Monitoring Report | '$today'*\nNumber of Completed Jobs: `'$comp_num'` \nNumber of failed Jobs: `'$failed_num'` \nTotal time taken: `'$total_time'`\nTotal events generated: `'$total_events'`\n\nDetailed Report:\n```'$file_content'```\n\nError: ```'"Job Failed"'```\n\nWarnings:```'$warnings'```", "icon_emoji": ":ghost:"}' -fi - -curl -X POST -H 'Content-Type: application/json' --data "$data" https://hooks.slack.com/services/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 diff --git a/ansible/roles/data-products-deploy/templates/replay-job.j2 b/ansible/roles/data-products-deploy/templates/replay-job.j2 index 72c9dcfcfe..4a2c0437b5 100644 --- a/ansible/roles/data-products-deploy/templates/replay-job.j2 +++ b/ansible/roles/data-products-deploy/templates/replay-job.j2 @@ -7,6 +7,8 @@ cd {{ analytics.home }}/scripts source model-config.sh source replay-utils.sh +libs_path="{{ analytics.home }}/models-{{ model_version }}/data-products-1.0" + if [ "$1" == "telemetry-replay" ] then if [ ! $# -eq 5 ] @@ -15,6 +17,17 @@ if [ "$1" == "telemetry-replay" ] exit fi fi + +get_report_job_model_name(){ + case "$1" in + "assessment-correction") echo 'org.sunbird.analytics.job.report.AssessmentCorrectionJob' + ;; + *) echo $1 + ;; + esac +} + +if [ ! -z "$1" ]; then job_id=$(get_report_job_model_name $1); fi if [ -z "$job_config" ]; then job_config=$(config $1 '__endDate__' $4 $5); fi start_date=$2 end_date=$3 @@ -33,7 +46,8 @@ if [ $? == 0 ] then echo "Backup completed Successfully..." >> "$DP_LOGS/$end_date-$1-replay.log" echo "Running the $1 job replay..." >> "$DP_LOGS/$end_date-$1-replay.log" - $SPARK_HOME/bin/spark-submit --master local[*] --jars $MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.3.2.jar --class org.ekstep.analytics.job.ReplaySupervisor $MODELS_HOME/batch-models-2.0.jar --model "$1" --fromDate "$start_date" --toDate "$end_date" --config "$job_config" >> "$DP_LOGS/$end_date-$1-replay.log" + echo "Job modelName - $job_id" >> "$DP_LOGS/$end_date-$1-replay.log" + $SPARK_HOME/bin/spark-submit --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.4.0.jar --class org.ekstep.analytics.job.ReplaySupervisor $MODELS_HOME/batch-models-2.0.jar --model "$job_id" --fromDate "$start_date" --toDate "$end_date" --config "$job_config" >> "$DP_LOGS/$end_date-$1-replay.log" else echo "Unable to take backup" >> "$DP_LOGS/$end_date-$1-replay.log" fi diff --git a/ansible/roles/data-products-deploy/templates/run-dock-job.j2 b/ansible/roles/data-products-deploy/templates/run-dock-job.j2 new file mode 100644 index 0000000000..d35aa1a064 --- /dev/null +++ b/ansible/roles/data-products-deploy/templates/run-dock-job.j2 @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +export SPARK_HOME={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7 +export MODELS_HOME={{ analytics.home }}/models-{{ model_version }} +export DP_LOGS={{ analytics.home }}/logs/data-products +## Job to run daily +cd {{ analytics.home }}/scripts +source model-dock-config.sh +today=$(date "+%Y-%m-%d") + +libs_path="{{ analytics.home }}/models-{{ model_version }}/data-products-1.0" +file_path="dock-{{ env }}.conf" + +get_report_job_model_name(){ + case "$1" in + "funnel-report") echo 'org.sunbird.analytics.sourcing.FunnelReport' + ;; + "sourcing-summary-report") echo 'org.sunbird.analytics.sourcing.SourcingSummaryReport' + ;; + "sourcing-metrics") echo 'org.sunbird.analytics.sourcing.SourcingMetrics' + ;; + "content-details") echo 'org.sunbird.analytics.sourcing.ContentDetailsReport' + ;; + *) echo $1 + ;; + esac +} + +if [ ! -z "$1" ]; then job_id=$(get_report_job_model_name $1); fi + +if [ ! -z "$1" ]; then job_config=$(config $1); else job_config="$2"; fi + +if [ ! -z "$2" ]; then batchIds=";$2"; else batchIds=""; fi + +echo "Starting the job - $1" >> "$DP_LOGS/$today-job-execution.log" + +echo "Job modelName - $job_id" >> "$DP_LOGS/$today-job-execution.log" + +nohup $SPARK_HOME/bin/spark-submit --conf spark.driver.extraJavaOptions="-Dconfig.file=$MODELS_HOME/$file_path" --conf spark.executor.extraJavaOptions="-Dconfig.file=$MODELS_HOME/$file_path" --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.4.0.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "$job_id" --config "$job_config$batchIds" \ >> "$DP_LOGS/$today-job-execution.log" 2>&1 + +echo "Job execution completed - $1" >> "$DP_LOGS/$today-job-execution.log" \ No newline at end of file diff --git a/ansible/roles/data-products-deploy/templates/run-job.j2 b/ansible/roles/data-products-deploy/templates/run-job.j2 index 681d7ad6c6..a09b03e39a 100644 --- a/ansible/roles/data-products-deploy/templates/run-job.j2 +++ b/ansible/roles/data-products-deploy/templates/run-job.j2 @@ -8,42 +8,16 @@ cd {{ analytics.home }}/scripts source model-config.sh today=$(date "+%Y-%m-%d") -libs_path="{{ analytics.home }}/models-{{ model_version }}/data-products-1.0" - -telemetry_converter_ver=0.0.8 - -get_report_job_model_name(){ - case "$1" in - "assessment-dashboard-metrics") echo 'org.sunbird.analytics.job.report.AssessmentMetricsJob' - ;; - "course-dashboard-metrics") echo 'org.sunbird.analytics.job.report.CourseMetricsJob' - ;; - "course-enrollment-report") echo 'org.sunbird.analytics.job.report.CourseEnrollmentJob' - ;; - "course-consumption-report") echo 'org.sunbird.analytics.job.report.CourseConsumptionJob' - ;; - "admin-geo-reports") echo 'org.sunbird.analytics.job.report.StateAdminGeoReportJob' - ;; - "etb-metrics") echo 'org.sunbird.analytics.job.report.ETBMetricsJob' - ;; - "admin-user-reports") echo 'org.sunbird.analytics.job.report.StateAdminReportJob' - ;; - "textbook-progress-report") echo 'org.sunbird.analytics.job.report.TextbookProgressJob' - ;; - *) echo $1 - ;; - esac -} - -if [ ! -z "$1" ]; then job_id=$(get_report_job_model_name $1); fi - -if [ -z "$2" ]; then job_config=$(config $1); else job_config="$2"; fi +mongo_host={{ mongo_node_bb_host }} +if [ ! -z "$1" ]; then job_id=$1; fi + +if [ ! -z "$1" ]; then job_config=$(config $1 $2); else job_config="$2"; fi echo "Starting the job - $1" >> "$DP_LOGS/$today-job-execution.log" echo "Job modelName - $job_id" >> "$DP_LOGS/$today-job-execution.log" -nohup $SPARK_HOME/bin/spark-submit --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.3.2.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "$job_id" --config "$job_config" >> "$DP_LOGS/$today-job-execution.log" 2>&1 +nohup $SPARK_HOME/bin/spark-submit --master local[*] --conf spark.sql.caseSensitive=true --conf spark.mongodb.input.uri=mongodb://$mongo_host/Nodebb.Objects --jars $MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.4.0.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "$job_id" --config "$job_config" >> "$DP_LOGS/$today-job-execution.log" 2>&1 echo "Job execution completed - $1" >> "$DP_LOGS/$today-job-execution.log" diff --git a/ansible/roles/data-products-deploy/templates/start-jobmanager.j2 b/ansible/roles/data-products-deploy/templates/start-jobmanager.j2 index bce73c9339..82c269781e 100644 --- a/ansible/roles/data-products-deploy/templates/start-jobmanager.j2 +++ b/ansible/roles/data-products-deploy/templates/start-jobmanager.j2 @@ -5,18 +5,13 @@ export DP_LOGS={{ analytics.home }}/logs/data-products export SERVICE_LOGS={{ analytics.home }}/logs/services export JM_HOME={{ analytics.home }}/job-manager -export azure_storage_key={{sunbird_private_storage_account_name}} -export azure_storage_secret={{sunbird_private_storage_account_key}} -export reports_azure_storage_key={{sunbird_private_storage_account_name}} -export reports_azure_storage_secret={{sunbird_private_storage_account_key}} - export heap_conf_str={{ spark.heap_conf_str }} today=$(date "+%Y-%m-%d") kill_job_manager() { echo "Killing currently running job-manager process" >> "$SERVICE_LOGS/$today-job-manager.log" - kill $(ps aux | grep '[j]ob-manager' | awk '{print $2}') >> "$SERVICE_LOGS/$today-job-manager.log" + kill $(ps aux | grep 'JobManager' | awk '{print $2}') >> "$SERVICE_LOGS/$today-job-manager.log" } start_job_manager() @@ -29,7 +24,7 @@ start_job_manager() echo "config: $job_config" >> "$SERVICE_LOGS/$today-job-manager.log" nohup java $heap_conf_str -cp "$SPARK_HOME/jars/*:$MODELS_HOME/*:$MODELS_HOME/data-products-1.0/lib/*" -Dconfig.file=$MODELS_HOME/{{ env }}.conf org.ekstep.analytics.job.JobManager --config "$job_config" >> $SERVICE_LOGS/$today-job-manager.log 2>&1 & - job_manager_pid=$(ps aux | grep '[j]ob-manager' | awk '{print $2}') # Once Job is started just we are making whether job is running or not. + job_manager_pid=$(ps aux | grep 'JobManager' | awk '{print $2}') # Once Job is started just we are making whether job is running or not. if [[ ! -z "$job_manager_pid" ]]; then echo "Job manager is started." >> "$SERVICE_LOGS/$today-job-manager.log" else diff --git a/ansible/roles/data-products-deploy/templates/submit-all-jobs.rb.j2 b/ansible/roles/data-products-deploy/templates/submit-all-jobs.rb.j2 index f16e51cc49..53c032cd29 100644 --- a/ansible/roles/data-products-deploy/templates/submit-all-jobs.rb.j2 +++ b/ansible/roles/data-products-deploy/templates/submit-all-jobs.rb.j2 @@ -14,8 +14,8 @@ end def submit_all_jobs report_jobs = { - "assessment-dashboard-metrics" => "org.sunbird.analytics.job.report.AssessmentMetricsJob", - "course-dashboard-metrics" => "org.sunbird.analytics.job.report.CourseMetricsJob", + "assessment-dashboard-metrics" => "org.sunbird.analytics.job.report.AssessmentMetricsJobV2", + "course-dashboard-metrics" => "org.sunbird.analytics.job.report.CourseMetricsJobV2", "course-enrollment-report" => "org.sunbird.analytics.job.report.CourseEnrollmentJob", "course-consumption-report" => "org.sunbird.analytics.job.report.CourseConsumptionJob", "etb-metrics" => "org.sunbird.analytics.job.report.ETBMetricsJob", diff --git a/ansible/roles/data-products-deploy/templates/submit-script.j2 b/ansible/roles/data-products-deploy/templates/submit-script.j2 new file mode 100644 index 0000000000..e8341dc1e8 --- /dev/null +++ b/ansible/roles/data-products-deploy/templates/submit-script.j2 @@ -0,0 +1,183 @@ +#!/usr/bin/env bash + +## Job to run daily +cd "{{ analytics_cluster.home }}" +source model-config.sh +today=$(date "+%Y-%m-%d") + +while :; do + case $1 in + -j|--job) shift + job="$1" + ;; + -m|--mode) shift + mode="$1" + ;; + -p|--parallelisation) shift + parallelisation=$1 + ;; + -pa|--partitions) shift + partitions=$1 + ;; + -sd|--startDate) shift + start_date=$1 + ;; + -ed|--endDate) shift + end_date=$1 + ;; + -h|--sparkMaster) shift + sparkMaster=$1 + ;; + -sp|--selectedPartitions) shift + selected_partitions=$1 + ;; + *) break + esac + shift +done + +get_report_job_model_name(){ + case "$1" in + "assessment-dashboard-metrics") echo 'org.sunbird.analytics.job.report.AssessmentMetricsJobV2' + ;; + "course-dashboard-metrics") echo 'org.sunbird.analytics.job.report.CourseMetricsJobV2' + ;; + "userinfo-exhaust") echo 'org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob' + ;; + "response-exhaust") echo 'org.sunbird.analytics.exhaust.collection.ResponseExhaustJob' + ;; + "response-exhaust-v2") echo 'org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2' + ;; + "progress-exhaust") echo 'org.sunbird.analytics.exhaust.collection.ProgressExhaustJob' + ;; + "progress-exhaust-v2") echo 'org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2' + ;; + "cassandra-migration") echo 'org.sunbird.analytics.updater.CassandraMigratorJob' + ;; + "uci-private-exhaust") echo 'org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob' + ;; + "uci-response-exhaust") echo 'org.sunbird.analytics.exhaust.uci.UCIResponseExhaustJob' + ;; + *) echo $1 + ;; + esac +} + +submit_cluster_job() { + # add batch number to config + echo "Running for below batch number $i" + batchNumberString="\\\"modelParams\\\":{\\\"batchNumber\\\":$i," + job_config=$(config $job) + cluster_job_config=${job_config//'"'/'\"'} + finalConfig=${cluster_job_config/'\"modelParams\":{'/$batchNumberString} + echo $finalConfig + echo "Running $job as parallel jobs" + classVariable="org.ekstep.analytics.job.JobExecutor" + argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\"]" + argsStr="\"className\": \"org.ekstep.analytics.job.JobExecutor\", $argsList" + clusterConfig=`cat cluster-config.json` + requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr} + finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable} + echo $finalRequestBody + response=$(curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: admin_name }}") + echo "Submitted job for batchNumer $i below is the response" + echo $response +} + +job_id=$(get_report_job_model_name $job) + +if [ -z "$sparkMaster" ]; then sparkMaster="local[*]"; else sparkMaster="$sparkMaster"; fi + +if [ "$mode" = "via-partition" ]; then + endPartitions=`expr $partitions - 1` + if [ -z "$parallelisation" ]; then parallelisation=1; else parallelisation=$parallelisation; fi + # add partitions to config and start jobs + for i in $(seq 0 $parallelisation $endPartitions) + do + # add partitions to config + partitionString="\\\"delta\\\":0,\\\"partitions\\\":[$(seq -s , $i `expr $i + $parallelisation - 1`)]" + if [ -z "$start_date" ]; then + job_config=$(config $job) + cluster_job_config=${job_config//'"'/'\"'} + finalConfig=${cluster_job_config/'\"delta\":0'/$partitionString} + echo $finalConfig + echo "Running $job by partitions." + classVariable="org.ekstep.analytics.job.JobExecutor" + argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\"]" + else + job_config=$(config $job '__endDate__') + cluster_job_config=${job_config//'"'/'\"'} + finalConfig=${cluster_job_config/'\"delta\":0'/$partitionString} + echo $finalConfig + echo "Running $job by partitions via Replay-Supervisor." + classVariable="org.ekstep.analytics.job.ReplaySupervisor" + argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\", \"--fromDate\", \"$start_date\", \"--toDate\", \"$end_date\"]" + fi + argsStr="\"className\": \"org.ekstep.analytics.job.JobExecutor\", $argsList" + clusterConfig=`cat cluster-config.json` + requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr} + finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable} + echo $finalRequestBody + curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}" + done + +elif [ "$mode" = "parallel-jobs" ]; then + # add batch number to config and submit jobs + echo "inside parallel-jobs block" + echo $parallelisation + if [ $parallelisation -ge 1 ]; then + for i in $(seq 1 $parallelisation) + do + submit_cluster_job $i & + done + else echo "No requests found in table"; fi + +elif [ "$mode" = "selected-partition" ]; then + # add partitions to config + partitionString="\\\"delta\\\":0,\\\"partitions\\\":[$selected_partitions]" + if [ -z "$start_date" ]; then + job_config=$(config $job) + cluster_job_config=${job_config//'"'/'\"'} + finalConfig=${cluster_job_config/'\"delta\":0'/$partitionString} + echo $finalConfig + echo "Running $job by partitions." + classVariable="org.ekstep.analytics.job.JobExecutor" + argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\"]" + else + job_config=$(config $job '__endDate__') + cluster_job_config=${job_config//'"'/'\"'} + finalConfig=${cluster_job_config/'\"delta\":0'/$partitionString} + echo $finalConfig + echo "Running $job by partitions via Replay-Supervisor." + classVariable="org.ekstep.analytics.job.ReplaySupervisor" + argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\", \"--fromDate\", \"$start_date\", \"--toDate\", \"$end_date\"]" + fi + argsStr="\"className\": \"org.ekstep.analytics.job.JobExecutor\", $argsList" + clusterConfig=`cat cluster-config.json` + requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr} + finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable} + echo $finalRequestBody + curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}" + +else + if [ -z "$start_date" ]; then + echo "Running $job without partition via run-job." + job_config=$(config $job) + cluster_job_config=${job_config//'"'/'\"'} + classVariable="org.ekstep.analytics.job.JobExecutor" + argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$cluster_job_config\"]" + else + job_config=$(config $job '__endDate__') + cluster_job_config=${job_config//'"'/'\"'} + echo "Running $job without partition via Replay-Supervisor." + classVariable="org.ekstep.analytics.job.ReplaySupervisor" + argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$cluster_job_config\", \"--fromDate\", \"$start_date\", \"--toDate\", \"$end_date\"]" + fi + argsStr="\"className\": \"org.ekstep.analytics.job.JobExecutor\", $argsList" + echo $argsStr + clusterConfig=`cat cluster-config.json` + requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr} + finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable} + echo $finalRequestBody + curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}" +fi diff --git a/ansible/roles/data-products-deploy/templates/update-job-requests.py.j2 b/ansible/roles/data-products-deploy/templates/update-job-requests.py.j2 new file mode 100644 index 0000000000..cfd986b008 --- /dev/null +++ b/ansible/roles/data-products-deploy/templates/update-job-requests.py.j2 @@ -0,0 +1,119 @@ +from __future__ import division +import math +import psycopg2 +import sys +import pandas as pd +from IPython.display import display +from psycopg2 import sql, connect +import json + + +def updateExhaustRequests(db, table, update_list): + for r in update_list: + cursor = db.cursor() + batchNum = r['batch_number'] + requestId = r['request_id'] + insertQry = "UPDATE {0} SET batch_number = {1} WHERE request_id = '{2}'".format(table, batchNum, requestId) + n = cursor.execute(insertQry) + +def updateDruidRequests(db, table, update_list): + for r in update_list: + cursor = db.cursor() + batchNum = r['batch_number'] + reportId = r['report_id'] + insertQry = "UPDATE {0} SET batch_number = {1} WHERE report_id = '{2}'".format(table, batchNum, reportId) + n = cursor.execute(insertQry) + +def processRequests(totalRequestsDf, jobId, batchSize, db, table,jobType): + # Compute parallelism from batchSize & totalRequests + # update batch_number to table + + totalRequests = len(totalRequestsDf.index) + print("totalRequests {0}".format(totalRequests)) + + parallelism = int(math.ceil(totalRequests/batchSize)) + print("parallelism computed {0}".format(parallelism)) + + if totalRequests > 0: + if jobType == 'exhaust': + totalRequestsDf["row_num"] = totalRequestsDf.groupby(by=['job_id'])['request_id'].transform(lambda x: x.rank()) + else: + totalRequestsDf["row_num"] = totalRequestsDf['report_id'].transform(lambda x: x.rank()) + #display(totalRequestsDf) + + start_index = 1 + end_index = batchSize + for i in range(1, parallelism+1): + subSetDf = totalRequestsDf[(totalRequestsDf['row_num'] >= start_index) & (totalRequestsDf['row_num'] <= end_index)] + subSetDf["batch_number"] = i + print(start_index,end_index) + if jobType == 'exhaust': + updateExhaustRequests(db, table, json.loads(subSetDf.to_json(orient='records'))) + else: + updateDruidRequests(db, table, json.loads(subSetDf.to_json(orient='records'))) + start_index = 1 + end_index + end_index = end_index + batchSize + db.commit() + db.close() + return parallelism + else: + return 0 + +def postgresql_to_dataframe(db, select_query, column_names): + cursor = db.cursor() + try: + cursor.execute(select_query) + except (Exception, psycopg2.DatabaseError) as error: + print("Error: %s" % error) + return 1 + + tupples = cursor.fetchall() + + df = pd.DataFrame(tupples, columns=column_names) + #display(df) + return df + +def get_columns_names(db,table): + columns = [] + col_cursor = db.cursor() + col_names_str = "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE " + col_names_str += "table_name = '{}';".format( table ) + try: + sql_object = sql.SQL(col_names_str).format(sql.Identifier( table)) + col_cursor.execute( sql_object ) + col_names = (col_cursor.fetchall()) + for tup in col_names: + columns += [ tup[0] ] + col_cursor.close() + except Exception as err: + print ("get_columns_names ERROR:", err) + return columns + +def main(batchSize, jobId,jobType,table): + host="{{postgres.db_url}}" + port={{postgres.db_port}} + user="{{postgres.db_username}}" + password="{{postgres.db_password}}" + database="{{postgres.db_name}}" + url_connect = "jdbc:postgresql://{0}:{1}/{2}".format(host, port, database) + + db = psycopg2.connect(host=host, user=user, password=password, database=database, port=port) + + column_names = get_columns_names(db, table) + + if jobType == 'exhaust': + jobId = jobId.split("-v2")[0] if "-v2" in jobId else jobId + selectQuery = "select * from {0} where job_id = '{1}' and status IN ('SUBMITTED', 'FAILED') and iteration < 3;".format(table, jobId) + else: + selectQuery = "select * from {0} where status IN ('ACTIVE')".format(table) + df = postgresql_to_dataframe(db, selectQuery, column_names) + + parallelism = processRequests(df, jobId, batchSize, db, table,jobType) + return parallelism + +batchSize =int(sys.argv[2]) +jobId=sys.argv[1] +jobType = sys.argv[3] +table = sys.argv[4] +parallelism = main(batchSize, jobId,jobType,table) +print("returning parallelism value: {0}".format(parallelism)) diff --git a/ansible/roles/data-products-fetch-logs/tasks/main.yml b/ansible/roles/data-products-fetch-logs/tasks/main.yml index 7655051a8b..4c442853f5 100644 --- a/ansible/roles/data-products-fetch-logs/tasks/main.yml +++ b/ansible/roles/data-products-fetch-logs/tasks/main.yml @@ -53,7 +53,7 @@ - name: Run Job become: yes become_user: "{{ analytics_user }}" - shell: "nohup {{ analytics.home }}/scripts/run-job.sh {{ job_id }} &" + shell: "nohup {{ analytics.home }}/scripts/run-job.sh {{ job_id }} {{ batch_id }} {{ keyword }} &" async: "{{ (pause_min * 60) }}" poll: 0 tags: diff --git a/ansible/roles/data-replay/tasks/main.yml b/ansible/roles/data-replay/tasks/main.yml index e546eee536..d40ea9dd9b 100644 --- a/ansible/roles/data-replay/tasks/main.yml +++ b/ansible/roles/data-replay/tasks/main.yml @@ -35,7 +35,7 @@ # - common - name: Install required python modules - pip: + pip: virtualenv: "{{ virtualenv_path }}" virtualenv_python: "python3.6" requirements: "{{ replay_script_path }}/replay-scripts/{{ requirements_file }}" @@ -50,7 +50,6 @@ environment: ENV: "{{ env }}" SPARK_HOME: "{{ spark_home }}" - JAVA_HOME: "/opt/jdk1.8.0_121" - PATH: "{{ lookup('env','PATH') }}:{{ spark_home }}/bin:/opt/jdk1.8.0_121/bin" + PATH: "{{ lookup('env','PATH') }}:{{ spark_home }}/bin" tags: - replay \ No newline at end of file diff --git a/ansible/roles/druid-anomaly-detection/defaults/main.yml b/ansible/roles/druid-anomaly-detection/defaults/main.yml new file mode 100644 index 0000000000..9828fc2a92 --- /dev/null +++ b/ansible/roles/druid-anomaly-detection/defaults/main.yml @@ -0,0 +1,22 @@ +rollup_druid_broker_host: "http://{{ groups['rollup-broker'][0] | default(groups['raw-broker'][0]) }}" +supervisord_process_name: anomaly_detection +anomaly_detection_module_artifact: anomaly_detection.tar.gz +anomaly_detection_module_path: /home/analytics/anomaly_detection +virtualenv_path: "{{ anomaly_detection_module_path }}/anomaly_venv" +anomaly_detection_job_config_file: "{{virtualenv_path}}/config.json" +anomaly_detection_data_dir: "/home/analytics/anomaly_detection/data" +anomaly_detection_queries: + content-mimetype-errors: + job_id: "content-mimetype-errors" + query_granularity: "hour" + query: 'WITH \"content_plays\" AS (SELECT \"content_mimetype\",\"context_pdata_id\",sum(total_count) AS \"plays\" FROM \"druid\".\"telemetry-hourly-rollup-syncts\" WHERE \"__time\" BETWEEN TIMESTAMP ''{start_time}'' AND TIMESTAMP ''{end_time}'' AND \"edata_type\" = ''content'' AND \"edata_mode\" = ''play'' GROUP BY \"content_mimetype\",\"context_pdata_id\") SELECT cp.content_mimetype, cp.context_pdata_id, SUM(cpe.errors) * 100.0/SUM(cp.plays) AS \"perc_errors\" FROM (SELECT \"content_mimetype\", \"context_pdata_id\",SUM(\"total_count\") AS \"errors\" FROM \"druid\".\"error-hourly-rollup-syncts\" WHERE \"__time\" BETWEEN TIMESTAMP ''{start_time}'' AND TIMESTAMP ''{end_time}'' GROUP BY \"content_mimetype\",\"context_pdata_id\") AS cpe INNER JOIN content_plays AS cp ON cp.content_mimetype = cpe.content_mimetype AND cp.context_pdata_id = cpe.context_pdata_id GROUP BY cp.content_mimetype, cp.context_pdata_id' + frequency: "hour" + cron: "0 */1 * * *" + offset_in_mins: 5 + collection-type-errors: + job_id: "collection-type-errors" + query_granularity: "hour" + query: 'WITH \"content_plays\" AS (SELECT \"collection_type\", \"context_pdata_id\",sum(total_count) AS \"plays\" FROM \"druid\".\"telemetry-hourly-rollup-syncts\" WHERE \"__time\" BETWEEN TIMESTAMP ''{start_time}'' AND TIMESTAMP ''{end_time}'' AND \"edata_type\" = ''content'' AND \"edata_mode\" = ''play'' GROUP BY \"collection_type\",\"context_pdata_id\") SELECT cp.collection_type,cp.context_pdata_id, SUM(cpe.errors) * 100.0/SUM(cp.plays) AS \"perc_errors\" FROM (SELECT \"collection_type\", \"context_pdata_id\",SUM(\"total_count\") AS \"errors\" FROM \"druid\".\"error-hourly-rollup-syncts\" WHERE \"__time\" BETWEEN TIMESTAMP ''{start_time}'' AND TIMESTAMP ''{end_time}'' GROUP BY \"collection_type\",\"context_pdata_id\") AS cpe INNER JOIN content_plays AS cp ON cp.collection_type = cpe.collection_type AND cp.context_pdata_id = cpe.context_pdata_id GROUP BY cp.collection_type, cp.context_pdata_id' + frequency: "hour" + cron: "0 */1 * * *" + offset_in_mins: 5 \ No newline at end of file diff --git a/ansible/roles/druid-anomaly-detection/tasks/main.yml b/ansible/roles/druid-anomaly-detection/tasks/main.yml new file mode 100644 index 0000000000..91e9b2e6ae --- /dev/null +++ b/ansible/roles/druid-anomaly-detection/tasks/main.yml @@ -0,0 +1,67 @@ +--- + +- name: Install supervisord + apt: + name: supervisor + state: present + update_cache: yes + become: true + +- name: copy supervisor conf + template: src="supervisor.conf.j2" dest="/etc/supervisor/conf.d/anomaly_detection.conf" + become: true + +- name: Start supervisord + service: + name: "supervisor" + state: started + enabled: yes + become: true + +- name: Add deadsnakes repo + apt_repository: repo="ppa:deadsnakes/ppa" + become: true + +- name: Install python + apt: + name: + - python3.8-dev + - python3-pip + - python3-setuptools + - python3.8-venv + state: present + become: true + +- name: create module directories + file: + path: "{{ item }}" + state: directory + owner: "{{ analytics_user }}" + group: "{{ analytics_user }}" + mode: 0775 + loop: + - "{{ anomaly_detection_module_path }}/anomaly_venv" + - "{{ anomaly_detection_module_path }}/data" + become_user: "{{ analytics_user }}" + +- name: Copy the anomaly detection artifact + copy: src={{ anomaly_detection_module_artifact }} dest={{ anomaly_detection_module_path }} + become_user: "{{ analytics_user }}" + +- name: Installing anomaly detection as a python module + pip: + name: "{{ anomaly_detection_module_path }}/{{ anomaly_detection_module_artifact }}" + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: "/usr/bin/python3.8 -m venv" + become_user: "{{ analytics_user }}" + +- name: copy query config to config.json + template: src="config.json.j2" dest="{{ virtualenv_path }}/config.json" + with_items: "{{ anomaly_detection_queries | list }}" + become_user: "{{ analytics_user }}" + +- name: Start anomaly detection process + supervisorctl: + name: "anomaly_detection" + state: restarted + become: true \ No newline at end of file diff --git a/ansible/roles/druid-anomaly-detection/templates/config.json.j2 b/ansible/roles/druid-anomaly-detection/templates/config.json.j2 new file mode 100644 index 0000000000..0faf6c00ad --- /dev/null +++ b/ansible/roles/druid-anomaly-detection/templates/config.json.j2 @@ -0,0 +1,14 @@ +{ + "jobs": [ + {% for item in anomaly_detection_queries.values() %} + { + "job_id": "{{ item.job_id }}", + "query_granularity": "{{ item.query_granularity }}", + "query": "{{ item.query }}", + "cron": "{{ item.cron }}", + "offset_in_mins": {{ item.offset_in_mins }}, + "frequency": "{{ item.frequency }}" + }{{ "," if not loop.last else "" }} + {% endfor %} + ] +} \ No newline at end of file diff --git a/ansible/roles/druid-anomaly-detection/templates/supervisor.conf.j2 b/ansible/roles/druid-anomaly-detection/templates/supervisor.conf.j2 new file mode 100644 index 0000000000..2541c43e82 --- /dev/null +++ b/ansible/roles/druid-anomaly-detection/templates/supervisor.conf.j2 @@ -0,0 +1,10 @@ +[program:{{ supervisord_process_name }}] +process_name={{ supervisord_process_name }} +command={{ virtualenv_path }}/bin/anomaly_detection druid_anomaly_detection --config_file={{ anomaly_detection_job_config_file }} --data_dir="{{ anomaly_detection_data_dir }}" --druid_broker_host="{{ rollup_druid_broker_host }}" --druid_broker_port=8082 +environment=PATH="{{ virtualenv_path }}/bin:%(ENV_PATH)s" +user={{analytics_user}} +group={{analytics_user}} +killasgroup=true +startsecs=5 +stopwaitsecs=5 +redirect_stderr=True diff --git a/ansible/roles/druid-ingestion/defaults/main.yml b/ansible/roles/druid-ingestion/defaults/main.yml index 2960c95a7b..feceb11844 100644 --- a/ansible/roles/druid-ingestion/defaults/main.yml +++ b/ansible/roles/druid-ingestion/defaults/main.yml @@ -1,17 +1,48 @@ remote_user: druid +remote_group: users broker_port: 8082 druid_home: /home/druid -druid_schema: - - telemetry_index_kafka - - summary_index_kafka - - log_index_kafka - - telemetry_feedback_index_kafka - - error_index_kafka - - pipeline_metrics_index_kafka -event_types: - - telemetry-events - - summary-events - - telemetry-log-events - - telemetry-feedback-events - - telemetry-error-events - - pipeline-metrics +desktop_pdata_id: "{{env}}.sunbird.desktop" + +druid_mm_heap_dump_file: /var/log/druid/crash_logs/middlemanager.hprof + +druid_overlord_port: 8090 + +ingestion_spec_configs: + raw_telemetry_events: + minTaskMemory: 512m + maxTaskMemory: 1024m + maxDirectTaskMemory: 1024m + +raw_offline_desktop_taskcount: 1 +raw_summary_events_taskcount: 1 +raw_telemetry_events_taskcount: 1 +raw_cb_audit_events_taskcount: 1 +raw_dashboards_user_course_program_progress_taskcount: 1 +raw_dashboards_user_assessment_taskcount: 1 +raw_dashboards_assessment_taskcount: 1 +raw_dashboards_user_org_taskcount: 1 +raw_dashboards_org_taskcount: 1 +raw_dashboards_course_taskcount: 1 +raw_dashboards_role_count_taskcount: 1 +raw_dashboards_org_role_count_taskcount: 1 +raw_dashboards_course_competency_taskcount: 1 +raw_dashboards_declared_user_competency_taskcount: 1 +raw_dashboards_expected_user_competency_taskcount: 1 +raw_dashboards_frac_competency_taskcount: 1 +raw_dashboards_user_competency_gap_taskcount: 1 +raw_dashboards_acbp_enrolment_taskcount: 1 +raw_telemetry_feedback_events_taskcount: 1 +rollup_error_hourly_syncts_taskcount: 1 +rollup_summary_distinct_counts_taskcount: 1 +rollup_summary_syncts_taskcount: 1 +rollup_telemetry_audit_syncts_taskcount: 1 +rollup_telemetry_hourly_syncts_taskcount: 1 +rollup_telemetry_syncts_taskcount: 1 +rollup_tpd_hourly_taskcount: 1 +sl_survey_evidence_task_count: 1 +sl_observation_task_count: 1 +sl_observation_evidence_task_count: 1 +sl_survey_task_count: 1 +rollup_course_completion_audit_syncts_taskcount: 1 +rollup_ml_project_taskcount: 1 diff --git a/ansible/roles/druid-ingestion/tasks/main.yml b/ansible/roles/druid-ingestion/tasks/main.yml index 199345fd4e..da8e8c5899 100644 --- a/ansible/roles/druid-ingestion/tasks/main.yml +++ b/ansible/roles/druid-ingestion/tasks/main.yml @@ -1,21 +1,35 @@ -- name: Copy the schema file - template: src="{{item}}" dest="{{druid_home}}/{{item}}.json" - with_items: "{{druid_schema}}" +- name: Load the context json + set_fact: {"{{ item }}_context":"{{ lookup('template', 'context.json.j2')}}"} + with_items: "{{ ingestion_spec_configs }}" tags: start-task +- name: Load the ingestion json + set_fact: {"{{ item }}_json":"{{ lookup('template', (item))}}"} + with_items: "{{ ingestion_task_names.split(',')|list }}" + tags: start-task + +- name: Override the schema spec with context and Copy schema file + copy: + content: "{{ (lookup('vars', item + '_context' , default={}) | combine(vars[(item + '_json')])) | to_json(indent=4) }}" + dest: "{{druid_home}}/{{item}}.json" + become_user: "{{remote_user}}" + with_items: "{{ ingestion_task_names.split(',')|list }}" + tags: start-task + + - name: Start kafka supervisor tasks for defined datasources - shell: 'curl -X POST -H "Content-Type:application/json" -d @{{druid_home}}/{{item}}.json http://{{overlord_host}}:8090/druid/indexer/v1/supervisor' - with_items: "{{druid_schema}}" + shell: 'curl -X POST -H "Content-Type:application/json" -d @{{druid_home}}/{{item}}.json http://{{overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/supervisor' + with_items: "{{ ingestion_task_names.split(',')|list }}" become_user: "{{remote_user}}" tags: start-task - name: Stop kafka supervisor tasks for defined datasources - shell: 'curl -X POST http://{{overlord_host}}:8090/druid/indexer/v1/supervisor/{{item}}/terminate' - with_items: "{{event_types}}" + shell: 'curl -X POST http://{{overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/supervisor/{{item}}/terminate' + with_items: "{{ ingestion_task_names.split(',')|list }}" tags: stop-task - name: Stop currently running ingestion tasks - shell: 'curl -XPOST http://{{overlord_host}}:8090/druid/indexer/v1/datasources/{{item}}/shutdownAllTasks' - with_items: "{{event_types}}" + shell: 'curl -XPOST http://{{overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/datasources/{{item}}/shutdownAllTasks' + with_items: "{{ ingestion_task_names.split(',')|list }}" tags: stop-task diff --git a/ansible/roles/druid-ingestion/templates/context.json.j2 b/ansible/roles/druid-ingestion/templates/context.json.j2 new file mode 100644 index 0000000000..11fc87a2f0 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/context.json.j2 @@ -0,0 +1 @@ +{"context":{"druid.indexer.runner.javaOpts":"-server -Xms{{ ingestion_spec_configs[item].minTaskMemory }} -Xmx{{ ingestion_spec_configs[item].maxTaskMemory }} -XX:MaxDirectMemorySize={{ ingestion_spec_configs[item].maxDirectTaskMemory }} -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"}} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/error_index_kafka b/ansible/roles/druid-ingestion/templates/error_index_kafka deleted file mode 100644 index cef137dc42..0000000000 --- a/ansible/roles/druid-ingestion/templates/error_index_kafka +++ /dev/null @@ -1,420 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "telemetry-error-events", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "flattenSpec": { - "useFieldDiscovery": false, - "fields": [ - { - "type": "root", - "name": "eid" - }, - { - "type": "root", - "name": "ets" - }, - { - "type": "root", - "name": "@timestamp" - }, - { - "type": "root", - "name": "mid" - }, - { - "type": "root", - "name": "ver" - }, - { - "type": "path", - "name": "actor_id", - "expr": "$.actor.id" - }, - { - "type": "path", - "name": "actor_type", - "expr": "$.actor.type" - }, - { - "type": "path", - "name": "context_channel", - "expr": "$.context.channel" - }, - { - "type": "path", - "name": "context_pdata_id", - "expr": "$.context.pdata.id" - }, - { - "type": "path", - "name": "context_pdata_pid", - "expr": "$.context.pdata.pid" - }, - { - "type": "path", - "name": "context_pdata_ver", - "expr": "$.context.pdata.ver" - }, - { - "type": "path", - "name": "context_env", - "expr": "$.context.env" - }, - { - "type": "path", - "name": "context_sid", - "expr": "$.context.sid" - }, - { - "type": "path", - "name": "context_did", - "expr": "$.context.did" - }, - { - "type": "path", - "name": "context_cdata_type", - "expr": "$.context.cdata[*].type" - }, - { - "type": "path", - "name": "context_cdata_id", - "expr": "$.context.cdata[*].id" - }, - { - "type": "path", - "name": "context_rollup_l1", - "expr": "$.context.rollup.l1" - }, - { - "type": "path", - "name": "context_rollup_l2", - "expr": "$.context.rollup.l2" - }, - { - "type": "path", - "name": "context_rollup_l3", - "expr": "$.context.rollup.l3" - }, - { - "type": "path", - "name": "context_rollup_l4", - "expr": "$.context.rollup.l4" - }, - { - "type": "path", - "name": "object_id", - "expr": "$.object.id" - }, - { - "type": "path", - "name": "object_type", - "expr": "$.object.type" - }, - { - "type": "path", - "name": "object_version", - "expr": "$.object.ver" - }, - { - "type": "path", - "name": "object_rollup_l1", - "expr": "$.object.rollup.l1" - }, - { - "type": "path", - "name": "object_rollup_l2", - "expr": "$.object.rollup.l2" - }, - { - "type": "path", - "name": "object_rollup_l3", - "expr": "$.object.rollup.l3" - }, - { - "type": "path", - "name": "object_rollup_l4", - "expr": "$.object.rollup.l4" - }, - { - "type": "root", - "name": "tags" - }, - { - "type": "path", - "name": "edata_err", - "expr": "$.edata.err" - }, - { - "type": "path", - "name": "edata_errtype", - "expr": "$.edata.errtype" - }, - { - "type": "path", - "name": "edata_pageid", - "expr": "$.edata.pageid" - }, - { - "type": "path", - "name": "edata_object_id", - "expr": "$.edata.object.id" - }, - { - "type": "path", - "name": "edata_object_type", - "expr": "$.edata.object.type" - }, - { - "type": "path", - "name": "edata_object_ver", - "expr": "$.edata.object.ver" - }, - { - "type": "path", - "name": "edata_object_subtype", - "expr": "$.edata.object.subtype" - }, - { - "type": "path", - "name": "edata_object_name", - "expr": "$.edata.object.name" - }, - { - "type": "path", - "name": "edata_object_code", - "expr": "$.edata.object.code" - }, - { - "type": "path", - "name": "edata_object_parent_id", - "expr": "$.edata.object.parent.id" - }, - { - "type": "path", - "name": "edata_object_parent_type", - "expr": "$.edata.object.parent.type" - }, - { - "type": "path", - "name": "edata_plugin_id", - "expr": "$.edata.plugin.id" - }, - { - "type": "path", - "name": "edata_plugin_ver", - "expr": "$.edata.plugin.ver" - }, - { - "type": "path", - "name": "edata_plugin_category", - "expr": "$.edata.plugin.category" - } - ] - }, - "dimensionsSpec": { - "dimensions": [ - { - "type": "string", - "name": "eid" - }, - { - "type": "long", - "name": "ets" - }, - { - "type": "string", - "name": "@timestamp" - }, - { - "type": "string", - "name": "mid" - }, - { - "type": "string", - "name": "ver" - }, - { - "type": "string", - "name": "actor_id" - }, - { - "type": "string", - "name": "actor_type" - }, - { - "type": "string", - "name": "context_channel" - }, - { - "type": "string", - "name": "context_pdata_id" - }, - { - "type": "string", - "name": "context_pdata_pid" - }, - { - "type": "string", - "name": "context_pdata_ver" - }, - { - "type": "string", - "name": "context_env" - }, - { - "type": "string", - "name": "context_sid" - }, - { - "type": "string", - "name": "context_did" - }, - { - "type": "string", - "name": "context_cdata_type" - }, - { - "type": "string", - "name": "context_cdata_id" - }, - { - "type": "string", - "name": "context_rollup_l1" - }, - { - "type": "string", - "name": "context_rollup_l2" - }, - { - "type": "string", - "name": "context_rollup_l3" - }, - { - "type": "string", - "name": "context_rollup_l4" - }, - { - "type": "string", - "name": "object_id" - }, - { - "type": "string", - "name": "object_type" - }, - { - "type": "string", - "name": "object_version" - }, - { - "type": "string", - "name": "object_rollup_l1" - }, - { - "type": "string", - "name": "object_rollup_l2" - }, - { - "type": "string", - "name": "object_rollup_l3" - }, - { - "type": "string", - "name": "object_rollup_l4" - }, - { - "type": "string", - "name": "tags" - }, - { - "type": "string", - "name": "edata_err" - }, - { - "type": "string", - "name": "edata_errtype" - }, - { - "type": "string", - "name": "edata_pageid" - }, - { - "type": "string", - "name": "edata_object_id" - }, - { - "type": "string", - "name": "edata_object_type" - }, - { - "type": "string", - "name": "edata_object_ver" - }, - { - "type": "string", - "name": "edata_object_subtype" - }, - { - "type": "string", - "name": "edata_object_name" - }, - { - "type": "string", - "name": "edata_object_code" - }, - { - "type": "string", - "name": "edata_object_parent_id" - }, - { - "type": "string", - "name": "edata_object_parent_type" - }, - { - "type": "string", - "name": "edata_plugin_id" - }, - { - "type": "string", - "name": "edata_plugin_ver" - }, - { - "type": "string", - "name": "edata_plugin_category" - } - ] - }, - "timestampSpec": { - "column": "syncts", - "format": "auto" - } - } - }, - "metricsSpec": [], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "day", - "queryGranularity": "none", - "rollup": false - } - }, - "ioConfig": { - "topic": "{{env}}.events.error", - "consumerProperties": { - "bootstrap.servers": "{{kafka_brokers}}" - }, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT14400S", - "useEarliestOffset": false - }, - "tuningConfig": { - "type": "kafka", - "reportParseExceptions": false, - "logParseExceptions": true, - "maxSavedParseExceptions": 10 - } -} diff --git a/ansible/roles/druid-ingestion/templates/log_index_kafka b/ansible/roles/druid-ingestion/templates/log_index_kafka deleted file mode 100644 index f8cbfdb066..0000000000 --- a/ansible/roles/druid-ingestion/templates/log_index_kafka +++ /dev/null @@ -1,465 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "telemetry-log-events", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "flattenSpec": { - "useFieldDiscovery": false, - "fields": [ - { - "type": "root", - "name": "eid" - }, - { - "type": "root", - "name": "syncts" - }, - { - "type": "root", - "name": "@timestamp" - }, - { - "type": "root", - "name": "mid" - }, - { - "type": "root", - "name": "ver" - }, - { - "type": "path", - "name": "actor_id", - "expr": "$.actor.id" - }, - { - "type": "path", - "name": "actor_type", - "expr": "$.actor.type" - }, - { - "type": "path", - "name": "context_channel", - "expr": "$.context.channel" - }, - { - "type": "path", - "name": "context_pdata_id", - "expr": "$.context.pdata.id" - }, - { - "type": "path", - "name": "context_pdata_pid", - "expr": "$.context.pdata.pid" - }, - { - "type": "path", - "name": "context_pdata_ver", - "expr": "$.context.pdata.ver" - }, - { - "type": "path", - "name": "context_env", - "expr": "$.context.env" - }, - { - "type": "path", - "name": "context_sid", - "expr": "$.context.sid" - }, - { - "type": "path", - "name": "context_did", - "expr": "$.context.did" - }, - { - "type": "path", - "name": "context_cdata_type", - "expr": "$.context.cdata[*].type" - }, - { - "type": "path", - "name": "context_cdata_id", - "expr": "$.context.cdata[*].id" - }, - { - "type": "path", - "name": "context_rollup_l1", - "expr": "$.context.rollup.l1" - }, - { - "type": "path", - "name": "context_rollup_l2", - "expr": "$.context.rollup.l2" - }, - { - "type": "path", - "name": "context_rollup_l3", - "expr": "$.context.rollup.l3" - }, - { - "type": "path", - "name": "context_rollup_l4", - "expr": "$.context.rollup.l4" - }, - { - "type": "path", - "name": "object_id", - "expr": "$.object.id" - }, - { - "type": "path", - "name": "object_type", - "expr": "$.object.type" - }, - { - "type": "path", - "name": "object_version", - "expr": "$.object.ver" - }, - { - "type": "path", - "name": "object_rollup_l1", - "expr": "$.object.rollup.l1" - }, - { - "type": "path", - "name": "object_rollup_l2", - "expr": "$.object.rollup.l2" - }, - { - "type": "path", - "name": "object_rollup_l3", - "expr": "$.object.rollup.l3" - }, - { - "type": "path", - "name": "object_rollup_l4", - "expr": "$.object.rollup.l4" - }, - { - "type": "root", - "name": "tags" - }, - { - "type": "path", - "name": "edata_type", - "expr": "$.edata.type" - }, - { - "type": "path", - "name": "edata_level", - "expr": "$.edata.level" - }, - { - "type": "path", - "name": "edata_message", - "expr": "$.edata.message" - }, - { - "type": "path", - "name": "edata_pageid", - "expr": "$.edata.pageid" - }, - { - "type": "path", - "name": "edata_params_rid", - "expr": "$.edata.params[*].rid" - }, - { - "type": "path", - "name": "edata_params_uip", - "expr": "$.edata.params[*].uip" - }, - { - "type": "path", - "name": "edata_params_title", - "expr": "$.edata.params[*].title" - }, - { - "type": "path", - "name": "edata_params_category", - "expr": "$.edata.params[*].category" - }, - { - "type": "path", - "name": "edata_params_url", - "expr": "$.edata.params[*].url" - }, - { - "type": "path", - "name": "edata_params_size", - "expr": "$.edata.params[*].size" - }, - { - "type": "path", - "name": "edata_params_duration", - "expr": "$.edata.params[*].duration" - }, - { - "type": "path", - "name": "edata_params_status", - "expr": "$.edata.params[*].status" - }, - { - "type": "path", - "name": "edata_params_protocol", - "expr": "$.edata.params[*].protocol" - }, - { - "type": "path", - "name": "edata_params_method", - "expr": "$.edata.params[*].method" - }, - { - "type": "path", - "name": "edata_params_req", - "expr": "$.edata.params[*].req" - }, - { - "type": "path", - "name": "edata_params_consumer_id", - "expr": "$.edata.params[*].consumer_id" - }, - { - "type": "path", - "name": "edata_params_ver", - "expr": "$.edata.params[*].ver" - }, - { - "type": "path", - "name": "edata_params_events_count", - "expr": "$.edata.params[*].events_count" - }, - { - "type": "path", - "name": "edata_params_", - "expr": "$.edata.params[*].sync_status" - } - ] - }, - "dimensionsSpec": { - "dimensions": [ - { - "type": "string", - "name": "eid" - }, - { - "type": "long", - "name": "syncts" - }, - { - "type": "string", - "name": "@timestamp" - }, - { - "type": "string", - "name": "mid" - }, - { - "type": "string", - "name": "ver" - }, - { - "type": "string", - "name": "actor_id" - }, - { - "type": "string", - "name": "actor_type" - }, - { - "type": "string", - "name": "context_channel" - }, - { - "type": "string", - "name": "context_pdata_id" - }, - { - "type": "string", - "name": "context_pdata_pid" - }, - { - "type": "string", - "name": "context_pdata_ver" - }, - { - "type": "string", - "name": "context_env" - }, - { - "type": "string", - "name": "context_sid" - }, - { - "type": "string", - "name": "context_did" - }, - { - "type": "string", - "name": "context_cdata_type" - }, - { - "type": "string", - "name": "context_cdata_id" - }, - { - "type": "string", - "name": "context_rollup_l1" - }, - { - "type": "string", - "name": "context_rollup_l2" - }, - { - "type": "string", - "name": "context_rollup_l3" - }, - { - "type": "string", - "name": "context_rollup_l4" - }, - { - "type": "string", - "name": "object_id" - }, - { - "type": "string", - "name": "object_type" - }, - { - "type": "string", - "name": "object_version" - }, - { - "type": "string", - "name": "object_rollup_l1" - }, - { - "type": "string", - "name": "object_rollup_l2" - }, - { - "type": "string", - "name": "object_rollup_l3" - }, - { - "type": "string", - "name": "object_rollup_l4" - }, - { - "type": "string", - "name": "tags" - }, - { - "type": "string", - "name": "edata_type" - }, - { - "type": "string", - "name": "edata_level" - }, - { - "type": "string", - "name": "edata_message" - }, - { - "type": "string", - "name": "edata_pageid" - }, - { - "type": "string", - "name": "edata_params_rid" - }, - { - "type": "string", - "name": "edata_params_uip" - }, - { - "type": "string", - "name": "edata_params_title" - }, - { - "type": "string", - "name": "edata_params_category" - }, - { - "type": "string", - "name": "edata_params_url" - }, - { - "type": "string", - "name": "edata_params_size" - }, - { - "type": "string", - "name": "edata_params_duration" - }, - { - "type": "string", - "name": "edata_params_status" - }, - { - "type": "string", - "name": "edata_params_protocol" - }, - { - "type": "string", - "name": "edata_params_method" - }, - { - "type": "string", - "name": "edata_params_req" - }, - { - "type": "string", - "name": "edata_params_consumer_id" - }, - { - "type": "string", - "name": "edata_params_ver" - }, - { - "type": "string", - "name": "edata_params_events_count" - }, - { - "type": "string", - "name": "edata_params_events_sync_status" - } - ] - }, - "timestampSpec": { - "column": "ets", - "format": "auto" - } - } - }, - "metricsSpec": [], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "day", - "queryGranularity": "none", - "rollup": false - } - }, - "ioConfig": { - "topic": "{{env}}.events.log", - "consumerProperties": { - "bootstrap.servers": "{{kafka_brokers}}" - }, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT14400S", - "useEarliestOffset": false - }, - "tuningConfig": { - "type": "kafka", - "reportParseExceptions": false, - "logParseExceptions": true, - "maxSavedParseExceptions": 10 - } -} diff --git a/ansible/roles/druid-ingestion/templates/offline_desktop_kafka b/ansible/roles/druid-ingestion/templates/offline_desktop_kafka deleted file mode 100644 index b1b050facb..0000000000 --- a/ansible/roles/druid-ingestion/templates/offline_desktop_kafka +++ /dev/null @@ -1,1353 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "offline-desktop-events", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "flattenSpec": { - "useFieldDiscovery": false, - "fields": [ - { - "type": "root", - "name": "eid" - }, - { - "type": "root", - "name": "mid" - }, - { - "type": "root", - "name": "syncts" - }, - { - "type": "root", - "name": "@timestamp" - }, - { - "type": "path", - "name": "actor_id", - "expr": "$.actor.id" - }, - { - "type": "path", - "name": "actor_type", - "expr": "$.actor.type" - }, - { - "type": "path", - "name": "context_channel", - "expr": "$.context.channel" - }, - { - "type": "path", - "name": "context_pdata_id", - "expr": "$.context.pdata.id" - }, - { - "type": "path", - "name": "context_pdata_pid", - "expr": "$.context.pdata.pid" - }, - { - "type": "path", - "name": "context_pdata_ver", - "expr": "$.context.pdata.ver" - }, - { - "type": "path", - "name": "context_env", - "expr": "$.context.env" - }, - { - "type": "path", - "name": "context_sid", - "expr": "$.context.sid" - }, - { - "type": "path", - "name": "context_did", - "expr": "$.context.did" - }, - { - "type": "path", - "name": "context_cdata_type", - "expr": "$.context.cdata[*].type" - }, - { - "type": "path", - "name": "context_cdata_id", - "expr": "$.context.cdata[*].id" - }, - { - "type": "path", - "name": "context_rollup_l1", - "expr": "$.context.rollup.l1" - }, - { - "type": "path", - "name": "context_rollup_l2", - "expr": "$.context.rollup.l2" - }, - { - "type": "path", - "name": "context_rollup_l3", - "expr": "$.context.rollup.l3" - }, - { - "type": "path", - "name": "context_rollup_l4", - "expr": "$.context.rollup.l4" - }, - { - "type": "path", - "name": "object_id", - "expr": "$.object.id" - }, - { - "type": "path", - "name": "object_type", - "expr": "$.object.type" - }, - { - "type": "path", - "name": "object_version", - "expr": "$.object.ver" - }, - { - "type": "path", - "name": "object_rollup_l1", - "expr": "$.object.rollup.l1" - }, - { - "type": "path", - "name": "object_rollup_l2", - "expr": "$.object.rollup.l2" - }, - { - "type": "path", - "name": "object_rollup_l3", - "expr": "$.object.rollup.l3" - }, - { - "type": "path", - "name": "object_rollup_l4", - "expr": "$.object.rollup.l4" - }, - { - "type": "root", - "name": "tags" - }, - { - "type": "path", - "name": "edata_type", - "expr": "$.edata.type" - }, - { - "type": "path", - "name": "edata_subtype", - "expr": "$.edata.subtype" - }, - { - "type": "path", - "name": "edata_mode", - "expr": "$.edata.mode" - }, - { - "type": "path", - "name": "edata_pageid", - "expr": "$.edata.pageid" - }, - { - "type": "path", - "name": "edata_uri", - "expr": "$.edata.uri" - }, - { - "type": "path", - "name": "edata_id", - "expr": "$.edata.id" - }, - { - "type": "path", - "name": "edata_duration", - "expr": "$.edata.duration" - }, - { - "type": "path", - "name": "edata_index", - "expr": "$.edata.index" - }, - { - "type": "path", - "name": "edata_pass", - "expr": "$.edata.pass" - }, - { - "type": "path", - "name": "edata_score", - "expr": "$.edata.score" - }, - { - "type": "path", - "name": "edata_resvalues", - "expr": "$.edata.resvalues[*]" - }, - { - "type": "path", - "name": "edata_item_id", - "expr": "$.edata.item.id" - }, - { - "type": "path", - "name": "edata_item_title", - "expr": "$.edata.item.title" - }, - { - "type": "path", - "name": "edata_item_maxscore", - "expr": "$.edata.item.maxscore" - }, - { - "type": "path", - "name": "edata_target_id", - "expr": "$.edata.target.id" - }, - { - "type": "path", - "name": "edata_target_type", - "expr": "$.edata.target.type" - }, - { - "type": "path", - "name": "edata_rating", - "expr": "$.edata.rating" - }, - { - "type": "path", - "name": "edata_comments", - "expr": "$.edata.comments" - }, - { - "type": "path", - "name": "edata_dir", - "expr": "$.edata.dir" - }, - { - "type": "path", - "name": "edata_items_id", - "expr": "$.edata.items[*].id" - }, - { - "type": "path", - "name": "edata_items_type", - "expr": "$.edata.items[*].type" - }, - { - "type": "path", - "name": "edata_items_origin_id", - "expr": "$.edata.items[*].origin.id" - }, - { - "type": "path", - "name": "edata_items_origin_type", - "expr": "$.edata.items[*].origin.type" - }, - { - "type": "path", - "name": "edata_items_to_id", - "expr": "$.edata.items[*].to.id" - }, - { - "type": "path", - "name": "edata_items_to_type", - "expr": "$.edata.items[*].to.type" - }, - { - "type": "path", - "name": "edata_plugin_id", - "expr": "$.edata.plugin.id" - }, - { - "type": "path", - "name": "edata_plugin_ver", - "expr": "$.edata.plugin.ver" - }, - { - "type": "path", - "name": "edata_plugin_category", - "expr": "$.edata.plugin.category" - }, - { - "type": "path", - "name": "edata_props", - "expr": "$.edata.props[*]" - }, - { - "type": "path", - "name": "edata_state", - "expr": "$.edata.state" - }, - { - "type": "path", - "name": "edata_prevstate", - "expr": "$.edata.prevstate" - }, - { - "type": "path", - "name": "edata_size", - "expr": "$.edata.size" - }, - { - "type": "path", - "name": "edata_filters_dialcodes", - "expr": "$.edata.filters.dialcodes" - }, - { - "type": "path", - "name": "edata_topn_identifier", - "expr": "$.edata.topn[*].identifier" - }, - { - "type": "path", - "name": "edata_visits_objid", - "expr": "$.edata.visits[*].objid" - }, - { - "type": "path", - "name": "edata_visits_objtype", - "expr": "$.edata.visits[*].objtype" - }, - { - "type": "path", - "name": "edata_visits_objver", - "expr": "$.edata.visits[*].objver" - }, - { - "type": "path", - "name": "edata_visits_index", - "expr": "$.edata.visits[*].index" - }, - { - "type": "path", - "name": "device_loc_state", - "expr": "$.devicedata.state" - }, - { - "type": "path", - "name": "device_loc_state_code", - "expr": "$.devicedata.statecode" - }, - { - "type": "path", - "name": "device_loc_city", - "expr": "$.devicedata.city" - }, - { - "type": "path", - "name": "device_loc_country_code", - "expr": "$.devicedata.countrycode" - }, - { - "type": "path", - "name": "device_loc_country", - "expr": "$.devicedata.country" - }, - { - "type": "path", - "name": "device_os", - "expr": "$.devicedata.devicespec.os" - }, - { - "type": "path", - "name": "device_make", - "expr": "$.devicedata.devicespec.make" - }, - { - "type": "path", - "name": "device_id", - "expr": "$.devicedata.devicespec.id" - }, - { - "type": "path", - "name": "device_mem", - "expr": "$.devicedata.devicespec.mem" - }, - { - "type": "path", - "name": "device_idisk", - "expr": "$.devicedata.devicespec.idisk" - }, - { - "type": "path", - "name": "device_edisk", - "expr": "$.devicedata.devicespec.edisk" - }, - { - "type": "path", - "name": "device_scrn", - "expr": "$.devicedata.devicespec.scrn" - }, - { - "type": "path", - "name": "device_camera", - "expr": "$.devicedata.devicespec.camera" - }, - { - "type": "path", - "name": "device_cpu", - "expr": "$.devicedata.devicespec.cpu" - }, - { - "type": "path", - "name": "device_sims", - "expr": "$.devicedata.devicespec.sims" - }, - { - "type": "path", - "name": "device_uaspec_agent", - "expr": "$.devicedata.uaspec.agent" - }, - { - "type": "path", - "name": "device_uaspec_ver", - "expr": "$.devicedata.uaspec.ver" - }, - { - "type": "path", - "name": "device_uaspec_system", - "expr": "$.devicedata.uaspec.system" - }, - { - "type": "path", - "name": "device_uaspec_platform", - "expr": "$.devicedata.uaspec.platform" - }, - { - "type": "path", - "name": "device_uaspec_raw", - "expr": "$.devicedata.uaspec.raw" - }, - { - "type": "path", - "name": "device_first_access", - "expr": "$.devicedata.firstaccess" - }, - { - "type": "path", - "name": "device_loc_state_custom_code", - "expr": "$.devicedata.statecustomcode" - }, - { - "type": "path", - "name": "device_loc_state_custom_name", - "expr": "$.devicedata.statecustomname" - }, - { - "type": "path", - "name": "device_loc_district", - "expr": "$.devicedata.districtcustom" - }, - { - "type": "path", - "name": "user_declared_state", - "expr": "$.devicedata.userdeclared.state" - }, - { - "type": "path", - "name": "user_declared_district", - "expr": "$.devicedata.userdeclared.district" - }, - { - "type": "path", - "name": "derived_loc_state", - "expr": "$.derivedlocationdata.state" - }, - { - "type": "path", - "name": "derived_loc_district", - "expr": "$.derivedlocationdata.district" - }, - { - "type": "path", - "name": "derived_loc_from", - "expr": "$.derivedlocationdata.from" - }, - { - "type": "path", - "name": "content_name", - "expr": "$.contentdata.name" - }, - { - "type": "path", - "name": "content_object_type", - "expr": "$.contentdata.objecttype" - }, - { - "type": "path", - "name": "content_type", - "expr": "$.contentdata.contenttype" - }, - { - "type": "path", - "name": "content_media_type", - "expr": "$.contentdata.mediatype" - }, - { - "type": "path", - "name": "content_language", - "expr": "$.contentdata.language[*]" - }, - { - "type": "path", - "name": "content_medium", - "expr": "$.contentdata.medium[*]" - }, - { - "type": "path", - "name": "content_gradelevel", - "expr": "$.contentdata.gradelevel[*]" - }, - { - "type": "path", - "name": "content_subjects", - "expr": "$.contentdata.subject[*]" - }, - { - "type": "path", - "name": "content_mimetype", - "expr": "$.contentdata.mimetype" - }, - { - "type": "path", - "name": "content_framework", - "expr": "$.contentdata.framework" - }, - { - "type": "path", - "name": "content_board", - "expr": "$.contentdata.board" - }, - { - "type": "path", - "name": "content_status", - "expr": "$.contentdata.status" - }, - { - "type": "path", - "name": "content_version", - "expr": "$.contentdata.pkgversion" - }, - { - "type": "path", - "name": "content_last_submitted_on", - "expr": "$.contentdata.lastsubmittedon" - }, - { - "type": "path", - "name": "content_last_published_on", - "expr": "$.contentdata.lastpublishedon" - }, - { - "type": "path", - "name": "content_last_updated_on", - "expr": "$.contentdata.lastupdatedon" - }, - { - "type": "path", - "name": "content_created_by", - "expr": "$.contentdata.createdby" - }, - { - "type": "path", - "name": "content_created_for", - "expr": "$.contentdata.createdfor" - }, - { - "type": "path", - "name": "collection_name", - "expr": "$.collectiondata.name" - }, - { - "type": "path", - "name": "collection_object_type", - "expr": "$.collectiondata.objecttype" - }, - { - "type": "path", - "name": "collection_type", - "expr": "$.collectiondata.contenttype" - }, - { - "type": "path", - "name": "collection_media_type", - "expr": "$.collectiondata.mediatype" - }, - { - "type": "path", - "name": "collection_language", - "expr": "$.collectiondata.language[*]" - }, - { - "type": "path", - "name": "collection_medium", - "expr": "$.collectiondata.medium[*]" - }, - { - "type": "path", - "name": "collection_gradelevel", - "expr": "$.collectiondata.gradelevel[*]" - }, - { - "type": "path", - "name": "collection_subjects", - "expr": "$.collectiondata.subject[*]" - }, - { - "type": "path", - "name": "collection_mimetype", - "expr": "$.collectiondata.mimetype" - }, - { - "type": "path", - "name": "collection_framework", - "expr": "$.collectiondata.framework" - }, - { - "type": "path", - "name": "collection_board", - "expr": "$.collectiondata.board" - }, - { - "type": "path", - "name": "collection_status", - "expr": "$.collectiondata.status" - }, - { - "type": "path", - "name": "collection_version", - "expr": "$.collectiondata.pkgversion" - }, - { - "type": "path", - "name": "collection_last_submitted_on", - "expr": "$.collectiondata.lastsubmittedon" - }, - { - "type": "path", - "name": "collection_last_published_on", - "expr": "$.collectiondata.lastpublishedon" - }, - { - "type": "path", - "name": "collection_last_updated_on", - "expr": "$.collectiondata.lastupdatedon" - }, - { - "type": "path", - "name": "collection_created_by", - "expr": "$.collectiondata.createdby" - }, - { - "type": "path", - "name": "collection_created_for", - "expr": "$.collectiondata.createdfor" - }, - { - "type": "path", - "name": "user_grade_list", - "expr": "$.userdata.gradelist[*]" - }, - { - "type": "path", - "name": "user_language_list", - "expr": "$.userdata.languagelist[*]" - }, - { - "type": "path", - "name": "user_subject_list", - "expr": "$.userdata.subjectlist[*]" - }, - { - "type": "path", - "name": "user_type", - "expr": "$.userdata.usertype" - }, - { - "type": "path", - "name": "user_roles", - "expr": "$.userdata.roles[*]" - }, - { - "type": "path", - "name": "user_loc_state", - "expr": "$.userdata.state" - }, - { - "type": "path", - "name": "user_loc_district", - "expr": "$.userdata.district" - }, - { - "type": "path", - "name": "user_signin_type", - "expr": "$.userdata.usersignintype" - }, - { - "type": "path", - "name": "user_login_type", - "expr": "$.userdata.userlogintype" - }, - { - "type": "path", - "name": "dialcode_channel", - "expr": "$.dialcodedata.channel" - }, - { - "type": "path", - "name": "dialcode_batchcode", - "expr": "$.dialcodedata.batchcode" - }, - { - "type": "path", - "name": "dialcode_publisher", - "expr": "$.dialcodedata.publisher" - }, - { - "type": "path", - "name": "dialcode_generated_on", - "expr": "$.dialcodedata.generatedon" - }, - { - "type": "path", - "name": "dialcode_published_on", - "expr": "$.dialcodedata.publishedon" - }, - { - "type": "path", - "name": "dialcode_status", - "expr": "$.dialcodedata.status" - }, - { - "type": "path", - "name": "dialcode_object_type", - "expr": "$.dialcodedata.objecttype" - }, - { - "type": "path", - "name": "device_loc_iso_state_code", - "expr": "$.devicedata.iso3166statecode" - } - ] - }, - "dimensionsSpec": { - "dimensions": [ - { - "type": "string", - "name": "eid" - }, - { - "type": "string", - "name": "mid" - }, - { - "type": "long", - "name": "syncts" - }, - { - "type": "string", - "name": "@timestamp" - }, - { - "type": "string", - "name": "actor_id" - }, - { - "type": "string", - "name": "actor_type" - }, - { - "type": "string", - "name": "context_channel" - }, - { - "type": "string", - "name": "context_pdata_id" - }, - { - "type": "string", - "name": "context_pdata_pid" - }, - { - "type": "string", - "name": "context_pdata_ver" - }, - { - "type": "string", - "name": "context_env" - }, - { - "type": "string", - "name": "context_sid" - }, - { - "type": "string", - "name": "context_did" - }, - { - "name": "context_cdata_type" - }, - { - "name": "context_cdata_id" - }, - { - "type": "string", - "name": "context_rollup_l1" - }, - { - "type": "string", - "name": "context_rollup_l2" - }, - { - "type": "string", - "name": "context_rollup_l3" - }, - { - "type": "string", - "name": "context_rollup_l4" - }, - { - "type": "string", - "name": "object_id" - }, - { - "type": "string", - "name": "object_type" - }, - { - "type": "string", - "name": "object_version" - }, - { - "type": "string", - "name": "object_rollup_l1" - }, - { - "type": "string", - "name": "object_rollup_l2" - }, - { - "type": "string", - "name": "object_rollup_l3" - }, - { - "type": "string", - "name": "object_rollup_l4" - }, - { - "name": "tags" - }, - { - "type": "string", - "name": "edata_type" - }, - { - "type": "string", - "name": "edata_subtype" - }, - { - "type": "string", - "name": "edata_mode" - }, - { - "type": "string", - "name": "edata_pageid" - }, - { - "type": "string", - "name": "edata_uri" - }, - { - "type": "string", - "name": "edata_id" - }, - { - "type": "double", - "name": "edata_duration" - }, - { - "type": "long", - "name": "edata_index" - }, - { - "type": "string", - "name": "edata_pass" - }, - { - "type": "double", - "name": "edata_score" - }, - { - "name": "edata_resvalues" - }, - { - "type": "string", - "name": "edata_item_id" - }, - { - "type": "string", - "name": "edata_item_title" - }, - { - "type": "double", - "name": "edata_item_maxscore" - }, - { - "type": "string", - "name": "edata_target_id" - }, - { - "type": "string", - "name": "edata_target_type" - }, - { - "type": "long", - "name": "edata_rating" - }, - { - "type": "string", - "name": "edata_comments" - }, - { - "type": "string", - "name": "edata_dir" - }, - { - "type": "string", - "name": "edata_items_id" - }, - { - "type": "string", - "name": "edata_items_type" - }, - { - "type": "string", - "name": "edata_items_origin_id" - }, - { - "type": "string", - "name": "edata_items_origin_type" - }, - { - "type": "string", - "name": "edata_items_to_id" - }, - { - "type": "string", - "name": "edata_items_to_type" - }, - { - "type": "string", - "name": "edata_plugin_id" - }, - { - "type": "string", - "name": "edata_plugin_ver" - }, - { - "type": "string", - "name": "edata_plugin_category" - }, - { - "name": "edata_props" - }, - { - "type": "string", - "name": "edata_state" - }, - { - "type": "string", - "name": "edata_prevstate" - }, - { - "type": "long", - "name": "edata_size" - }, - { - "name": "edata_filters_dialcodes" - }, - { - "type": "string", - "name": "edata_topn_identifier" - }, - { - "type": "string", - "name": "edata_visits_objid" - }, - { - "type": "string", - "name": "edata_visits_objtype" - }, - { - "type": "string", - "name": "edata_visits_objver" - }, - { - "type": "string", - "name": "edata_visits_index" - }, - { - "type": "string", - "name": "device_loc_state" - }, - { - "type": "string", - "name": "device_loc_state_code" - }, - { - "type": "string", - "name": "device_loc_city" - }, - { - "type": "string", - "name": "device_loc_country_code" - }, - { - "type": "string", - "name": "device_loc_country" - }, - { - "type": "string", - "name": "device_os" - }, - { - "type": "string", - "name": "device_make" - }, - { - "type": "string", - "name": "device_id" - }, - { - "type": "long", - "name": "device_mem" - }, - { - "type": "string", - "name": "device_idisk" - }, - { - "type": "string", - "name": "device_edisk" - }, - { - "type": "string", - "name": "device_scrn" - }, - { - "type": "string", - "name": "device_camera" - }, - { - "type": "string", - "name": "device_cpu" - }, - { - "type": "long", - "name": "device_sims" - }, - { - "type": "string", - "name": "device_uaspec_agent" - }, - { - "type": "string", - "name": "device_uaspec_ver" - }, - { - "type": "string", - "name": "device_uaspec_system" - }, - { - "type": "string", - "name": "device_uaspec_platform" - }, - { - "type": "string", - "name": "device_uaspec_raw" - }, - { - "type": "long", - "name": "device_first_access" - }, - { - "type": "string", - "name": "device_loc_state_custom_code" - }, - { - "type": "string", - "name": "device_loc_state_custom_name" - }, - { - "type": "string", - "name": "device_loc_district" - }, - { - "type": "string", - "name": "user_declared_state" - }, - { - "type": "string", - "name": "user_declared_district" - }, - { - "type": "string", - "name": "derived_loc_state" - }, - { - "type": "string", - "name": "derived_loc_district" - }, - { - "type": "string", - "name": "derived_loc_from" - }, - { - "type": "string", - "name": "content_name" - }, - { - "type": "string", - "name": "content_object_type" - }, - { - "type": "string", - "name": "content_type" - }, - { - "type": "string", - "name": "content_media_type" - }, - { - "name": "content_language" - }, - { - "name": "content_medium" - }, - { - "name": "content_gradelevel" - }, - { - "name": "content_subjects" - }, - { - "type": "string", - "name": "content_mimetype" - }, - { - "type": "string", - "name": "content_framework" - }, - { - "type": "string", - "name": "content_board" - }, - { - "type": "string", - "name": "content_status" - }, - { - "type": "double", - "name": "content_version" - }, - { - "type": "long", - "name": "content_last_submitted_on" - }, - { - "type": "long", - "name": "content_last_published_on" - }, - { - "type": "long", - "name": "content_last_updated_on" - }, - { - "type": "string", - "name": "content_created_by" - }, - { - "name": "content_created_for" - }, - { - "type": "string", - "name": "collection_name" - }, - { - "type": "string", - "name": "collection_object_type" - }, - { - "type": "string", - "name": "collection_type" - }, - { - "type": "string", - "name": "collection_media_type" - }, - { - "name": "collection_language" - }, - { - "name": "collection_medium" - }, - { - "name": "collection_gradelevel" - }, - { - "name": "collection_subjects" - }, - { - "type": "string", - "name": "collection_mimetype" - }, - { - "type": "string", - "name": "collection_framework" - }, - { - "type": "string", - "name": "collection_board" - }, - { - "type": "string", - "name": "collection_status" - }, - { - "type": "double", - "name": "collection_version" - }, - { - "type": "long", - "name": "collection_last_submitted_on" - }, - { - "type": "long", - "name": "collection_last_published_on" - }, - { - "type": "long", - "name": "collection_last_updated_on" - }, - { - "type": "string", - "name": "collection_created_by" - }, - { - "name": "collection_created_for" - }, - { - "name": "user_grade_list" - }, - { - "name": "user_language_list" - }, - { - "name": "user_subject_list" - }, - { - "type": "string", - "name": "user_type" - }, - { - "name": "user_roles" - }, - { - "type": "string", - "name": "user_loc_state" - }, - { - "type": "string", - "name": "user_loc_district" - }, - { - "type": "string", - "name": "user_signin_type" - }, - { - "type": "string", - "name": "user_login_type" - }, - { - "type": "string", - "name": "dialcode_channel" - }, - { - "type": "string", - "name": "dialcode_batchcode" - }, - { - "type": "string", - "name": "dialcode_publisher" - }, - { - "type": "long", - "name": "dialcode_generated_on" - }, - { - "type": "long", - "name": "dialcode_published_on" - }, - { - "type": "string", - "name": "dialcode_status" - }, - { - "type": "string", - "name": "dialcode_object_type" - }, - { - "type": "string", - "name": "device_loc_iso_state_code" - } - ], - "dimensionsExclusions": [] - }, - "timestampSpec": { - "column": "ets", - "format": "auto" - } - } - }, - "metricsSpec": [], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "day", - "queryGranularity": "none", - "rollup": false - }, - "transformSpec": { - "filter": { - "type": "selector", - "dimension": "context_pdata_id", - "value": "prod.diksha.desktop" - } - } - }, - "ioConfig": { - "topic": "{{env}}.events.telemetry", - "consumerProperties": { - "bootstrap.servers": "{{kafka_brokers}}" - }, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT14400S", - "useEarliestOffset": false, - "completionTimeout": "PT30M" - }, - "tuningConfig": { - "type": "kafka", - "reportParseExceptions": false - } -} diff --git a/ansible/roles/druid-ingestion/templates/pipeline_metrics_index_kafka b/ansible/roles/druid-ingestion/templates/pipeline_metrics_index_kafka deleted file mode 100644 index 66089429fc..0000000000 --- a/ansible/roles/druid-ingestion/templates/pipeline_metrics_index_kafka +++ /dev/null @@ -1,391 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "pipeline-metrics", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "flattenSpec": { - "useFieldDiscovery": false, - "fields": [ - { - "type": "root", - "name": "job-name" - }, - { - "type": "root", - "name": "success-message-count" - }, - { - "type": "root", - "name": "failed-message-count" - }, - { - "type": "root", - "name": "error-message-count" - }, - { - "type": "root", - "name": "skipped-message-count" - }, - { - "type": "root", - "name": "batch-success-count" - }, - { - "type": "root", - "name": "batch-error-count" - }, - { - "type": "root", - "name": "primary-route-success-count" - }, - { - "type": "root", - "name": "secondary-route-success-count" - }, - { - "type": "root", - "name": "consumer-lag" - }, - { - "type": "root", - "name": "partition" - }, - { - "type": "root", - "name": "cache-hit-count" - }, - { - "type": "root", - "name": "cache-miss-count" - }, - { - "type": "root", - "name": "cache-expired-count" - }, - { - "type": "root", - "name": "cache-error-count" - }, - { - "type": "root", - "name": "cache-empty-values-count" - }, - { - "type": "root", - "name": "processed-message-count" - }, - { - "type": "root", - "name": "unprocessed-message-count" - }, - { - "type": "root", - "name": "db-hit-count" - }, - { - "type": "root", - "name": "db-error-count" - }, - { - "type": "root", - "name": "device-cache-hit-count" - }, - { - "type": "root", - "name": "user-cache-hit-count" - }, - { - "type": "root", - "name": "device-db-hit-count" - }, - { - "type": "root", - "name": "user-db-hit-count" - }, - { - "type": "root", - "name": "device-db-error-count" - }, - { - "type": "root", - "name": "user-db-error-count" - }, - { - "type": "root", - "name": "expired-event-count" - }, - { - "type": "root", - "name": "duplicate-event-count" - }, - { - "type": "root", - "name": "device-db-update-count" - }, - { - "type": "root", - "name": "device-cache-update-count" - }, - { - "type": "root", - "name": "user-declared-hit-count" - }, - { - "type": "root", - "name": "ip-location-hit-count" - }, - { - "type": "root", - "name": "no-cache-hit-count" - }, - { - "type": "root", - "name": "audit-route-success-count" - }, - { - "type": "root", - "name": "db-insert-count" - }, - { - "type": "root", - "name": "db-update-count" - }, - { - "type": "root", - "name": "share-route-success-count" - }, - { - "type": "root", - "name": "dialcodes-count" - }, - { - "type": "root", - "name": "dialcodes-api-hit" - }, - { - "type": "root", - "name": "dialcodes-cache-hit" - }, - { - "type": "root", - "name": "assess-route-success-count" - }, - { - "type": "root", - "name": "log-route-success-count" - }, - { - "type": "root", - "name": "error-route-success-count" - } - ] - }, - "dimensionsSpec": { - "dimensions": [ - { - "type": "string", - "name": "job-name" - }, - { - "type": "long", - "name": "success-message-count" - }, - { - "type": "long", - "name": "failed-message-count" - }, - { - "type": "long", - "name": "error-message-count" - }, - { - "type": "long", - "name": "batch-success-count" - }, - { - "type": "long", - "name": "batch-error-count" - }, - { - "type": "long", - "name": "primary-route-success-count" - }, - { - "type": "long", - "name": "secondary-route-success-count" - }, - { - "type": "long", - "name": "skipped-message-count" - }, - { - "type": "long", - "name": "consumer-lag" - }, - { - "type": "long", - "name": "partition" - }, - { - "type": "long", - "name": "cache-hit-count" - }, - { - "type": "long", - "name": "cache-miss-count" - }, - { - "type": "long", - "name": "cache-expired-count" - }, - { - "type": "long", - "name": "cache-error-count" - }, - { - "type": "long", - "name": "cache-empty-values-count" - }, - { - "type": "long", - "name": "processed-message-count" - }, - { - "type": "long", - "name": "unprocessed-message-count" - }, - { - "type": "long", - "name": "db-hit-count" - }, - { - "type": "long", - "name": "db-error-count" - }, - { - "type": "long", - "name": "device-cache-hit-count" - }, - { - "type": "long", - "name": "user-cache-hit-count" - }, - { - "type": "long", - "name": "device-db-hit-count" - }, - { - "type": "long", - "name": "user-db-hit-count" - }, - { - "type": "long", - "name": "device-db-error-count" - }, - { - "type": "long", - "name": "user-db-error-count" - }, - { - "type": "long", - "name": "expired-event-count" - }, - { - "type": "long", - "name": "duplicate-event-count" - }, - { - "type": "long", - "name": "device-db-update-count" - }, - { - "type": "long", - "name": "device-cache-update-count" - }, - { - "type": "long", - "name": "user-declared-hit-count" - }, - { - "type": "long", - "name": "ip-location-hit-count" - }, - { - "type": "long", - "name": "no-cache-hit-count" - }, - { - "type": "long", - "name": "audit-route-success-count" - }, - { - "type": "long", - "name": "db-insert-count" - }, - { - "type": "long", - "name": "db-update-count" - }, - { - "type": "long", - "name": "share-route-success-count" - }, - { - "type": "long", - "name": "dialcodes-count" - }, - { - "type": "long", - "name": "dialcodes-api-hit" - }, - { - "type": "long", - "name": "dialcodes-cache-hit" - }, - { - "type": "long", - "name": "assess-route-success-count" - }, - { - "type": "long", - "name": "log-route-success-count" - }, - { - "type": "long", - "name": "error-route-success-count" - } - ], - "dimensionsExclusions": [] - }, - "timestampSpec": { - "column": "metricts", - "format": "auto" - } - } - }, - "metricsSpec": [], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "day", - "queryGranularity": "none", - "rollup": false - } - }, - "ioConfig": { - "topic": "{{env}}.pipeline_metrics", - "consumerProperties": { - "bootstrap.servers": "{{kafka_brokers}}" - }, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT14400S", - "useEarliestOffset": false - }, - "tuningConfig": { - "type": "kafka", - "reportParseExceptions": false - } -} diff --git a/ansible/roles/druid-ingestion/templates/raw_cb_audit_events b/ansible/roles/druid-ingestion/templates/raw_cb_audit_events new file mode 100644 index 0000000000..87db82cbe8 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_cb_audit_events @@ -0,0 +1,401 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "cb-audit-events", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "root", + "name": "mid" + }, + { + "type": "root", + "name": "ets" + }, + { + "type": "root", + "name": "@timestamp" + }, + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "actor_type", + "expr": "$.actor.type" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "context_sid", + "expr": "$.context.sid" + }, + { + "type": "path", + "name": "context_did", + "expr": "$.context.did" + }, + { + "type": "path", + "name": "context_rollup_l1", + "expr": "$.context.rollup.l1" + }, + { + "type": "path", + "name": "context_rollup_l2", + "expr": "$.context.rollup.l2" + }, + { + "type": "path", + "name": "context_rollup_l3", + "expr": "$.context.rollup.l3" + }, + { + "type": "path", + "name": "context_rollup_l4", + "expr": "$.context.rollup.l4" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_version", + "expr": "$.object.ver" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "object_rollup_l2", + "expr": "$.object.rollup.l2" + }, + { + "type": "path", + "name": "object_rollup_l3", + "expr": "$.object.rollup.l3" + }, + { + "type": "path", + "name": "object_rollup_l4", + "expr": "$.object.rollup.l4" + }, + { + "type": "root", + "name": "tags" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "edata_props", + "expr": "$.edata.props[*]" + }, + { + "type": "path", + "name": "edata_state", + "expr": "$.edata.state" + }, + { + "type": "path", + "name": "edata_prevstate", + "expr": "$.edata.prevstate" + }, + { + "type": "path", + "name": "edata_duration", + "expr": "$.edata.duration" + }, + { + "type": "path", + "name": "edata_cb_object_id", + "expr": "$.edata.cb_object.id" + }, + { + "type": "path", + "name": "edata_cb_object_type", + "expr": "$.edata.cb_object.type" + }, + { + "type": "path", + "name": "edata_cb_object_ver", + "expr": "$.edata.cb_object.ver" + }, + { + "type": "path", + "name": "edata_cb_object_name", + "expr": "$.edata.cb_object.name" + }, + { + "type": "path", + "name": "edata_cb_object_org", + "expr": "$.edata.cb_object.org" + }, + { + "type": "path", + "name": "edata_cb_object_sub_type", + "expr": "$.edata.cb_object.sub_type" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l1", + "expr": "$.edata.cb_object.rollup.l1" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l2", + "expr": "$.edata.cb_object.rollup.l2" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l3", + "expr": "$.edata.cb_object.rollup.l3" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l4", + "expr": "$.edata.cb_object.rollup.l4" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "mid" + }, + { + "type": "long", + "name": "ets" + }, + { + "type": "string", + "name": "@timestamp" + }, + { + "type": "string", + "name": "actor_id" + }, + { + "type": "string", + "name": "actor_type" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "context_sid" + }, + { + "type": "string", + "name": "context_did" + }, + { + "type": "string", + "name": "context_rollup_l1" + }, + { + "type": "string", + "name": "context_rollup_l2" + }, + { + "type": "string", + "name": "context_rollup_l3" + }, + { + "type": "string", + "name": "context_rollup_l4" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_version" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "object_rollup_l2" + }, + { + "type": "string", + "name": "object_rollup_l3" + }, + { + "type": "string", + "name": "object_rollup_l4" + }, + { + "name": "tags" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "name": "edata_props" + }, + { + "type": "string", + "name": "edata_state" + }, + { + "type": "string", + "name": "edata_prevstate" + }, + { + "type": "double", + "name": "edata_duration" + }, + { + "type": "string", + "name": "edata_cb_object_id" + }, + { + "type": "string", + "name": "edata_cb_object_type" + }, + { + "type": "string", + "name": "edata_cb_object_ver" + }, + { + "type": "string", + "name": "edata_cb_object_name" + }, + { + "type": "string", + "name": "edata_cb_object_org" + }, + { + "type": "string", + "name": "edata_cb_object_sub_type" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l1" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l2" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l3" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l4" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{env}}.druid.cb.audit", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_cb_audit_events_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_cb_work_order_officers b/ansible/roles/druid-ingestion/templates/raw_cb_work_order_officers new file mode 100644 index 0000000000..5867852ee1 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_cb_work_order_officers @@ -0,0 +1,671 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "cb-work-order-officers", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "root", + "name": "mid" + }, + { + "type": "root", + "name": "ets" + }, + { + "type": "root", + "name": "@timestamp" + }, + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "actor_type", + "expr": "$.actor.type" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "context_sid", + "expr": "$.context.sid" + }, + { + "type": "path", + "name": "context_did", + "expr": "$.context.did" + }, + { + "type": "path", + "name": "context_rollup_l1", + "expr": "$.context.rollup.l1" + }, + { + "type": "path", + "name": "context_rollup_l2", + "expr": "$.context.rollup.l2" + }, + { + "type": "path", + "name": "context_rollup_l3", + "expr": "$.context.rollup.l3" + }, + { + "type": "path", + "name": "context_rollup_l4", + "expr": "$.context.rollup.l4" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_version", + "expr": "$.object.ver" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "object_rollup_l2", + "expr": "$.object.rollup.l2" + }, + { + "type": "path", + "name": "object_rollup_l3", + "expr": "$.object.rollup.l3" + }, + { + "type": "path", + "name": "object_rollup_l4", + "expr": "$.object.rollup.l4" + }, + { + "type": "root", + "name": "tags" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "edata_props", + "expr": "$.edata.props[*]" + }, + { + "type": "path", + "name": "edata_state", + "expr": "$.edata.state" + }, + { + "type": "path", + "name": "edata_prevstate", + "expr": "$.edata.prevstate" + }, + { + "type": "path", + "name": "edata_duration", + "expr": "$.edata.duration" + }, + { + "type": "path", + "name": "edata_cb_object_id", + "expr": "$.edata.cb_object.id" + }, + { + "type": "path", + "name": "edata_cb_object_type", + "expr": "$.edata.cb_object.type" + }, + { + "type": "path", + "name": "edata_cb_object_ver", + "expr": "$.edata.cb_object.ver" + }, + { + "type": "path", + "name": "edata_cb_object_name", + "expr": "$.edata.cb_object.name" + }, + { + "type": "path", + "name": "edata_cb_object_org", + "expr": "$.edata.cb_object.org" + }, + { + "type": "path", + "name": "edata_cb_object_sub_type", + "expr": "$.edata.cb_object.sub_type" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l1", + "expr": "$.edata.cb_object.rollup.l1" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l2", + "expr": "$.edata.cb_object.rollup.l2" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l3", + "expr": "$.edata.cb_object.rollup.l3" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l4", + "expr": "$.edata.cb_object.rollup.l4" + }, + { + "name": "edata_cb_data_id", + "type": "path", + "expr": "$.edata.cb_data.id" + }, + { + "name": "edata_cb_data_name", + "type": "path", + "expr": "$.edata.cb_data.name" + }, + { + "name": "edata_cb_data_deptId", + "type": "path", + "expr": "$.edata.cb_data.deptId" + }, + { + "name": "edata_cb_data_deptName", + "type": "path", + "expr": "$.edata.cb_data.deptName" + }, + { + "name": "edata_cb_data_status", + "type": "path", + "expr": "$.edata.cb_data.status" + }, + { + "name": "edata_cb_data_userIds", + "type": "path", + "expr": "$.edata.cb_data.userIds" + }, + { + "name": "edata_cb_data_createdBy", + "type": "path", + "expr": "$.edata.cb_data.createdBy" + }, + { + "name": "edata_cb_data_createdAt", + "type": "path", + "expr": "$.edata.cb_data.createdAt" + }, + { + "name": "edata_cb_data_updatedBy", + "type": "path", + "expr": "$.edata.cb_data.updatedBy" + }, + { + "name": "edata_cb_data_updatedAt", + "type": "path", + "expr": "$.edata.cb_data.updatedAt" + }, + { + "name": "edata_cb_data_progress", + "type": "path", + "expr": "$.edata.cb_data.progress" + }, + { + "name": "edata_cb_data_errorCount", + "type": "path", + "expr": "$.edata.cb_data.errorCount" + }, + { + "name": "edata_cb_data_rolesCount", + "type": "path", + "expr": "$.edata.cb_data.rolesCount" + }, + { + "name": "edata_cb_data_activitiesCount", + "type": "path", + "expr": "$.edata.cb_data.activitiesCount" + }, + { + "name": "edata_cb_data_competenciesCount", + "type": "path", + "expr": "$.edata.cb_data.competenciesCount" + }, + { + "name": "edata_cb_data_publishedPdfLink", + "type": "path", + "expr": "$.edata.cb_data.publishedPdfLink" + }, + { + "name": "edata_cb_data_signedPdfLink", + "type": "path", + "expr": "$.edata.cb_data.signedPdfLink" + }, + { + "name": "edata_cb_data_mdo_name", + "type": "path", + "expr": "$.edata.cb_data.mdo_name" + }, + { + "name": "edata_cb_data_wa_id", + "type": "path", + "expr": "$.edata.cb_data.wa_id" + }, + { + "name": "edata_cb_data_wa_userId", + "type": "path", + "expr": "$.edata.cb_data.wa_userId" + }, + { + "name": "edata_cb_data_wa_userPosition", + "type": "path", + "expr": "$.edata.cb_data.wa_userPosition" + }, + { + "name": "edata_cb_data_wa_positionId", + "type": "path", + "expr": "$.edata.cb_data.wa_positionId" + }, + { + "name": "edata_cb_data_wa_positionDescription", + "type": "path", + "expr": "$.edata.cb_data.wa_positionDescription" + }, + { + "name": "edata_cb_data_wa_workOrderId", + "type": "path", + "expr": "$.edata.cb_data.wa_workOrderId" + }, + { + "name": "edata_cb_data_wa_updatedAt", + "type": "path", + "expr": "$.edata.cb_data.wa_updatedAt" + }, + { + "name": "edata_cb_data_wa_updatedBy", + "type": "path", + "expr": "$.edata.cb_data.wa_updatedBy" + }, + { + "name": "edata_cb_data_wa_errorCount", + "type": "path", + "expr": "$.edata.cb_data.wa_errorCount" + }, + { + "name": "edata_cb_data_wa_progress", + "type": "path", + "expr": "$.edata.cb_data.wa_progress" + }, + { + "name": "edata_cb_data_wa_createdAt", + "type": "path", + "expr": "$.edata.cb_data.wa_createdAt" + }, + { + "name": "edata_cb_data_wa_createdBy", + "type": "path", + "expr": "$.edata.cb_data.wa_createdBy" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "mid" + }, + { + "type": "long", + "name": "ets" + }, + { + "type": "string", + "name": "@timestamp" + }, + { + "type": "string", + "name": "actor_id" + }, + { + "type": "string", + "name": "actor_type" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "context_sid" + }, + { + "type": "string", + "name": "context_did" + }, + { + "type": "string", + "name": "context_rollup_l1" + }, + { + "type": "string", + "name": "context_rollup_l2" + }, + { + "type": "string", + "name": "context_rollup_l3" + }, + { + "type": "string", + "name": "context_rollup_l4" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_version" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "object_rollup_l2" + }, + { + "type": "string", + "name": "object_rollup_l3" + }, + { + "type": "string", + "name": "object_rollup_l4" + }, + { + "name": "tags" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "name": "edata_props" + }, + { + "type": "string", + "name": "edata_state" + }, + { + "type": "string", + "name": "edata_prevstate" + }, + { + "type": "double", + "name": "edata_duration" + }, + { + "type": "string", + "name": "edata_cb_object_id" + }, + { + "type": "string", + "name": "edata_cb_object_type" + }, + { + "type": "string", + "name": "edata_cb_object_ver" + }, + { + "type": "string", + "name": "edata_cb_object_name" + }, + { + "type": "string", + "name": "edata_cb_object_org" + }, + { + "type": "string", + "name": "edata_cb_object_sub_type" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l1" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l2" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l3" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l4" + }, + { + "type": "string", + "name": "edata_cb_data_id" + }, + { + "type": "string", + "name": "edata_cb_data_name" + }, + { + "type": "string", + "name": "edata_cb_data_deptId" + }, + { + "type": "string", + "name": "edata_cb_data_deptName" + }, + { + "type": "string", + "name": "edata_cb_data_status" + }, + { + "type": "string", + "name": "edata_cb_data_userIds" + }, + { + "type": "string", + "name": "edata_cb_data_createdBy" + }, + { + "type": "long", + "name": "edata_cb_data_createdAt" + }, + { + "type": "string", + "name": "edata_cb_data_updatedBy" + }, + { + "type": "long", + "name": "edata_cb_data_updatedAt" + }, + { + "type": "long", + "name": "edata_cb_data_progress" + }, + { + "type": "long", + "name": "edata_cb_data_errorCount" + }, + { + "type": "long", + "name": "edata_cb_data_rolesCount" + }, + { + "type": "long", + "name": "edata_cb_data_activitiesCount" + }, + { + "type": "long", + "name": "edata_cb_data_competenciesCount" + }, + { + "type": "string", + "name": "edata_cb_data_publishedPdfLink" + }, + { + "type": "string", + "name": "edata_cb_data_signedPdfLink" + }, + { + "type": "string", + "name": "edata_cb_data_mdo_name" + }, + { + "type": "string", + "name": "edata_cb_data_wa_id" + }, + { + "type": "string", + "name": "edata_cb_data_wa_userId" + }, + { + "type": "string", + "name": "edata_cb_data_wa_userPosition" + }, + { + "type": "string", + "name": "edata_cb_data_wa_positionId" + }, + { + "type": "string", + "name": "edata_cb_data_wa_positionDescription" + }, + { + "type": "string", + "name": "edata_cb_data_wa_workOrderId" + }, + { + "type": "long", + "name": "edata_cb_data_wa_updatedAt" + }, + { + "type": "string", + "name": "edata_cb_data_wa_updatedBy" + }, + { + "type": "long", + "name": "edata_cb_data_wa_errorCount" + }, + { + "type": "long", + "name": "edata_cb_data_wa_progress" + }, + { + "type": "long", + "name": "edata_cb_data_wa_createdAt" + }, + { + "type": "string", + "name": "edata_cb_data_wa_createdBy" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{env}}.druid.cb.work.order.officer", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_cb_audit_events_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout": "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_cb_work_order_properties b/ansible/roles/druid-ingestion/templates/raw_cb_work_order_properties new file mode 100644 index 0000000000..9f971a21ff --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_cb_work_order_properties @@ -0,0 +1,941 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "cb-work-order-properties", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "root", + "name": "mid" + }, + { + "type": "root", + "name": "ets" + }, + { + "type": "root", + "name": "@timestamp" + }, + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "actor_type", + "expr": "$.actor.type" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "context_sid", + "expr": "$.context.sid" + }, + { + "type": "path", + "name": "context_did", + "expr": "$.context.did" + }, + { + "type": "path", + "name": "context_rollup_l1", + "expr": "$.context.rollup.l1" + }, + { + "type": "path", + "name": "context_rollup_l2", + "expr": "$.context.rollup.l2" + }, + { + "type": "path", + "name": "context_rollup_l3", + "expr": "$.context.rollup.l3" + }, + { + "type": "path", + "name": "context_rollup_l4", + "expr": "$.context.rollup.l4" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_version", + "expr": "$.object.ver" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "object_rollup_l2", + "expr": "$.object.rollup.l2" + }, + { + "type": "path", + "name": "object_rollup_l3", + "expr": "$.object.rollup.l3" + }, + { + "type": "path", + "name": "object_rollup_l4", + "expr": "$.object.rollup.l4" + }, + { + "type": "root", + "name": "tags" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "edata_props", + "expr": "$.edata.props[*]" + }, + { + "type": "path", + "name": "edata_state", + "expr": "$.edata.state" + }, + { + "type": "path", + "name": "edata_prevstate", + "expr": "$.edata.prevstate" + }, + { + "type": "path", + "name": "edata_duration", + "expr": "$.edata.duration" + }, + { + "type": "path", + "name": "edata_cb_object_id", + "expr": "$.edata.cb_object.id" + }, + { + "type": "path", + "name": "edata_cb_object_type", + "expr": "$.edata.cb_object.type" + }, + { + "type": "path", + "name": "edata_cb_object_ver", + "expr": "$.edata.cb_object.ver" + }, + { + "type": "path", + "name": "edata_cb_object_name", + "expr": "$.edata.cb_object.name" + }, + { + "type": "path", + "name": "edata_cb_object_org", + "expr": "$.edata.cb_object.org" + }, + { + "type": "path", + "name": "edata_cb_object_sub_type", + "expr": "$.edata.cb_object.sub_type" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l1", + "expr": "$.edata.cb_object.rollup.l1" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l2", + "expr": "$.edata.cb_object.rollup.l2" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l3", + "expr": "$.edata.cb_object.rollup.l3" + }, + { + "type": "path", + "name": "edata_cb_object_rollup_l4", + "expr": "$.edata.cb_object.rollup.l4" + }, + { + "name": "edata_cb_data_id", + "type": "path", + "expr": "$.edata.cb_data.id" + }, + { + "name": "edata_cb_data_name", + "type": "path", + "expr": "$.edata.cb_data.name" + }, + { + "name": "edata_cb_data_deptId", + "type": "path", + "expr": "$.edata.cb_data.deptId" + }, + { + "name": "edata_cb_data_deptName", + "type": "path", + "expr": "$.edata.cb_data.deptName" + }, + { + "name": "edata_cb_data_status", + "type": "path", + "expr": "$.edata.cb_data.status" + }, + { + "name": "edata_cb_data_userIds", + "type": "path", + "expr": "$.edata.cb_data.userIds" + }, + { + "name": "edata_cb_data_createdBy", + "type": "path", + "expr": "$.edata.cb_data.createdBy" + }, + { + "name": "edata_cb_data_createdAt", + "type": "path", + "expr": "$.edata.cb_data.createdAt" + }, + { + "name": "edata_cb_data_updatedBy", + "type": "path", + "expr": "$.edata.cb_data.updatedBy" + }, + { + "name": "edata_cb_data_updatedAt", + "type": "path", + "expr": "$.edata.cb_data.updatedAt" + }, + { + "name": "edata_cb_data_progress", + "type": "path", + "expr": "$.edata.cb_data.progress" + }, + { + "name": "edata_cb_data_errorCount", + "type": "path", + "expr": "$.edata.cb_data.errorCount" + }, + { + "name": "edata_cb_data_rolesCount", + "type": "path", + "expr": "$.edata.cb_data.rolesCount" + }, + { + "name": "edata_cb_data_activitiesCount", + "type": "path", + "expr": "$.edata.cb_data.activitiesCount" + }, + { + "name": "edata_cb_data_competenciesCount", + "type": "path", + "expr": "$.edata.cb_data.competenciesCount" + }, + { + "name": "edata_cb_data_publishedPdfLink", + "type": "path", + "expr": "$.edata.cb_data.publishedPdfLink" + }, + { + "name": "edata_cb_data_signedPdfLink", + "type": "path", + "expr": "$.edata.cb_data.signedPdfLink" + }, + { + "name": "edata_cb_data_mdo_name", + "type": "path", + "expr": "$.edata.cb_data.mdo_name" + }, + { + "name": "edata_cb_data_wa_id", + "type": "path", + "expr": "$.edata.cb_data.wa_id" + }, + { + "name": "edata_cb_data_wa_userId", + "type": "path", + "expr": "$.edata.cb_data.wa_userId" + }, + { + "name": "edata_cb_data_wa_userPosition", + "type": "path", + "expr": "$.edata.cb_data.wa_userPosition" + }, + { + "name": "edata_cb_data_wa_positionId", + "type": "path", + "expr": "$.edata.cb_data.wa_positionId" + }, + { + "name": "edata_cb_data_wa_positionDescription", + "type": "path", + "expr": "$.edata.cb_data.wa_positionDescription" + }, + { + "name": "edata_cb_data_wa_workOrderId", + "type": "path", + "expr": "$.edata.cb_data.wa_workOrderId" + }, + { + "name": "edata_cb_data_wa_updatedAt", + "type": "path", + "expr": "$.edata.cb_data.wa_updatedAt" + }, + { + "name": "edata_cb_data_wa_updatedBy", + "type": "path", + "expr": "$.edata.cb_data.wa_updatedBy" + }, + { + "name": "edata_cb_data_wa_errorCount", + "type": "path", + "expr": "$.edata.cb_data.wa_errorCount" + }, + { + "name": "edata_cb_data_wa_progress", + "type": "path", + "expr": "$.edata.cb_data.wa_progress" + }, + { + "name": "edata_cb_data_wa_createdAt", + "type": "path", + "expr": "$.edata.cb_data.wa_createdAt" + }, + { + "name": "edata_cb_data_wa_createdBy", + "type": "path", + "expr": "$.edata.cb_data.wa_createdBy" + }, + { + "name": "edata_cb_data_wa_role_type", + "type": "path", + "expr": "$.edata.cb_data.wa_role_type" + }, + { + "name": "edata_cb_data_wa_role_id", + "type": "path", + "expr": "$.edata.cb_data.wa_role_id" + }, + { + "name": "edata_cb_data_wa_role_name", + "type": "path", + "expr": "$.edata.cb_data.wa_role_name" + }, + { + "name": "edata_cb_data_wa_role_description", + "type": "path", + "expr": "$.edata.cb_data.wa_role_description" + }, + { + "name": "edata_cb_data_wa_role_status", + "type": "path", + "expr": "$.edata.cb_data.wa_role_status" + }, + { + "name": "edata_cb_data_wa_role_source", + "type": "path", + "expr": "$.edata.cb_data.wa_role_source" + }, + { + "name": "edata_cb_data_wa_role_addedAt", + "type": "path", + "expr": "$.edata.cb_data.wa_role_addedAt" + }, + { + "name": "edata_cb_data_wa_role_updatedAt", + "type": "path", + "expr": "$.edata.cb_data.wa_role_updatedAt" + }, + { + "name": "edata_cb_data_wa_role_updatedBy", + "type": "path", + "expr": "$.edata.cb_data.wa_role_updatedBy" + }, + { + "name": "edata_cb_data_wa_role_archivedAt", + "type": "path", + "expr": "$.edata.cb_data.wa_role_archivedAt" + }, + { + "name": "edata_cb_data_wa_role_archived", + "type": "path", + "expr": "$.edata.cb_data.wa_role_archived" + }, + { + "name": "edata_cb_data_wa_activity_type", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_type" + }, + { + "name": "edata_cb_data_wa_activity_id", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_id" + }, + { + "name": "edata_cb_data_wa_activity_name", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_name" + }, + { + "name": "edata_cb_data_wa_activity_description", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_description" + }, + { + "name": "edata_cb_data_wa_activity_status", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_status" + }, + { + "name": "edata_cb_data_wa_activity_source", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_source" + }, + { + "name": "edata_cb_data_wa_activity_parentRole", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_parentRole" + }, + { + "name": "edata_cb_data_wa_activity_submittedFromId", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_submittedFromId" + }, + { + "name": "edata_cb_data_wa_activity_submittedToId", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_submittedToId" + }, + { + "name": "edata_cb_data_wa_activity_level", + "type": "path", + "expr": "$.edata.cb_data.wa_activity_level" + }, + { + "name": "edata_cb_data_wa_competency_type", + "type": "path", + "expr": "$.edata.cb_data.wa_competency_type" + }, + { + "name": "edata_cb_data_wa_competency_id", + "type": "path", + "expr": "$.edata.cb_data.wa_competency_id" + }, + { + "name": "edata_cb_data_wa_competency_name", + "type": "path", + "expr": "$.edata.cb_data.wa_competency_name" + }, + { + "name": "edata_cb_data_wa_competency_description", + "type": "path", + "expr": "$.edata.cb_data.wa_competency_description" + }, + { + "name": "edata_cb_data_wa_competency_source", + "type": "path", + "expr": "$.edata.cb_data.wa_competency_source" + }, + { + "name": "edata_cb_data_wa_competency_status", + "type": "path", + "expr": "$.edata.cb_data.wa_competency_status" + }, + { + "name": "edata_cb_data_wa_competency_level", + "type": "path", + "expr": "$.edata.cb_data.wa_competency_level" + }, + { + "name": "edata_cb_data_wa_competency_additionalProperties_competencyArea", + "type": "path", + "expr": "$.edata.cb_data.wa_competency_additionalProperties.competencyArea" + }, + { + "name": "edata_cb_data_wa_competency_additionalProperties_competencyType", + "type": "path", + "expr": "$.edata.cb_data.wa_competency_additionalProperties.competencyType" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "mid" + }, + { + "type": "long", + "name": "ets" + }, + { + "type": "string", + "name": "@timestamp" + }, + { + "type": "string", + "name": "actor_id" + }, + { + "type": "string", + "name": "actor_type" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "context_sid" + }, + { + "type": "string", + "name": "context_did" + }, + { + "type": "string", + "name": "context_rollup_l1" + }, + { + "type": "string", + "name": "context_rollup_l2" + }, + { + "type": "string", + "name": "context_rollup_l3" + }, + { + "type": "string", + "name": "context_rollup_l4" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_version" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "object_rollup_l2" + }, + { + "type": "string", + "name": "object_rollup_l3" + }, + { + "type": "string", + "name": "object_rollup_l4" + }, + { + "name": "tags" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "name": "edata_props" + }, + { + "type": "string", + "name": "edata_state" + }, + { + "type": "string", + "name": "edata_prevstate" + }, + { + "type": "double", + "name": "edata_duration" + }, + { + "type": "string", + "name": "edata_cb_object_id" + }, + { + "type": "string", + "name": "edata_cb_object_type" + }, + { + "type": "string", + "name": "edata_cb_object_ver" + }, + { + "type": "string", + "name": "edata_cb_object_name" + }, + { + "type": "string", + "name": "edata_cb_object_org" + }, + { + "type": "string", + "name": "edata_cb_object_sub_type" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l1" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l2" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l3" + }, + { + "type": "string", + "name": "edata_cb_object_rollup_l4" + }, + { + "type": "string", + "name": "edata_cb_data_id" + }, + { + "type": "string", + "name": "edata_cb_data_name" + }, + { + "type": "string", + "name": "edata_cb_data_deptId" + }, + { + "type": "string", + "name": "edata_cb_data_deptName" + }, + { + "type": "string", + "name": "edata_cb_data_status" + }, + { + "type": "string", + "name": "edata_cb_data_userIds" + }, + { + "type": "string", + "name": "edata_cb_data_createdBy" + }, + { + "type": "long", + "name": "edata_cb_data_createdAt" + }, + { + "type": "string", + "name": "edata_cb_data_updatedBy" + }, + { + "type": "long", + "name": "edata_cb_data_updatedAt" + }, + { + "type": "long", + "name": "edata_cb_data_progress" + }, + { + "type": "long", + "name": "edata_cb_data_errorCount" + }, + { + "type": "long", + "name": "edata_cb_data_rolesCount" + }, + { + "type": "long", + "name": "edata_cb_data_activitiesCount" + }, + { + "type": "long", + "name": "edata_cb_data_competenciesCount" + }, + { + "type": "string", + "name": "edata_cb_data_publishedPdfLink" + }, + { + "type": "string", + "name": "edata_cb_data_signedPdfLink" + }, + { + "type": "string", + "name": "edata_cb_data_mdo_name" + }, + { + "type": "string", + "name": "edata_cb_data_wa_id" + }, + { + "type": "string", + "name": "edata_cb_data_wa_userId" + }, + { + "type": "string", + "name": "edata_cb_data_wa_userPosition" + }, + { + "type": "string", + "name": "edata_cb_data_wa_positionId" + }, + { + "type": "string", + "name": "edata_cb_data_wa_positionDescription" + }, + { + "type": "string", + "name": "edata_cb_data_wa_workOrderId" + }, + { + "type": "long", + "name": "edata_cb_data_wa_updatedAt" + }, + { + "type": "string", + "name": "edata_cb_data_wa_updatedBy" + }, + { + "type": "long", + "name": "edata_cb_data_wa_errorCount" + }, + { + "type": "long", + "name": "edata_cb_data_wa_progress" + }, + { + "type": "long", + "name": "edata_cb_data_wa_createdAt" + }, + { + "type": "string", + "name": "edata_cb_data_wa_createdBy" + }, + { + "type": "string", + "name": "edata_cb_data_wa_role_type" + }, + { + "type": "string", + "name": "edata_cb_data_wa_role_id" + }, + { + "type": "string", + "name": "edata_cb_data_wa_role_name" + }, + { + "type": "string", + "name": "edata_cb_data_wa_role_description" + }, + { + "type": "string", + "name": "edata_cb_data_wa_role_status" + }, + { + "type": "string", + "name": "edata_cb_data_wa_role_source" + }, + { + "type": "long", + "name": "edata_cb_data_wa_role_addedAt" + }, + { + "type": "long", + "name": "edata_cb_data_wa_role_updatedAt" + }, + { + "type": "string", + "name": "edata_cb_data_wa_role_updatedBy" + }, + { + "type": "long", + "name": "edata_cb_data_wa_role_archivedAt" + }, + { + "type": "string", + "name": "edata_cb_data_wa_role_archived" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_type" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_id" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_name" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_description" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_status" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_source" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_parentRole" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_submittedFromId" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_submittedToId" + }, + { + "type": "string", + "name": "edata_cb_data_wa_activity_level" + }, + { + "type": "string", + "name": "edata_cb_data_wa_competency_type" + }, + { + "type": "string", + "name": "edata_cb_data_wa_competency_id" + }, + { + "type": "string", + "name": "edata_cb_data_wa_competency_name" + }, + { + "type": "string", + "name": "edata_cb_data_wa_competency_description" + }, + { + "type": "string", + "name": "edata_cb_data_wa_competency_source" + }, + { + "type": "string", + "name": "edata_cb_data_wa_competency_status" + }, + { + "type": "string", + "name": "edata_cb_data_wa_competency_level" + }, + { + "type": "string", + "name": "edata_cb_data_wa_competency_additionalProperties_competencyArea" + }, + { + "type": "string", + "name": "edata_cb_data_wa_competency_additionalProperties_competencyType" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{env}}.druid.cb.work.order.row", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_cb_audit_events_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout": "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_acbp_enrolment b/ansible/roles/druid-ingestion/templates/raw_dashboards_acbp_enrolment new file mode 100644 index 0000000000..7789ca519e --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_acbp_enrolment @@ -0,0 +1,338 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-acbp-enrolment", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "courseID" + }, + { + "type": "root", + "name": "fullName" + }, + { + "type": "root", + "name": "userPrimaryEmail" + }, + { + "type": "root", + "name": "userMobile" + }, + { + "type": "root", + "name": "designation" + }, + { + "type": "root", + "name": "group" + }, + { + "type": "root", + "name": "userOrgID" + }, + { + "type": "path", + "name": "ministryName", + "expr": "$.ministry_name" + }, + { + "type": "path", + "name": "deptName", + "expr": "$.dept_name" + }, + { + "type": "root", + "name": "userOrgName" + }, + { + "type": "root", + "name": "acbpID" + }, + { + "type": "root", + "name": "assignmentType" + }, + { + "type": "root", + "name": "allocatedOn" + }, + { + "type": "root", + "name": "completionDueDate" + }, + { + "type": "root", + "name": "courseOrgID" + }, + { + "type": "root", + "name": "category" + }, + { + "type": "root", + "name": "courseName" + }, + { + "type": "root", + "name": "courseStatus" + }, + { + "type": "root", + "name": "courseReviewStatus" + }, + { + "type": "root", + "name": "courseChannel" + }, + { + "type": "root", + "name": "courseLastPublishedOn" + }, + { + "type": "root", + "name": "courseDuration" + }, + { + "type": "root", + "name": "courseResourceCount" + }, + { + "type": "root", + "name": "lastStatusChangedOn" + }, + { + "type": "root", + "name": "identifier" + }, + { + "type": "root", + "name": "courseOrgName" + }, + { + "type": "root", + "name": "courseOrgStatus" + }, + { + "type": "root", + "name": "batchID" + }, + { + "type": "root", + "name": "courseProgress" + }, + { + "type": "root", + "name": "dbCompletionStatus" + }, + { + "type": "root", + "name": "courseEnrolledTimestamp" + }, + { + "type": "root", + "name": "courseCompletedTimestamp" + }, + { + "type": "root", + "name": "lastContentAccessTimestamp" + }, + { + "type": "root", + "name": "issuedCertificateCount" + }, + { + "type": "root", + "name": "certificateGeneratedOn" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "courseID" + }, + { + "type": "string", + "name": "fullName" + }, + { + "type": "string", + "name": "userPrimaryEmail" + }, + { + "type": "string", + "name": "userMobile" + }, + { + "type": "string", + "name": "designation" + }, + { + "type": "string", + "name": "group" + }, + { + "type": "string", + "name": "userOrgID" + }, + { + "type": "string", + "name": "ministryName" + }, + { + "type": "string", + "name": "deptName" + }, + { + "type": "string", + "name": "userOrgName" + }, + { + "type": "string", + "name": "acbpID" + }, + { + "type": "string", + "name": "assignmentType" + }, + { + "type": "long", + "name": "allocatedOn" + }, + { + "type": "long", + "name": "completionDueDate" + }, + { + "type": "string", + "name": "courseOrgID" + }, + { + "type": "string", + "name": "category" + }, + { + "type": "string", + "name": "courseName" + }, + { + "type": "string", + "name": "courseStatus" + }, + { + "type": "string", + "name": "courseReviewStatus" + }, + { + "type": "string", + "name": "courseChannel" + }, + { + "type": "long", + "name": "courseLastPublishedOn" + }, + { + "type": "float", + "name": "courseDuration" + }, + { + "type": "long", + "name": "courseResourceCount" + }, + { + "type": "long", + "name": "lastStatusChangedOn" + }, + { + "type": "string", + "name": "identifier" + }, + { + "type": "string", + "name": "courseOrgName" + }, + { + "type": "long", + "name": "courseOrgStatus" + }, + { + "type": "string", + "name": "batchID" + }, + { + "type": "long", + "name": "courseProgress" + }, + { + "type": "root", + "name": "dbCompletionStatus" + }, + { + "type": "long", + "name": "courseEnrolledTimestamp" + }, + { + "type": "long", + "name": "courseCompletedTimestamp" + }, + { + "type": "long", + "name": "lastContentAccessTimestamp" + }, + { + "type": "long", + "name": "issuedCertificateCount" + }, + { + "type": "long", + "name": "certificateGeneratedOn" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.acbp.enrolment", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_acbp_enrolment_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_assessment b/ansible/roles/druid-ingestion/templates/raw_dashboards_assessment new file mode 100644 index 0000000000..a3134da119 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_assessment @@ -0,0 +1,208 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-assessment", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "assessID" + }, + { + "type": "root", + "name": "assessCategory" + }, + { + "type": "root", + "name": "assessName" + }, + { + "type": "root", + "name": "assessStatus" + }, + { + "type": "root", + "name": "assessReviewStatus" + }, + { + "type": "root", + "name": "assessOrgID" + }, + { + "type": "root", + "name": "assessOrgName" + }, + { + "type": "root", + "name": "assessOrgStatus" + }, + { + "type": "root", + "name": "assessDuration" + }, + { + "type": "root", + "name": "assessChildCount" + }, + { + "type": "root", + "name": "assessPublishType" + }, + { + "type": "root", + "name": "assessIsExternal" + }, + { + "type": "root", + "name": "assessContentType" + }, + { + "type": "root", + "name": "assessObjectType" + }, + { + "type": "root", + "name": "assessUserConsent" + }, + { + "type": "root", + "name": "assessVisibility" + }, + { + "type": "root", + "name": "assessCreatedOn" + }, + { + "type": "root", + "name": "assessLastUpdatedOn" + }, + { + "type": "root", + "name": "assessLastPublishedOn" + }, + { + "type": "root", + "name": "assessLastSubmittedOn" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "assessID" + }, + { + "type": "string", + "name": "assessCategory" + }, + { + "type": "string", + "name": "assessName" + }, + { + "type": "string", + "name": "assessStatus" + }, + { + "type": "string", + "name": "assessReviewStatus" + }, + { + "type": "string", + "name": "assessOrgID" + }, + { + "type": "string", + "name": "assessOrgName" + }, + { + "type": "long", + "name": "assessOrgStatus" + }, + { + "type": "float", + "name": "assessDuration" + }, + { + "type": "long", + "name": "assessChildCount" + }, + { + "type": "string", + "name": "assessPublishType" + }, + { + "type": "long", + "name": "assessIsExternal" + }, + { + "type": "string", + "name": "assessContentType" + }, + { + "type": "string", + "name": "assessObjectType" + }, + { + "type": "string", + "name": "assessUserConsent" + }, + { + "type": "string", + "name": "assessVisibility" + }, + { + "type": "long", + "name": "assessCreatedOn" + }, + { + "type": "long", + "name": "assessLastUpdatedOn" + }, + { + "type": "long", + "name": "assessLastPublishedOn" + }, + { + "type": "long", + "name": "assessLastSubmittedOn" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.assessment", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_assessment_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_course b/ansible/roles/druid-ingestion/templates/raw_dashboards_course new file mode 100644 index 0000000000..4fbe81d0c9 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_course @@ -0,0 +1,192 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-course", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "courseID" + }, + { + "type": "root", + "name": "category" + }, + { + "type": "root", + "name": "courseName" + }, + { + "type": "root", + "name": "courseStatus" + }, + { + "type": "root", + "name": "courseReviewStatus" + }, + { + "type": "root", + "name": "courseOrgID" + }, + { + "type": "root", + "name": "courseOrgName" + }, + { + "type": "root", + "name": "courseOrgStatus" + }, + { + "type": "root", + "name": "courseDuration" + }, + { + "type": "root", + "name": "courseResourceCount" + }, + { + "type": "root", + "name": "ratingSum" + }, + { + "type": "root", + "name": "ratingCount" + }, + { + "type": "root", + "name": "ratingAverage" + }, + { + "type": "root", + "name": "count1Star" + }, + { + "type": "root", + "name": "count2Star" + }, + { + "type": "root", + "name": "count3Star" + }, + { + "type": "root", + "name": "count4Star" + }, + { + "type": "root", + "name": "count5Star" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "courseID" + }, + { + "type": "string", + "name": "category" + }, + { + "type": "string", + "name": "courseName" + }, + { + "type": "string", + "name": "courseStatus" + }, + { + "type": "string", + "name": "courseReviewStatus" + }, + { + "type": "string", + "name": "courseOrgID" + }, + { + "type": "string", + "name": "courseOrgName" + }, + { + "type": "long", + "name": "courseOrgStatus" + }, + { + "type": "float", + "name": "courseDuration" + }, + { + "type": "long", + "name": "courseResourceCount" + }, + { + "type": "float", + "name": "ratingSum" + }, + { + "type": "float", + "name": "ratingCount" + }, + { + "type": "float", + "name": "ratingAverage" + }, + { + "type": "float", + "name": "count1Star" + }, + { + "type": "float", + "name": "count2Star" + }, + { + "type": "float", + "name": "count3Star" + }, + { + "type": "float", + "name": "count4Star" + }, + { + "type": "float", + "name": "count5Star" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.course", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_course_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_course_competency b/ansible/roles/druid-ingestion/templates/raw_dashboards_course_competency new file mode 100644 index 0000000000..741e0e4271 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_course_competency @@ -0,0 +1,160 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-course-competency", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "courseID" + }, + { + "type": "root", + "name": "category" + }, + { + "type": "root", + "name": "courseName" + }, + { + "type": "root", + "name": "courseStatus" + }, + { + "type": "root", + "name": "courseReviewStatus" + }, + { + "type": "root", + "name": "courseOrgID" + }, + { + "type": "root", + "name": "courseOrgName" + }, + { + "type": "root", + "name": "courseOrgStatus" + }, + { + "type": "root", + "name": "courseDuration" + }, + { + "type": "root", + "name": "courseResourceCount" + }, + { + "type": "root", + "name": "competencyID" + }, + { + "type": "root", + "name": "competencyName" + }, + { + "type": "root", + "name": "competencyType" + }, + { + "type": "root", + "name": "competencyLevel" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "courseID" + }, + { + "type": "string", + "name": "category" + }, + { + "type": "string", + "name": "courseName" + }, + { + "type": "string", + "name": "courseStatus" + }, + { + "type": "string", + "name": "courseReviewStatus" + }, + { + "type": "string", + "name": "courseOrgID" + }, + { + "type": "string", + "name": "courseOrgName" + }, + { + "type": "long", + "name": "courseOrgStatus" + }, + { + "type": "float", + "name": "courseDuration" + }, + { + "type": "long", + "name": "courseResourceCount" + }, + { + "type": "string", + "name": "competencyID" + }, + { + "type": "string", + "name": "competencyName" + }, + { + "type": "string", + "name": "competencyType" + }, + { + "type": "long", + "name": "competencyLevel" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.competency.course", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_course_competency_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_declared_user_competency b/ansible/roles/druid-ingestion/templates/raw_dashboards_declared_user_competency new file mode 100644 index 0000000000..a9edd5ab86 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_declared_user_competency @@ -0,0 +1,72 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-declared-user-competency", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "competencyID" + }, + { + "type": "root", + "name": "declaredCompetencyLevel" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "competencyID" + }, + { + "type": "long", + "name": "declaredCompetencyLevel" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.competency.declared", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_declared_user_competency_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_expected_user_competency b/ansible/roles/druid-ingestion/templates/raw_dashboards_expected_user_competency new file mode 100644 index 0000000000..7f76212981 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_expected_user_competency @@ -0,0 +1,96 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-expected-user-competency", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "orgID" + }, + { + "type": "root", + "name": "workOrderID" + }, + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "competencyID" + }, + { + "type": "root", + "name": "expectedCompetencyLevel" + }, + { + "type": "root", + "name": "liveCourseCount" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "orgID" + }, + { + "type": "string", + "name": "workOrderID" + }, + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "competencyID" + }, + { + "type": "long", + "name": "expectedCompetencyLevel" + }, + { + "type": "long", + "name": "liveCourseCount" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.competency.expected", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_expected_user_competency_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_frac_competency b/ansible/roles/druid-ingestion/templates/raw_dashboards_frac_competency new file mode 100644 index 0000000000..729f0b4075 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_frac_competency @@ -0,0 +1,96 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-frac-competency", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "competencyID" + }, + { + "type": "root", + "name": "competencyName" + }, + { + "type": "root", + "name": "competencyStatus" + }, + { + "type": "root", + "name": "liveCourseCount" + }, + { + "type": "root", + "name": "officerCountExpected" + }, + { + "type": "root", + "name": "officerCountDeclared" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "competencyID" + }, + { + "type": "string", + "name": "competencyName" + }, + { + "type": "string", + "name": "competencyStatus" + }, + { + "type": "long", + "name": "liveCourseCount" + }, + { + "type": "long", + "name": "officerCountExpected" + }, + { + "type": "long", + "name": "officerCountDeclared" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.competency.frac", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_frac_competency_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_org b/ansible/roles/druid-ingestion/templates/raw_dashboards_org new file mode 100644 index 0000000000..e4a3b1b27c --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_org @@ -0,0 +1,96 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-org", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "orgID" + }, + { + "type": "root", + "name": "orgName" + }, + { + "type": "root", + "name": "orgStatus" + }, + { + "type": "root", + "name": "orgCreatedDate" + }, + { + "type": "root", + "name": "orgType" + }, + { + "type": "root", + "name": "orgSubType" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "orgID" + }, + { + "type": "string", + "name": "orgName" + }, + { + "type": "long", + "name": "orgStatus" + }, + { + "type": "long", + "name": "orgCreatedDate" + }, + { + "type": "long", + "name": "orgType" + }, + { + "type": "long", + "name": "orgSubType" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.org", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_org_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_org_role_count b/ansible/roles/druid-ingestion/templates/raw_dashboards_org_role_count new file mode 100644 index 0000000000..cad3a07969 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_org_role_count @@ -0,0 +1,80 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-org-role-count", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "orgID" + }, + { + "type": "root", + "name": "orgName" + }, + { + "type": "root", + "name": "role" + }, + { + "type": "root", + "name": "count" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "orgID" + }, + { + "type": "string", + "name": "orgName" + }, + { + "type": "string", + "name": "role" + }, + { + "type": "long", + "name": "count" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.org.role.count", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_org_role_count_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_role_count b/ansible/roles/druid-ingestion/templates/raw_dashboards_role_count new file mode 100644 index 0000000000..0c38f9bfea --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_role_count @@ -0,0 +1,64 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-role-count", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "role" + }, + { + "type": "root", + "name": "count" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "role" + }, + { + "type": "long", + "name": "count" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.role.count", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_role_count_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_user_assessment b/ansible/roles/druid-ingestion/templates/raw_dashboards_user_assessment new file mode 100644 index 0000000000..1b1d4b7113 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_user_assessment @@ -0,0 +1,624 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-user-assessment", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "courseID" + }, + { + "type": "root", + "name": "assessChildID" + }, + { + "type": "root", + "name": "assessStartTime" + }, + { + "type": "root", + "name": "assessEndTime" + }, + { + "type": "root", + "name": "assessUserStatus" + }, + { + "type": "root", + "name": "assessTotalQuestions" + }, + { + "type": "root", + "name": "assessMaxQuestions" + }, + { + "type": "root", + "name": "assessExpectedDuration" + }, + { + "type": "root", + "name": "assessVersion" + }, + { + "type": "root", + "name": "assessMaxRetakeAttempts" + }, + { + "type": "root", + "name": "assessReadStatus" + }, + { + "type": "root", + "name": "assessBatchID" + }, + { + "type": "root", + "name": "assessPrimaryCategory" + }, + { + "type": "root", + "name": "assessIsAssessment" + }, + { + "type": "root", + "name": "assessTimeLimit" + }, + { + "type": "root", + "name": "assessResult" + }, + { + "type": "root", + "name": "assessTotal" + }, + { + "type": "root", + "name": "assessBlank" + }, + { + "type": "root", + "name": "assessCorrect" + }, + { + "type": "root", + "name": "assessIncorrect" + }, + { + "type": "root", + "name": "assessPass" + }, + { + "type": "root", + "name": "assessOverallResult" + }, + { + "type": "root", + "name": "assessPassPercentage" + }, + { + "type": "root", + "name": "assessID" + }, + { + "type": "root", + "name": "assessChildName" + }, + { + "type": "root", + "name": "assessChildDuration" + }, + { + "type": "root", + "name": "assessChildPrimaryCategory" + }, + { + "type": "root", + "name": "assessChildContentType" + }, + { + "type": "root", + "name": "assessChildObjectType" + }, + { + "type": "root", + "name": "assessChildShowTimer" + }, + { + "type": "root", + "name": "assessChildAllowSkip" + }, + { + "type": "root", + "name": "assessCategory" + }, + { + "type": "root", + "name": "assessName" + }, + { + "type": "root", + "name": "assessStatus" + }, + { + "type": "root", + "name": "assessReviewStatus" + }, + { + "type": "root", + "name": "assessOrgID" + }, + { + "type": "root", + "name": "assessOrgName" + }, + { + "type": "root", + "name": "assessOrgStatus" + }, + { + "type": "root", + "name": "assessDuration" + }, + { + "type": "root", + "name": "assessChildCount" + }, + { + "type": "root", + "name": "assessPublishType" + }, + { + "type": "root", + "name": "assessIsExternal" + }, + { + "type": "root", + "name": "assessContentType" + }, + { + "type": "root", + "name": "assessObjectType" + }, + { + "type": "root", + "name": "assessUserConsent" + }, + { + "type": "root", + "name": "assessVisibility" + }, + { + "type": "root", + "name": "assessCreatedOn" + }, + { + "type": "root", + "name": "assessLastUpdatedOn" + }, + { + "type": "root", + "name": "assessLastPublishedOn" + }, + { + "type": "root", + "name": "assessLastSubmittedOn" + }, + { + "type": "root", + "name": "category" + }, + { + "type": "root", + "name": "courseName" + }, + { + "type": "root", + "name": "courseStatus" + }, + { + "type": "root", + "name": "courseReviewStatus" + }, + { + "type": "root", + "name": "courseDuration" + }, + { + "type": "root", + "name": "courseResourceCount" + }, + { + "type": "root", + "name": "courseOrgID" + }, + { + "type": "root", + "name": "courseOrgName" + }, + { + "type": "root", + "name": "courseOrgStatus" + }, + { + "type": "root", + "name": "ratingSum" + }, + { + "type": "root", + "name": "ratingCount" + }, + { + "type": "root", + "name": "ratingAverage" + }, + { + "type": "root", + "name": "userOrgID" + }, + { + "type": "root", + "name": "userOrgName" + }, + { + "type": "root", + "name": "userOrgStatus" + }, + { + "type": "root", + "name": "firstName" + }, + { + "type": "root", + "name": "lastName" + }, + { + "type": "root", + "name": "maskedEmail" + }, + { + "type": "root", + "name": "userStatus" + }, + { + "type": "root", + "name": "userCreatedTimestamp" + }, + { + "type": "root", + "name": "userUpdatedTimestamp" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "courseID" + }, + { + "type": "string", + "name": "assessChildID" + }, + { + "type": "long", + "name": "assessStartTime" + }, + { + "type": "long", + "name": "assessEndTime" + }, + { + "type": "string", + "name": "assessUserStatus" + }, + { + "type": "long", + "name": "assessTotalQuestions" + }, + { + "type": "long", + "name": "assessMaxQuestions" + }, + { + "type": "float", + "name": "assessExpectedDuration" + }, + { + "type": "long", + "name": "assessVersion" + }, + { + "type": "long", + "name": "assessMaxRetakeAttempts" + }, + { + "type": "string", + "name": "assessReadStatus" + }, + { + "type": "string", + "name": "assessBatchID" + }, + { + "type": "string", + "name": "assessPrimaryCategory" + }, + { + "type": "long", + "name": "assessIsAssessment" + }, + { + "type": "float", + "name": "assessTimeLimit" + }, + { + "type": "float", + "name": "assessResult" + }, + { + "type": "long", + "name": "assessTotal" + }, + { + "type": "long", + "name": "assessBlank" + }, + { + "type": "long", + "name": "assessCorrect" + }, + { + "type": "long", + "name": "assessIncorrect" + }, + { + "type": "long", + "name": "assessPass" + }, + { + "type": "float", + "name": "assessOverallResult" + }, + { + "type": "float", + "name": "assessPassPercentage" + }, + { + "type": "string", + "name": "assessID" + }, + { + "type": "string", + "name": "assessChildName" + }, + { + "type": "float", + "name": "assessChildDuration" + }, + { + "type": "string", + "name": "assessChildPrimaryCategory" + }, + { + "type": "string", + "name": "assessChildContentType" + }, + { + "type": "string", + "name": "assessChildObjectType" + }, + { + "type": "string", + "name": "assessChildShowTimer" + }, + { + "type": "string", + "name": "assessChildAllowSkip" + }, + { + "type": "string", + "name": "assessCategory" + }, + { + "type": "string", + "name": "assessName" + }, + { + "type": "string", + "name": "assessStatus" + }, + { + "type": "string", + "name": "assessReviewStatus" + }, + { + "type": "string", + "name": "assessOrgID" + }, + { + "type": "string", + "name": "assessOrgName" + }, + { + "type": "long", + "name": "assessOrgStatus" + }, + { + "type": "float", + "name": "assessDuration" + }, + { + "type": "long", + "name": "assessChildCount" + }, + { + "type": "string", + "name": "assessPublishType" + }, + { + "type": "long", + "name": "assessIsExternal" + }, + { + "type": "string", + "name": "assessContentType" + }, + { + "type": "string", + "name": "assessObjectType" + }, + { + "type": "string", + "name": "assessUserConsent" + }, + { + "type": "string", + "name": "assessVisibility" + }, + { + "type": "long", + "name": "assessCreatedOn" + }, + { + "type": "long", + "name": "assessLastUpdatedOn" + }, + { + "type": "long", + "name": "assessLastPublishedOn" + }, + { + "type": "long", + "name": "assessLastSubmittedOn" + }, + { + "type": "string", + "name": "category" + }, + { + "type": "string", + "name": "courseName" + }, + { + "type": "string", + "name": "courseStatus" + }, + { + "type": "string", + "name": "courseReviewStatus" + }, + { + "type": "float", + "name": "courseDuration" + }, + { + "type": "long", + "name": "courseResourceCount" + }, + { + "type": "string", + "name": "courseOrgID" + }, + { + "type": "string", + "name": "courseOrgName" + }, + { + "type": "long", + "name": "courseOrgStatus" + }, + { + "type": "float", + "name": "ratingSum" + }, + { + "type": "float", + "name": "ratingCount" + }, + { + "type": "float", + "name": "ratingAverage" + }, + { + "type": "string", + "name": "userOrgID" + }, + { + "type": "string", + "name": "userOrgName" + }, + { + "type": "long", + "name": "userOrgStatus" + }, + { + "type": "string", + "name": "firstName" + }, + { + "type": "string", + "name": "lastName" + }, + { + "type": "string", + "name": "maskedEmail" + }, + { + "type": "long", + "name": "userStatus" + }, + { + "type": "long", + "name": "userCreatedTimestamp" + }, + { + "type": "long", + "name": "userUpdatedTimestamp" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.user.assessment", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_user_assessment_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_user_competency_gap b/ansible/roles/druid-ingestion/templates/raw_dashboards_user_competency_gap new file mode 100644 index 0000000000..c175430727 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_user_competency_gap @@ -0,0 +1,120 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-user-competency-gap", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "competencyID" + }, + { + "type": "root", + "name": "orgID" + }, + { + "type": "root", + "name": "workOrderID" + }, + { + "type": "root", + "name": "expectedCompetencyLevel" + }, + { + "type": "root", + "name": "declaredCompetencyLevel" + }, + { + "type": "root", + "name": "competencyGap" + }, + { + "type": "root", + "name": "completionPercentage" + }, + { + "type": "root", + "name": "completionStatus" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "competencyID" + }, + { + "type": "string", + "name": "orgID" + }, + { + "type": "string", + "name": "workOrderID" + }, + { + "type": "long", + "name": "expectedCompetencyLevel" + }, + { + "type": "long", + "name": "declaredCompetencyLevel" + }, + { + "type": "long", + "name": "competencyGap" + }, + { + "type": "float", + "name": "completionPercentage" + }, + { + "type": "string", + "name": "completionStatus" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.competency.gap", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_user_competency_gap_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_user_course_program_progress b/ansible/roles/druid-ingestion/templates/raw_dashboards_user_course_program_progress new file mode 100644 index 0000000000..b4a08af16d --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_user_course_program_progress @@ -0,0 +1,256 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-user-course-program-progress", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "courseID" + }, + { + "type": "root", + "name": "batchID" + }, + { + "type": "root", + "name": "courseCompletedTimestamp" + }, + { + "type": "root", + "name": "courseEnrolledTimestamp" + }, + { + "type": "root", + "name": "lastContentAccessTimestamp" + }, + { + "type": "root", + "name": "courseProgress" + }, + { + "type": "root", + "name": "dbCompletionStatus" + }, + { + "type": "root", + "name": "category" + }, + { + "type": "root", + "name": "courseName" + }, + { + "type": "root", + "name": "courseStatus" + }, + { + "type": "root", + "name": "courseReviewStatus" + }, + { + "type": "root", + "name": "courseDuration" + }, + { + "type": "root", + "name": "courseResourceCount" + }, + { + "type": "root", + "name": "courseOrgID" + }, + { + "type": "root", + "name": "courseOrgName" + }, + { + "type": "root", + "name": "courseOrgStatus" + }, + { + "type": "root", + "name": "userOrgID" + }, + { + "type": "root", + "name": "userOrgName" + }, + { + "type": "root", + "name": "userOrgStatus" + }, + { + "type": "root", + "name": "firstName" + }, + { + "type": "root", + "name": "lastName" + }, + { + "type": "root", + "name": "maskedEmail" + }, + { + "type": "root", + "name": "userStatus" + }, + { + "type": "root", + "name": "completionPercentage" + }, + { + "type": "root", + "name": "completionStatus" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "courseID" + }, + { + "type": "string", + "name": "batchID" + }, + { + "type": "long", + "name": "courseCompletedTimestamp" + }, + { + "type": "long", + "name": "courseEnrolledTimestamp" + }, + { + "type": "long", + "name": "lastContentAccessTimestamp" + }, + { + "type": "long", + "name": "courseProgress" + }, + { + "type": "long", + "name": "dbCompletionStatus" + }, + { + "type": "string", + "name": "category" + }, + { + "type": "string", + "name": "courseName" + }, + { + "type": "string", + "name": "courseStatus" + }, + { + "type": "string", + "name": "courseReviewStatus" + }, + { + "type": "float", + "name": "courseDuration" + }, + { + "type": "long", + "name": "courseResourceCount" + }, + { + "type": "string", + "name": "courseOrgID" + }, + { + "type": "string", + "name": "courseOrgName" + }, + { + "type": "long", + "name": "courseOrgStatus" + }, + { + "type": "string", + "name": "userOrgID" + }, + { + "type": "string", + "name": "userOrgName" + }, + { + "type": "long", + "name": "userOrgStatus" + }, + { + "type": "string", + "name": "firstName" + }, + { + "type": "string", + "name": "lastName" + }, + { + "type": "string", + "name": "maskedEmail" + }, + { + "type": "long", + "name": "userStatus" + }, + { + "type": "float", + "name": "completionPercentage" + }, + { + "type": "string", + "name": "completionStatus" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.user.course.program.progress", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_user_course_program_progress_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_dashboards_user_org b/ansible/roles/druid-ingestion/templates/raw_dashboards_user_org new file mode 100644 index 0000000000..3d24fe1857 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_dashboards_user_org @@ -0,0 +1,128 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "dashboards-user-org", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "firstName" + }, + { + "type": "root", + "name": "lastName" + }, + { + "type": "root", + "name": "maskedEmail" + }, + { + "type": "root", + "name": "userStatus" + }, + { + "type": "root", + "name": "userCreatedTimestamp" + }, + { + "type": "root", + "name": "userUpdatedTimestamp" + }, + { + "type": "root", + "name": "userOrgID" + }, + { + "type": "root", + "name": "userOrgName" + }, + { + "type": "root", + "name": "userOrgStatus" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "firstName" + }, + { + "type": "string", + "name": "lastName" + }, + { + "type": "string", + "name": "maskedEmail" + }, + { + "type": "long", + "name": "userStatus" + }, + { + "type": "long", + "name": "userCreatedTimestamp" + }, + { + "type": "long", + "name": "userUpdatedTimestamp" + }, + { + "type": "string", + "name": "userOrgID" + }, + { + "type": "string", + "name": "userOrgName" + }, + { + "type": "long", + "name": "userOrgStatus" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.dashboards.user.org", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_dashboards_user_org_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} \ No newline at end of file diff --git a/ansible/roles/druid-ingestion/templates/raw_nps_upgraded_users_data b/ansible/roles/druid-ingestion/templates/raw_nps_upgraded_users_data new file mode 100644 index 0000000000..0fc3467da4 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_nps_upgraded_users_data @@ -0,0 +1,337 @@ +{ + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "nps-upgraded-users-data", + "timestampSpec": null, + "dimensionsSpec": null, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "submitted" + }, + { + "type": "root", + "name": "rating" + }, + { + "type": "root", + "name": "activityDesc" + }, + { + "type": "root", + "name": "activityID" + }, + { + "type": "root", + "name": "userType" + }, + { + "type": "root", + "name": "originPlatform" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "submitted" + }, + { + "type": "long", + "name": "rating" + }, + { + "type": "string", + "name": "activityDesc" + }, + { + "type": "long", + "name": "activityID" + }, + { + "type": "string", + "name": "userType" + }, + { + "type": "string", + "name": "originPlatform" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + } + }, + "ioConfig": { + "topic": "{{env}}.nps_upgraded_submission", + "inputFormat": null, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT7200S", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT7200S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "stream": "prod.nps_upgraded_submission", + "useEarliestSequenceNumber": true, + "autoscalerConfig": null + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap" + }, + "maxRowsInMemory": 1000000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 2000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "basePersistDirectory": "/var/tmp/druid-realtime-persist6027210174357931843", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring", + "compressRunOnSerialization": true + }, + "dimensionCompression": "lz4", + "metricCompression": "lz4", + "longEncoding": "longs", + "segmentLoader": null + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring", + "compressRunOnSerialization": true + }, + "dimensionCompression": "lz4", + "metricCompression": "lz4", + "longEncoding": "longs", + "segmentLoader": null + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 0, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "dataSchema": { + "dataSource": "nps-upgraded-users-data", + "timestampSpec": null, + "dimensionsSpec": null, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "submitted" + }, + { + "type": "root", + "name": "rating" + }, + { + "type": "root", + "name": "activityDesc" + }, + { + "type": "root", + "name": "activityID" + }, + { + "type": "root", + "name": "userType" + }, + { + "type": "root", + "name": "originPlatform" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "submitted" + }, + { + "type": "long", + "name": "rating" + }, + { + "type": "string", + "name": "activityDesc" + }, + { + "type": "long", + "name": "activityID" + }, + { + "type": "string", + "name": "userType" + }, + { + "type": "string", + "name": "originPlatform" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + } + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap" + }, + "maxRowsInMemory": 1000000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 2000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "basePersistDirectory": "/var/tmp/druid-realtime-persist6027210174357931843", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring", + "compressRunOnSerialization": true + }, + "dimensionCompression": "lz4", + "metricCompression": "lz4", + "longEncoding": "longs", + "segmentLoader": null + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring", + "compressRunOnSerialization": true + }, + "dimensionCompression": "lz4", + "metricCompression": "lz4", + "longEncoding": "longs", + "segmentLoader": null + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 0, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + }, + "ioConfig": { + "topic": "{{ env }}.nps_upgraded_submission", + "inputFormat": null, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT7200S", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT7200S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "stream": "prod.nps_upgraded_submission", + "useEarliestSequenceNumber": true, + "autoscalerConfig": null + } +} diff --git a/ansible/roles/druid-ingestion/templates/raw_nps_users_data b/ansible/roles/druid-ingestion/templates/raw_nps_users_data new file mode 100644 index 0000000000..931be0ea89 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_nps_users_data @@ -0,0 +1,96 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "nps-users-data", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "userID" + }, + { + "type": "root", + "name": "submitted" + }, + { + "type": "root", + "name": "rating" + }, + { + "type": "root", + "name": "activityDesc" + }, + { + "type": "root", + "name": "activityID" + }, + { + "type": "root", + "name": "originPlatform" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "userID" + }, + { + "type": "string", + "name": "submitted" + }, + { + "type": "long", + "name": "rating" + }, + { + "type": "string", + "name": "activityDesc" + }, + { + "type": "long", + "name": "activityID" + }, + { + "type": "string", + "name": "originPlatform" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{ env }}.nps_submission", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout": "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": "2000000" + } +} diff --git a/ansible/roles/druid-ingestion/templates/raw_offline_desktop b/ansible/roles/druid-ingestion/templates/raw_offline_desktop new file mode 100644 index 0000000000..d31f22d320 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_offline_desktop @@ -0,0 +1,1353 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "offline-desktop-events", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "root", + "name": "mid" + }, + { + "type": "root", + "name": "syncts" + }, + { + "type": "root", + "name": "@timestamp" + }, + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "actor_type", + "expr": "$.actor.type" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "context_sid", + "expr": "$.context.sid" + }, + { + "type": "path", + "name": "context_did", + "expr": "$.context.did" + }, + { + "type": "path", + "name": "context_cdata_type", + "expr": "$.context.cdata[*].type" + }, + { + "type": "path", + "name": "context_cdata_id", + "expr": "$.context.cdata[*].id" + }, + { + "type": "path", + "name": "context_rollup_l1", + "expr": "$.context.rollup.l1" + }, + { + "type": "path", + "name": "context_rollup_l2", + "expr": "$.context.rollup.l2" + }, + { + "type": "path", + "name": "context_rollup_l3", + "expr": "$.context.rollup.l3" + }, + { + "type": "path", + "name": "context_rollup_l4", + "expr": "$.context.rollup.l4" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_version", + "expr": "$.object.ver" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "object_rollup_l2", + "expr": "$.object.rollup.l2" + }, + { + "type": "path", + "name": "object_rollup_l3", + "expr": "$.object.rollup.l3" + }, + { + "type": "path", + "name": "object_rollup_l4", + "expr": "$.object.rollup.l4" + }, + { + "type": "root", + "name": "tags" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "edata_subtype", + "expr": "$.edata.subtype" + }, + { + "type": "path", + "name": "edata_mode", + "expr": "$.edata.mode" + }, + { + "type": "path", + "name": "edata_pageid", + "expr": "$.edata.pageid" + }, + { + "type": "path", + "name": "edata_uri", + "expr": "$.edata.uri" + }, + { + "type": "path", + "name": "edata_id", + "expr": "$.edata.id" + }, + { + "type": "path", + "name": "edata_duration", + "expr": "$.edata.duration" + }, + { + "type": "path", + "name": "edata_index", + "expr": "$.edata.index" + }, + { + "type": "path", + "name": "edata_pass", + "expr": "$.edata.pass" + }, + { + "type": "path", + "name": "edata_score", + "expr": "$.edata.score" + }, + { + "type": "path", + "name": "edata_resvalues", + "expr": "$.edata.resvalues[*]" + }, + { + "type": "path", + "name": "edata_item_id", + "expr": "$.edata.item.id" + }, + { + "type": "path", + "name": "edata_item_title", + "expr": "$.edata.item.title" + }, + { + "type": "path", + "name": "edata_item_maxscore", + "expr": "$.edata.item.maxscore" + }, + { + "type": "path", + "name": "edata_target_id", + "expr": "$.edata.target.id" + }, + { + "type": "path", + "name": "edata_target_type", + "expr": "$.edata.target.type" + }, + { + "type": "path", + "name": "edata_rating", + "expr": "$.edata.rating" + }, + { + "type": "path", + "name": "edata_comments", + "expr": "$.edata.comments" + }, + { + "type": "path", + "name": "edata_dir", + "expr": "$.edata.dir" + }, + { + "type": "path", + "name": "edata_items_id", + "expr": "$.edata.items[*].id" + }, + { + "type": "path", + "name": "edata_items_type", + "expr": "$.edata.items[*].type" + }, + { + "type": "path", + "name": "edata_items_origin_id", + "expr": "$.edata.items[*].origin.id" + }, + { + "type": "path", + "name": "edata_items_origin_type", + "expr": "$.edata.items[*].origin.type" + }, + { + "type": "path", + "name": "edata_items_to_id", + "expr": "$.edata.items[*].to.id" + }, + { + "type": "path", + "name": "edata_items_to_type", + "expr": "$.edata.items[*].to.type" + }, + { + "type": "path", + "name": "edata_plugin_id", + "expr": "$.edata.plugin.id" + }, + { + "type": "path", + "name": "edata_plugin_ver", + "expr": "$.edata.plugin.ver" + }, + { + "type": "path", + "name": "edata_plugin_category", + "expr": "$.edata.plugin.category" + }, + { + "type": "path", + "name": "edata_props", + "expr": "$.edata.props[*]" + }, + { + "type": "path", + "name": "edata_state", + "expr": "$.edata.state" + }, + { + "type": "path", + "name": "edata_prevstate", + "expr": "$.edata.prevstate" + }, + { + "type": "path", + "name": "edata_size", + "expr": "$.edata.size" + }, + { + "type": "path", + "name": "edata_filters_dialcodes", + "expr": "$.edata.filters.dialcodes" + }, + { + "type": "path", + "name": "edata_topn_identifier", + "expr": "$.edata.topn[*].identifier" + }, + { + "type": "path", + "name": "edata_visits_objid", + "expr": "$.edata.visits[*].objid" + }, + { + "type": "path", + "name": "edata_visits_objtype", + "expr": "$.edata.visits[*].objtype" + }, + { + "type": "path", + "name": "edata_visits_objver", + "expr": "$.edata.visits[*].objver" + }, + { + "type": "path", + "name": "edata_visits_index", + "expr": "$.edata.visits[*].index" + }, + { + "type": "path", + "name": "device_loc_state", + "expr": "$.devicedata.state" + }, + { + "type": "path", + "name": "device_loc_state_code", + "expr": "$.devicedata.statecode" + }, + { + "type": "path", + "name": "device_loc_city", + "expr": "$.devicedata.city" + }, + { + "type": "path", + "name": "device_loc_country_code", + "expr": "$.devicedata.countrycode" + }, + { + "type": "path", + "name": "device_loc_country", + "expr": "$.devicedata.country" + }, + { + "type": "path", + "name": "device_os", + "expr": "$.devicedata.devicespec.os" + }, + { + "type": "path", + "name": "device_make", + "expr": "$.devicedata.devicespec.make" + }, + { + "type": "path", + "name": "device_id", + "expr": "$.devicedata.devicespec.id" + }, + { + "type": "path", + "name": "device_mem", + "expr": "$.devicedata.devicespec.mem" + }, + { + "type": "path", + "name": "device_idisk", + "expr": "$.devicedata.devicespec.idisk" + }, + { + "type": "path", + "name": "device_edisk", + "expr": "$.devicedata.devicespec.edisk" + }, + { + "type": "path", + "name": "device_scrn", + "expr": "$.devicedata.devicespec.scrn" + }, + { + "type": "path", + "name": "device_camera", + "expr": "$.devicedata.devicespec.camera" + }, + { + "type": "path", + "name": "device_cpu", + "expr": "$.devicedata.devicespec.cpu" + }, + { + "type": "path", + "name": "device_sims", + "expr": "$.devicedata.devicespec.sims" + }, + { + "type": "path", + "name": "device_uaspec_agent", + "expr": "$.devicedata.uaspec.agent" + }, + { + "type": "path", + "name": "device_uaspec_ver", + "expr": "$.devicedata.uaspec.ver" + }, + { + "type": "path", + "name": "device_uaspec_system", + "expr": "$.devicedata.uaspec.system" + }, + { + "type": "path", + "name": "device_uaspec_platform", + "expr": "$.devicedata.uaspec.platform" + }, + { + "type": "path", + "name": "device_uaspec_raw", + "expr": "$.devicedata.uaspec.raw" + }, + { + "type": "path", + "name": "device_first_access", + "expr": "$.devicedata.firstaccess" + }, + { + "type": "path", + "name": "device_loc_state_custom_code", + "expr": "$.devicedata.statecustomcode" + }, + { + "type": "path", + "name": "device_loc_state_custom_name", + "expr": "$.devicedata.statecustomname" + }, + { + "type": "path", + "name": "device_loc_district", + "expr": "$.devicedata.districtcustom" + }, + { + "type": "path", + "name": "user_declared_state", + "expr": "$.devicedata.userdeclared.state" + }, + { + "type": "path", + "name": "user_declared_district", + "expr": "$.devicedata.userdeclared.district" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_object_type", + "expr": "$.contentdata.objecttype" + }, + { + "type": "path", + "name": "content_type", + "expr": "$.contentdata.contenttype" + }, + { + "type": "path", + "name": "content_media_type", + "expr": "$.contentdata.mediatype" + }, + { + "type": "path", + "name": "content_language", + "expr": "$.contentdata.language[*]" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subjects", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "content_framework", + "expr": "$.contentdata.framework" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_status", + "expr": "$.contentdata.status" + }, + { + "type": "path", + "name": "content_version", + "expr": "$.contentdata.pkgversion" + }, + { + "type": "path", + "name": "content_last_submitted_on", + "expr": "$.contentdata.lastsubmittedon" + }, + { + "type": "path", + "name": "content_last_published_on", + "expr": "$.contentdata.lastpublishedon" + }, + { + "type": "path", + "name": "content_last_updated_on", + "expr": "$.contentdata.lastupdatedon" + }, + { + "type": "path", + "name": "content_created_by", + "expr": "$.contentdata.createdby" + }, + { + "type": "path", + "name": "content_created_for", + "expr": "$.contentdata.createdfor" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_object_type", + "expr": "$.collectiondata.objecttype" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_media_type", + "expr": "$.collectiondata.mediatype" + }, + { + "type": "path", + "name": "collection_language", + "expr": "$.collectiondata.language[*]" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "collection_subjects", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_mimetype", + "expr": "$.collectiondata.mimetype" + }, + { + "type": "path", + "name": "collection_framework", + "expr": "$.collectiondata.framework" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_status", + "expr": "$.collectiondata.status" + }, + { + "type": "path", + "name": "collection_version", + "expr": "$.collectiondata.pkgversion" + }, + { + "type": "path", + "name": "collection_last_submitted_on", + "expr": "$.collectiondata.lastsubmittedon" + }, + { + "type": "path", + "name": "collection_last_published_on", + "expr": "$.collectiondata.lastpublishedon" + }, + { + "type": "path", + "name": "collection_last_updated_on", + "expr": "$.collectiondata.lastupdatedon" + }, + { + "type": "path", + "name": "collection_created_by", + "expr": "$.collectiondata.createdby" + }, + { + "type": "path", + "name": "collection_created_for", + "expr": "$.collectiondata.createdfor" + }, + { + "type": "path", + "name": "user_grade_list", + "expr": "$.userdata.gradelist[*]" + }, + { + "type": "path", + "name": "user_language_list", + "expr": "$.userdata.languagelist[*]" + }, + { + "type": "path", + "name": "user_subject_list", + "expr": "$.userdata.subjectlist[*]" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_roles", + "expr": "$.userdata.roles[*]" + }, + { + "type": "path", + "name": "user_loc_state", + "expr": "$.userdata.state" + }, + { + "type": "path", + "name": "user_loc_district", + "expr": "$.userdata.district" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "dialcode_channel", + "expr": "$.dialcodedata.channel" + }, + { + "type": "path", + "name": "dialcode_batchcode", + "expr": "$.dialcodedata.batchcode" + }, + { + "type": "path", + "name": "dialcode_publisher", + "expr": "$.dialcodedata.publisher" + }, + { + "type": "path", + "name": "dialcode_generated_on", + "expr": "$.dialcodedata.generatedon" + }, + { + "type": "path", + "name": "dialcode_published_on", + "expr": "$.dialcodedata.publishedon" + }, + { + "type": "path", + "name": "dialcode_status", + "expr": "$.dialcodedata.status" + }, + { + "type": "path", + "name": "dialcode_object_type", + "expr": "$.dialcodedata.objecttype" + }, + { + "type": "path", + "name": "device_loc_iso_state_code", + "expr": "$.devicedata.iso3166statecode" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "mid" + }, + { + "type": "long", + "name": "syncts" + }, + { + "type": "string", + "name": "@timestamp" + }, + { + "type": "string", + "name": "actor_id" + }, + { + "type": "string", + "name": "actor_type" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "context_sid" + }, + { + "type": "string", + "name": "context_did" + }, + { + "name": "context_cdata_type" + }, + { + "name": "context_cdata_id" + }, + { + "type": "string", + "name": "context_rollup_l1" + }, + { + "type": "string", + "name": "context_rollup_l2" + }, + { + "type": "string", + "name": "context_rollup_l3" + }, + { + "type": "string", + "name": "context_rollup_l4" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_version" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "object_rollup_l2" + }, + { + "type": "string", + "name": "object_rollup_l3" + }, + { + "type": "string", + "name": "object_rollup_l4" + }, + { + "name": "tags" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "type": "string", + "name": "edata_subtype" + }, + { + "type": "string", + "name": "edata_mode" + }, + { + "type": "string", + "name": "edata_pageid" + }, + { + "type": "string", + "name": "edata_uri" + }, + { + "type": "string", + "name": "edata_id" + }, + { + "type": "double", + "name": "edata_duration" + }, + { + "type": "long", + "name": "edata_index" + }, + { + "type": "string", + "name": "edata_pass" + }, + { + "type": "double", + "name": "edata_score" + }, + { + "name": "edata_resvalues" + }, + { + "type": "string", + "name": "edata_item_id" + }, + { + "type": "string", + "name": "edata_item_title" + }, + { + "type": "double", + "name": "edata_item_maxscore" + }, + { + "type": "string", + "name": "edata_target_id" + }, + { + "type": "string", + "name": "edata_target_type" + }, + { + "type": "long", + "name": "edata_rating" + }, + { + "type": "string", + "name": "edata_comments" + }, + { + "type": "string", + "name": "edata_dir" + }, + { + "type": "string", + "name": "edata_items_id" + }, + { + "type": "string", + "name": "edata_items_type" + }, + { + "type": "string", + "name": "edata_items_origin_id" + }, + { + "type": "string", + "name": "edata_items_origin_type" + }, + { + "type": "string", + "name": "edata_items_to_id" + }, + { + "type": "string", + "name": "edata_items_to_type" + }, + { + "type": "string", + "name": "edata_plugin_id" + }, + { + "type": "string", + "name": "edata_plugin_ver" + }, + { + "type": "string", + "name": "edata_plugin_category" + }, + { + "name": "edata_props" + }, + { + "type": "string", + "name": "edata_state" + }, + { + "type": "string", + "name": "edata_prevstate" + }, + { + "type": "long", + "name": "edata_size" + }, + { + "name": "edata_filters_dialcodes" + }, + { + "type": "string", + "name": "edata_topn_identifier" + }, + { + "type": "string", + "name": "edata_visits_objid" + }, + { + "type": "string", + "name": "edata_visits_objtype" + }, + { + "type": "string", + "name": "edata_visits_objver" + }, + { + "type": "string", + "name": "edata_visits_index" + }, + { + "type": "string", + "name": "device_loc_state" + }, + { + "type": "string", + "name": "device_loc_state_code" + }, + { + "type": "string", + "name": "device_loc_city" + }, + { + "type": "string", + "name": "device_loc_country_code" + }, + { + "type": "string", + "name": "device_loc_country" + }, + { + "type": "string", + "name": "device_os" + }, + { + "type": "string", + "name": "device_make" + }, + { + "type": "string", + "name": "device_id" + }, + { + "type": "long", + "name": "device_mem" + }, + { + "type": "string", + "name": "device_idisk" + }, + { + "type": "string", + "name": "device_edisk" + }, + { + "type": "string", + "name": "device_scrn" + }, + { + "type": "string", + "name": "device_camera" + }, + { + "type": "string", + "name": "device_cpu" + }, + { + "type": "long", + "name": "device_sims" + }, + { + "type": "string", + "name": "device_uaspec_agent" + }, + { + "type": "string", + "name": "device_uaspec_ver" + }, + { + "type": "string", + "name": "device_uaspec_system" + }, + { + "type": "string", + "name": "device_uaspec_platform" + }, + { + "type": "string", + "name": "device_uaspec_raw" + }, + { + "type": "long", + "name": "device_first_access" + }, + { + "type": "string", + "name": "device_loc_state_custom_code" + }, + { + "type": "string", + "name": "device_loc_state_custom_name" + }, + { + "type": "string", + "name": "device_loc_district" + }, + { + "type": "string", + "name": "user_declared_state" + }, + { + "type": "string", + "name": "user_declared_district" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_object_type" + }, + { + "type": "string", + "name": "content_type" + }, + { + "type": "string", + "name": "content_media_type" + }, + { + "name": "content_language" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subjects" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "content_framework" + }, + { + "type": "string", + "name": "content_board" + }, + { + "type": "string", + "name": "content_status" + }, + { + "type": "double", + "name": "content_version" + }, + { + "type": "long", + "name": "content_last_submitted_on" + }, + { + "type": "long", + "name": "content_last_published_on" + }, + { + "type": "long", + "name": "content_last_updated_on" + }, + { + "type": "string", + "name": "content_created_by" + }, + { + "name": "content_created_for" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_object_type" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "type": "string", + "name": "collection_media_type" + }, + { + "name": "collection_language" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_subjects" + }, + { + "type": "string", + "name": "collection_mimetype" + }, + { + "type": "string", + "name": "collection_framework" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "type": "string", + "name": "collection_status" + }, + { + "type": "double", + "name": "collection_version" + }, + { + "type": "long", + "name": "collection_last_submitted_on" + }, + { + "type": "long", + "name": "collection_last_published_on" + }, + { + "type": "long", + "name": "collection_last_updated_on" + }, + { + "type": "string", + "name": "collection_created_by" + }, + { + "name": "collection_created_for" + }, + { + "name": "user_grade_list" + }, + { + "name": "user_language_list" + }, + { + "name": "user_subject_list" + }, + { + "type": "string", + "name": "user_type" + }, + { + "name": "user_roles" + }, + { + "type": "string", + "name": "user_loc_state" + }, + { + "type": "string", + "name": "user_loc_district" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "dialcode_channel" + }, + { + "type": "string", + "name": "dialcode_batchcode" + }, + { + "type": "string", + "name": "dialcode_publisher" + }, + { + "type": "long", + "name": "dialcode_generated_on" + }, + { + "type": "long", + "name": "dialcode_published_on" + }, + { + "type": "string", + "name": "dialcode_status" + }, + { + "type": "string", + "name": "dialcode_object_type" + }, + { + "type": "string", + "name": "device_loc_iso_state_code" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "ets", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + }, + "transformSpec": { + "filter": { + "type": "selector", + "dimension": "context_pdata_id", + "value": "{{desktop_pdata_id}}" + } + } + }, + "ioConfig": { + "topic": "{{env}}.druid.events.telemetry", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_offline_desktop_taskcount}}, + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": false, + "completionTimeout" : "PT1800S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 2000000 + } +} diff --git a/ansible/roles/druid-ingestion/templates/raw_summary_events b/ansible/roles/druid-ingestion/templates/raw_summary_events new file mode 100644 index 0000000000..3fe8bcfcfc --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_summary_events @@ -0,0 +1,1211 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "summary-events", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "root", + "name": "mid" + }, + { + "type": "root", + "name": "ver" + }, + { + "type": "root", + "name": "ets" + }, + { + "type": "root", + "name": "uid" + }, + { + "type": "root", + "name": "syncts" + }, + { + "type": "path", + "name": "context_date_range_from", + "expr": "$.context.date_range.from" + }, + { + "type": "path", + "name": "context_date_range_to", + "expr": "$.context.date_range.to" + }, + { + "type": "path", + "name": "context_cdata_type", + "expr": "$.context.cdata[*].type" + }, + { + "type": "path", + "name": "context_cdata_id", + "expr": "$.context.cdata[*].id" + }, + { + "type": "path", + "name": "context_rollup_l1", + "expr": "$.context.rollup.l1" + }, + { + "type": "path", + "name": "context_rollup_l2", + "expr": "$.context.rollup.l2" + }, + { + "type": "path", + "name": "context_rollup_l3", + "expr": "$.context.rollup.l3" + }, + { + "type": "path", + "name": "context_rollup_l4", + "expr": "$.context.rollup.l4" + }, + { + "type": "path", + "name": "dimension_channel", + "expr": "$.dimensions.channel" + }, + { + "type": "path", + "name": "dimensions_did", + "expr": "$.dimensions.did" + }, + { + "type": "path", + "name": "dimensions_pdata_id", + "expr": "$.dimensions.pdata.id" + }, + { + "type": "path", + "name": "dimensions_pdata_pid", + "expr": "$.dimensions.pdata.pid" + }, + { + "type": "path", + "name": "dimensions_pdata_ver", + "expr": "$.dimensions.pdata.ver" + }, + { + "type": "path", + "name": "dimensions_sid", + "expr": "$.dimensions.sid" + }, + { + "type": "path", + "name": "dimensions_type", + "expr": "$.dimensions.type" + }, + { + "type": "path", + "name": "dimensions_mode", + "expr": "$.dimensions.mode" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_version", + "expr": "$.object.ver" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "object_rollup_l2", + "expr": "$.object.rollup.l2" + }, + { + "type": "path", + "name": "object_rollup_l3", + "expr": "$.object.rollup.l3" + }, + { + "type": "path", + "name": "object_rollup_l4", + "expr": "$.object.rollup.l4" + }, + { + "type": "root", + "name": "tags" + }, + { + "type": "path", + "name": "edata_time_spent", + "expr": "$.edata.eks.time_spent" + }, + { + "type": "path", + "name": "edata_time_difference", + "expr": "$.edata.eks.time_diff" + }, + { + "type": "path", + "name": "edata_interaction_count", + "expr": "$.edata.eks.interact_events_count" + }, + { + "type": "path", + "name": "edata_env_summary_env", + "expr": "$.edata.eks.env_summary[*].env" + }, + { + "type": "path", + "name": "edata_env_summary_count", + "expr": "$.edata.eks.env_summary[*].count" + }, + { + "type": "path", + "name": "edata_env_summary_time_spent", + "expr": "$.edata.eks.env_summary[*].time_spent" + }, + { + "type": "path", + "name": "edata_page_summary_id", + "expr": "$.edata.eks.page_summary[*].id" + }, + { + "type": "path", + "name": "edata_page_summary_type", + "expr": "$.edata.eks.page_summary[*].type" + }, + { + "type": "path", + "name": "edata_page_summary_env", + "expr": "$.edata.eks.page_summary[*].env" + }, + { + "type": "path", + "name": "edata_page_summary_visit_count", + "expr": "$.edata.eks.page_summary[*].visit_count" + }, + { + "type": "path", + "name": "edata_page_summary_time_spent", + "expr": "$.edata.eks.page_summary[*].time_spent" + }, + { + "type": "path", + "name": "edata_item_responses_item_id", + "expr": "$.edata.eks.item_responses[*].itemId" + }, + { + "type": "path", + "name": "edata_item_responses_time_spent", + "expr": "$.edata.eks.item_responses[*].timeSpent" + }, + { + "type": "path", + "name": "edata_item_responses_pass", + "expr": "$.edata.eks.item_responses[*].pass" + }, + { + "type": "path", + "name": "edata_item_responses_score", + "expr": "$.edata.eks.item_responses[*].score" + }, + { + "type": "path", + "name": "edata_item_responses_max_score", + "expr": "$.edata.eks.item_responses[*].maxScore" + }, + { + "type": "path", + "name": "edata_item_responses_timestamp", + "expr": "$.edata.eks.item_responses[*].time_stamp" + }, + { + "type": "path", + "name": "device_loc_state", + "expr": "$.devicedata.state" + }, + { + "type": "path", + "name": "device_loc_state_code", + "expr": "$.devicedata.statecode" + }, + { + "type": "path", + "name": "device_loc_city", + "expr": "$.devicedata.city" + }, + { + "type": "path", + "name": "device_loc_country_code", + "expr": "$.devicedata.countrycode" + }, + { + "type": "path", + "name": "device_loc_country", + "expr": "$.devicedata.country" + }, + { + "type": "path", + "name": "device_os", + "expr": "$.devicedata.devicespec.os" + }, + { + "type": "path", + "name": "device_make", + "expr": "$.devicedata.devicespec.make" + }, + { + "type": "path", + "name": "device_id", + "expr": "$.devicedata.devicespec.id" + }, + { + "type": "path", + "name": "device_mem", + "expr": "$.devicedata.devicespec.mem" + }, + { + "type": "path", + "name": "device_idisk", + "expr": "$.devicedata.devicespec.idisk" + }, + { + "type": "path", + "name": "device_edisk", + "expr": "$.devicedata.devicespec.edisk" + }, + { + "type": "path", + "name": "device_scrn", + "expr": "$.devicedata.devicespec.scrn" + }, + { + "type": "path", + "name": "device_camera", + "expr": "$.devicedata.devicespec.camera" + }, + { + "type": "path", + "name": "device_cpu", + "expr": "$.devicedata.devicespec.cpu" + }, + { + "type": "path", + "name": "device_sims", + "expr": "$.devicedata.devicespec.sims" + }, + { + "type": "path", + "name": "device_uaspec_agent", + "expr": "$.devicedata.uaspec.agent" + }, + { + "type": "path", + "name": "device_uaspec_ver", + "expr": "$.devicedata.uaspec.ver" + }, + { + "type": "path", + "name": "device_uaspec_system", + "expr": "$.devicedata.uaspec.system" + }, + { + "type": "path", + "name": "device_uaspec_platform", + "expr": "$.devicedata.uaspec.platform" + }, + { + "type": "path", + "name": "device_uaspec_raw", + "expr": "$.devicedata.uaspec.raw" + }, + { + "type": "path", + "name": "device_first_access", + "expr": "$.devicedata.firstaccess" + }, + { + "type": "path", + "name": "device_loc_state_custom_code", + "expr": "$.devicedata.statecustomcode" + }, + { + "type": "path", + "name": "device_loc_state_custom_name", + "expr": "$.devicedata.statecustomname" + }, + { + "type": "path", + "name": "device_loc_district", + "expr": "$.devicedata.districtcustom" + }, + { + "type": "path", + "name": "user_declared_state", + "expr": "$.devicedata.userdeclared.state" + }, + { + "type": "path", + "name": "user_declared_district", + "expr": "$.devicedata.userdeclared.district" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_object_type", + "expr": "$.contentdata.objecttype" + }, + { + "type": "path", + "name": "content_type", + "expr": "$.contentdata.contenttype" + }, + { + "type": "path", + "name": "content_media_type", + "expr": "$.contentdata.mediatype" + }, + { + "type": "path", + "name": "content_language", + "expr": "$.contentdata.language[*]" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subjects", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "content_framework", + "expr": "$.contentdata.framework" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_status", + "expr": "$.contentdata.status" + }, + { + "type": "path", + "name": "content_created_by", + "expr": "$.contentdata.createdby" + }, + { + "type": "path", + "name": "content_created_for", + "expr": "$.contentdata.createdfor" + }, + { + "type": "path", + "name": "content_version", + "expr": "$.contentdata.pkgversion" + }, + { + "type": "path", + "name": "content_last_submitted_on", + "expr": "$.contentdata.lastsubmittedon" + }, + { + "type": "path", + "name": "content_last_published_on", + "expr": "$.contentdata.lastpublishedon" + }, + { + "type": "path", + "name": "content_last_updated_on", + "expr": "$.contentdata.lastupdatedon" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_object_type", + "expr": "$.collectiondata.objecttype" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_media_type", + "expr": "$.collectiondata.mediatype" + }, + { + "type": "path", + "name": "collection_language", + "expr": "$.collectiondata.language[*]" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "collection_subjects", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_mimetype", + "expr": "$.collectiondata.mimetype" + }, + { + "type": "path", + "name": "collection_framework", + "expr": "$.collectiondata.framework" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_status", + "expr": "$.collectiondata.status" + }, + { + "type": "path", + "name": "collection_version", + "expr": "$.collectiondata.pkgversion" + }, + { + "type": "path", + "name": "collection_last_submitted_on", + "expr": "$.collectiondata.lastsubmittedon" + }, + { + "type": "path", + "name": "collection_last_published_on", + "expr": "$.collectiondata.lastpublishedon" + }, + { + "type": "path", + "name": "collection_last_updated_on", + "expr": "$.collectiondata.lastupdatedon" + }, + { + "type": "path", + "name": "collection_created_by", + "expr": "$.collectiondata.createdby" + }, + { + "type": "path", + "name": "collection_created_for", + "expr": "$.collectiondata.createdfor" + }, + { + "type": "path", + "name": "user_grade_list", + "expr": "$.userdata.gradelist[*]" + }, + { + "type": "path", + "name": "user_language_list", + "expr": "$.userdata.languagelist[*]" + }, + { + "type": "path", + "name": "user_subject_list", + "expr": "$.userdata.subjectlist[*]" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_roles", + "expr": "$.userdata.roles[*]" + }, + { + "type": "path", + "name": "user_loc_state", + "expr": "$.userdata.state" + }, + { + "type": "path", + "name": "user_loc_district", + "expr": "$.userdata.district" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "device_loc_iso_state_code", + "expr": "$.devicedata.iso3166statecode" + }, +{ + "type": "path", + "name": "object_l2_created_for", + "expr": "$.l2data.createdfor" + }, + { + "type": "path", + "name": "object_l2_framework", + "expr": "$.l2data.framework" + }, + { + "type": "path", + "name": "object_l2_name", + "expr": "$.l2data.name" + }, + { + "type": "path", + "name": "object_l2_channel", + "expr": "$.l2data.channel" + }, + { + "type": "path", + "name": "object_l2_mimetype", + "expr": "$.l2data.mimetype" + }, + { + "type": "path", + "name": "object_l2_medium", + "expr": "$.l2data.medium[*]" + }, + { + "type": "path", + "name": "object_l2_board", + "expr": "$.l2data.board" + }, + { + "type": "path", + "name": "object_l2_contenttype", + "expr": "$.l2data.contenttype" + }, + { + "type": "path", + "name": "object_l2_gradelevel", + "expr": "$.l2data.gradelevel[*]" + }, + { + "type": "path", + "name": "object_l2_subjects", + "expr": "$.l2data.subject[*]" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "mid" + }, + { + "type": "string", + "name": "ver" + }, + { + "type": "long", + "name": "syncts" + }, + { + "type": "string", + "name": "uid" + }, + { + "type": "long", + "name": "context_date_range_from" + }, + { + "name": "context_cdata_type" + }, + { + "name": "context_cdata_id" + }, + { + "type": "string", + "name": "context_rollup_l1" + }, + { + "type": "string", + "name": "context_rollup_l2" + }, + { + "type": "string", + "name": "context_rollup_l3" + }, + { + "type": "string", + "name": "context_rollup_l4" + }, + { + "type": "string", + "name": "dimension_channel" + }, + { + "type": "string", + "name": "dimensions_did" + }, + { + "type": "string", + "name": "dimensions_pdata_id" + }, + { + "type": "string", + "name": "dimensions_pdata_pid" + }, + { + "type": "string", + "name": "dimensions_pdata_ver" + }, + { + "type": "string", + "name": "dimensions_sid" + }, + { + "type": "string", + "name": "dimensions_type" + }, + { + "type": "string", + "name": "dimensions_mode" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_version" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "object_rollup_l2" + }, + { + "type": "string", + "name": "object_rollup_l3" + }, + { + "type": "string", + "name": "object_rollup_l4" + }, + { + "name": "tags" + }, + { + "type": "double", + "name": "edata_time_spent" + }, + { + "type": "double", + "name": "edata_time_difference" + }, + { + "type": "long", + "name": "edata_interaction_count" + }, + { + "type": "string", + "name": "edata_env_summary_env" + }, + { + "type": "string", + "name": "edata_env_summary_count" + }, + { + "type": "string", + "name": "edata_env_summary_time_spent" + }, + { + "type": "string", + "name": "edata_page_summary_id" + }, + { + "type": "string", + "name": "edata_page_summary_type" + }, + { + "type": "string", + "name": "edata_page_summary_env" + }, + { + "type": "string", + "name": "edata_page_summary_visit_count" + }, + { + "type": "string", + "name": "edata_page_summary_time_spent" + }, + { + "type": "string", + "name": "edata_item_responses_item_id" + }, + { + "type": "string", + "name": "edata_item_responses_time_spent" + }, + { + "type": "string", + "name": "edata_item_responses_pass" + }, + { + "type": "string", + "name": "edata_item_responses_score" + }, + { + "type": "string", + "name": "edata_item_responses_max_score" + }, + { + "type": "string", + "name": "edata_item_responses_timestamp" + }, + { + "type": "string", + "name": "device_loc_state" + }, + { + "type": "string", + "name": "device_loc_state_code" + }, + { + "type": "string", + "name": "device_loc_city" + }, + { + "type": "string", + "name": "device_loc_country_code" + }, + { + "type": "string", + "name": "device_loc_country" + }, + { + "type": "string", + "name": "device_os" + }, + { + "type": "string", + "name": "device_make" + }, + { + "type": "string", + "name": "device_id" + }, + { + "type": "long", + "name": "device_mem" + }, + { + "type": "string", + "name": "device_idisk" + }, + { + "type": "string", + "name": "device_edisk" + }, + { + "type": "string", + "name": "device_scrn" + }, + { + "type": "string", + "name": "device_camera" + }, + { + "type": "string", + "name": "device_cpu" + }, + { + "type": "long", + "name": "device_sims" + }, + { + "type": "string", + "name": "device_uaspec_agent" + }, + { + "type": "string", + "name": "device_uaspec_ver" + }, + { + "type": "string", + "name": "device_uaspec_system" + }, + { + "type": "string", + "name": "device_uaspec_platform" + }, + { + "type": "string", + "name": "device_uaspec_raw" + }, + { + "type": "long", + "name": "device_first_access" + }, + { + "type": "string", + "name": "device_loc_state_custom_code" + }, + { + "type": "string", + "name": "device_loc_state_custom_name" + }, + { + "type": "string", + "name": "device_loc_district" + }, + { + "type": "string", + "name": "user_declared_state" + }, + { + "type": "string", + "name": "user_declared_district" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_object_type" + }, + { + "type": "string", + "name": "content_type" + }, + { + "type": "string", + "name": "content_media_type" + }, + { + "name": "content_language" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subjects" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "content_framework" + }, + { + "type": "string", + "name": "content_board" + }, + { + "type": "string", + "name": "content_status" + }, + { + "type": "double", + "name": "content_version" + }, + { + "type": "long", + "name": "content_last_submitted_on" + }, + { + "type": "long", + "name": "content_last_published_on" + }, + { + "type": "long", + "name": "content_last_updated_on" + }, + { + "type": "string", + "name": "content_created_by" + }, + { + "name": "content_created_for" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_object_type" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "type": "string", + "name": "collection_media_type" + }, + { + "name": "collection_language" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_subjects" + }, + { + "type": "string", + "name": "collection_mimetype" + }, + { + "type": "string", + "name": "collection_framework" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "type": "string", + "name": "collection_status" + }, + { + "type": "double", + "name": "collection_version" + }, + { + "type": "long", + "name": "collection_last_submitted_on" + }, + { + "type": "long", + "name": "collection_last_published_on" + }, + { + "type": "long", + "name": "collection_last_updated_on" + }, + { + "type": "string", + "name": "collection_created_by" + }, + { + "name": "collection_created_for" + }, + { + "name": "user_grade_list" + }, + { + "name": "user_language_list" + }, + { + "name": "user_subject_list" + }, + { + "type": "string", + "name": "user_type" + }, + { + "name": "user_roles" + }, + { + "type": "string", + "name": "user_loc_state" + }, + { + "type": "string", + "name": "user_loc_district" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "device_loc_iso_state_code" + }, + { + "name": "object_l2_created_for" + }, + { + "type": "string", + "name": "object_l2_framework" + }, + { + "type": "string", + "name": "object_l2_name" + }, + { + "type": "string", + "name": "object_l2_channel" + }, + { + "type": "string", + "name": "object_l2_mimetype" + }, + { + "name": "object_l2_medium" + }, + { + "type": "string", + "name": "object_l2_board" + }, + { + "type": "string", + "name": "object_l2_contenttype" + }, + { + "name": "object_l2_gradelevel" + }, + { + "name": "object_l2_subjects" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [ + { + "type": "longSum", + "name": "total_interactions", + "fieldName": "edata_interaction_count" + }, + { + "type": "doubleSum", + "name": "total_time_spent", + "fieldName": "edata_time_spent" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": true + } + }, + "ioConfig": { + "topic": "{{env}}.druid.events.summary", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_summary_events_taskcount}}, + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 2000000 + } +} diff --git a/ansible/roles/druid-ingestion/templates/raw_telemetry_events b/ansible/roles/druid-ingestion/templates/raw_telemetry_events new file mode 100644 index 0000000000..f9d0ac39e4 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_telemetry_events @@ -0,0 +1,1586 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "telemetry-events-syncts", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "root", + "name": "mid" + }, + { + "type": "root", + "name": "ets" + }, + { + "type": "root", + "name": "@timestamp" + }, + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "actor_type", + "expr": "$.actor.type" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "context_hub", + "expr": "$.context.hub" + }, + { + "type": "path", + "name": "context_sid", + "expr": "$.context.sid" + }, + { + "type": "path", + "name": "context_did", + "expr": "$.context.did" + }, + { + "type": "path", + "name": "context_cdata_type", + "expr": "$.context.cdata[*].type" + }, + { + "type": "path", + "name": "context_cdata_id", + "expr": "$.context.cdata[*].id" + }, + { + "type": "path", + "name": "context_rollup_l1", + "expr": "$.context.rollup.l1" + }, + { + "type": "path", + "name": "context_rollup_l2", + "expr": "$.context.rollup.l2" + }, + { + "type": "path", + "name": "context_rollup_l3", + "expr": "$.context.rollup.l3" + }, + { + "type": "path", + "name": "context_rollup_l4", + "expr": "$.context.rollup.l4" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_version", + "expr": "$.object.ver" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "object_rollup_l2", + "expr": "$.object.rollup.l2" + }, + { + "type": "path", + "name": "object_rollup_l3", + "expr": "$.object.rollup.l3" + }, + { + "type": "path", + "name": "object_rollup_l4", + "expr": "$.object.rollup.l4" + }, + { + "type": "root", + "name": "tags" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "edata_query", + "expr": "$.edata.query" + }, + { + "type": "path", + "name": "edata_subtype", + "expr": "$.edata.subtype" + }, + { + "type": "path", + "name": "edata_mode", + "expr": "$.edata.mode" + }, + { + "type": "path", + "name": "edata_pageid", + "expr": "$.edata.pageid" + }, + { + "type": "path", + "name": "edata_uri", + "expr": "$.edata.uri" + }, + { + "type": "path", + "name": "edata_id", + "expr": "$.edata.id" + }, + { + "type": "path", + "name": "edata_duration", + "expr": "$.edata.duration" + }, + { + "type": "path", + "name": "edata_index", + "expr": "$.edata.index" + }, + { + "type": "path", + "name": "edata_pass", + "expr": "$.edata.pass" + }, + { + "type": "path", + "name": "edata_score", + "expr": "$.edata.score" + }, + { + "type": "path", + "name": "edata_resvalues", + "expr": "$.edata.resvalues[*]" + }, + { + "type": "path", + "name": "edata_item_id", + "expr": "$.edata.item.id" + }, + { + "type": "path", + "name": "edata_item_type", + "expr": "$.edata.item.type" + }, + { + "type": "path", + "name": "edata_item_title", + "expr": "$.edata.item.title" + }, + { + "type": "path", + "name": "edata_item_maxscore", + "expr": "$.edata.item.maxscore" + }, + { + "type": "path", + "name": "edata_target_id", + "expr": "$.edata.target.id" + }, + { + "type": "path", + "name": "edata_target_type", + "expr": "$.edata.target.type" + }, + { + "type": "path", + "name": "edata_rating", + "expr": "$.edata.rating" + }, + { + "type": "path", + "name": "edata_comments", + "expr": "$.edata.comments" + }, + { + "type": "path", + "name": "edata_dir", + "expr": "$.edata.dir" + }, + { + "type": "path", + "name": "edata_items_id", + "expr": "$.edata.items[*].id" + }, + { + "type": "path", + "name": "edata_items_type", + "expr": "$.edata.items[*].type" + }, + { + "type": "path", + "name": "edata_items_origin_id", + "expr": "$.edata.items[*].origin.id" + }, + { + "type": "path", + "name": "edata_items_origin_type", + "expr": "$.edata.items[*].origin.type" + }, + { + "type": "path", + "name": "edata_items_to_id", + "expr": "$.edata.items[*].to.id" + }, + { + "type": "path", + "name": "edata_items_to_type", + "expr": "$.edata.items[*].to.type" + }, + { + "type": "path", + "name": "edata_plugin_id", + "expr": "$.edata.plugin.id" + }, + { + "type": "path", + "name": "edata_plugin_ver", + "expr": "$.edata.plugin.ver" + }, + { + "type": "path", + "name": "edata_plugin_category", + "expr": "$.edata.plugin.category" + }, + { + "type": "path", + "name": "edata_props", + "expr": "$.edata.props[*]" + }, + { + "type": "path", + "name": "edata_state", + "expr": "$.edata.state" + }, + { + "type": "path", + "name": "edata_prevstate", + "expr": "$.edata.prevstate" + }, + { + "type": "path", + "name": "edata_size", + "expr": "$.edata.size" + }, + { + "type": "path", + "name": "edata_filters_dialcodes", + "expr": "$.edata.filters.dialcodes" + }, + { + "type": "path", + "name": "edata_topn_identifier", + "expr": "$.edata.topn[*].identifier" + }, + { + "type": "path", + "name": "edata_visits_objid", + "expr": "$.edata.visits[*].objid" + }, + { + "type": "path", + "name": "edata_visits_objtype", + "expr": "$.edata.visits[*].objtype" + }, + { + "type": "path", + "name": "edata_visits_objver", + "expr": "$.edata.visits[*].objver" + }, + { + "type": "path", + "name": "edata_visits_index", + "expr": "$.edata.visits[*].index" + }, + { + "type": "path", + "name": "edata_timespent", + "expr": "$.edata.timespent" + }, + { + "type": "path", + "name": "edata_pageviews", + "expr": "$.edata.pageviews" + }, + { + "type": "path", + "name": "edata_interactions", + "expr": "$.edata.interactions" + }, + { + "type": "path", + "name": "edata_starttime", + "expr": "$.edata.starttime" + }, + { + "type": "path", + "name": "edata_endtime", + "expr": "$.edata.endtime" + }, + { + "type": "path", + "name": "device_loc_state", + "expr": "$.devicedata.state" + }, + { + "type": "path", + "name": "device_loc_state_code", + "expr": "$.devicedata.statecode" + }, + { + "type": "path", + "name": "device_loc_city", + "expr": "$.devicedata.city" + }, + { + "type": "path", + "name": "device_loc_country_code", + "expr": "$.devicedata.countrycode" + }, + { + "type": "path", + "name": "device_loc_country", + "expr": "$.devicedata.country" + }, + { + "type": "path", + "name": "device_os", + "expr": "$.devicedata.devicespec.os" + }, + { + "type": "path", + "name": "device_make", + "expr": "$.devicedata.devicespec.make" + }, + { + "type": "path", + "name": "device_id", + "expr": "$.devicedata.devicespec.id" + }, + { + "type": "path", + "name": "device_mem", + "expr": "$.devicedata.devicespec.mem" + }, + { + "type": "path", + "name": "device_idisk", + "expr": "$.devicedata.devicespec.idisk" + }, + { + "type": "path", + "name": "device_edisk", + "expr": "$.devicedata.devicespec.edisk" + }, + { + "type": "path", + "name": "device_scrn", + "expr": "$.devicedata.devicespec.scrn" + }, + { + "type": "path", + "name": "device_camera", + "expr": "$.devicedata.devicespec.camera" + }, + { + "type": "path", + "name": "device_cpu", + "expr": "$.devicedata.devicespec.cpu" + }, + { + "type": "path", + "name": "device_sims", + "expr": "$.devicedata.devicespec.sims" + }, + { + "type": "path", + "name": "device_uaspec_agent", + "expr": "$.devicedata.uaspec.agent" + }, + { + "type": "path", + "name": "device_uaspec_ver", + "expr": "$.devicedata.uaspec.ver" + }, + { + "type": "path", + "name": "device_uaspec_system", + "expr": "$.devicedata.uaspec.system" + }, + { + "type": "path", + "name": "device_uaspec_platform", + "expr": "$.devicedata.uaspec.platform" + }, + { + "type": "path", + "name": "device_uaspec_raw", + "expr": "$.devicedata.uaspec.raw" + }, + { + "type": "path", + "name": "device_first_access", + "expr": "$.devicedata.firstaccess" + }, + { + "type": "path", + "name": "device_loc_state_custom_code", + "expr": "$.devicedata.statecustomcode" + }, + { + "type": "path", + "name": "device_loc_state_custom_name", + "expr": "$.devicedata.statecustomname" + }, + { + "type": "path", + "name": "device_loc_district", + "expr": "$.devicedata.districtcustom" + }, + { + "type": "path", + "name": "user_declared_state", + "expr": "$.devicedata.userdeclared.state" + }, + { + "type": "path", + "name": "user_declared_district", + "expr": "$.devicedata.userdeclared.district" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_channel", + "expr": "$.contentdata.channel" + }, + { + "type": "path", + "name": "content_object_type", + "expr": "$.contentdata.objecttype" + }, + { + "type": "path", + "name": "content_type", + "expr": "$.contentdata.contenttype" + }, + { + "type": "path", + "name": "content_media_type", + "expr": "$.contentdata.mediatype" + }, + { + "type": "path", + "name": "content_language", + "expr": "$.contentdata.language[*]" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subjects", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "content_framework", + "expr": "$.contentdata.framework" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_status", + "expr": "$.contentdata.status" + }, + { + "type": "path", + "name": "content_version", + "expr": "$.contentdata.pkgversion" + }, + { + "type": "path", + "name": "content_last_submitted_on", + "expr": "$.contentdata.lastsubmittedon" + }, + { + "type": "path", + "name": "content_last_published_on", + "expr": "$.contentdata.lastpublishedon" + }, + { + "type": "path", + "name": "content_last_updated_on", + "expr": "$.contentdata.lastupdatedon" + }, + { + "type": "path", + "name": "content_created_by", + "expr": "$.contentdata.createdby" + }, + { + "type": "path", + "name": "content_created_for", + "expr": "$.contentdata.createdfor" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_channel", + "expr": "$.collectiondata.channel" + }, + { + "type": "path", + "name": "collection_object_type", + "expr": "$.collectiondata.objecttype" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_media_type", + "expr": "$.collectiondata.mediatype" + }, + { + "type": "path", + "name": "collection_language", + "expr": "$.collectiondata.language[*]" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "collection_subjects", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_mimetype", + "expr": "$.collectiondata.mimetype" + }, + { + "type": "path", + "name": "collection_framework", + "expr": "$.collectiondata.framework" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_status", + "expr": "$.collectiondata.status" + }, + { + "type": "path", + "name": "collection_version", + "expr": "$.collectiondata.pkgversion" + }, + { + "type": "path", + "name": "collection_last_submitted_on", + "expr": "$.collectiondata.lastsubmittedon" + }, + { + "type": "path", + "name": "collection_last_published_on", + "expr": "$.collectiondata.lastpublishedon" + }, + { + "type": "path", + "name": "collection_last_updated_on", + "expr": "$.collectiondata.lastupdatedon" + }, + { + "type": "path", + "name": "collection_created_by", + "expr": "$.collectiondata.createdby" + }, + { + "type": "path", + "name": "collection_created_for", + "expr": "$.collectiondata.createdfor" + }, + { + "type": "path", + "name": "user_grade_list", + "expr": "$.userdata.gradelist[*]" + }, + { + "type": "path", + "name": "user_language_list", + "expr": "$.userdata.languagelist[*]" + }, + { + "type": "path", + "name": "user_subject_list", + "expr": "$.userdata.subjectlist[*]" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_roles", + "expr": "$.userdata.roles[*]" + }, + { + "type": "path", + "name": "user_loc_state", + "expr": "$.userdata.state" + }, + { + "type": "path", + "name": "user_loc_district", + "expr": "$.userdata.district" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "user_loc_block", + "expr": "$.userdata.block" + }, + { + "type": "path", + "name": "user_loc_cluster", + "expr": "$.userdata.cluster" + }, + { + "type": "path", + "name": "user_school_name", + "expr": "$.userdata.schoolname" + }, + { + "type": "path", + "name": "dialcode_channel", + "expr": "$.dialcodedata.channel" + }, + { + "type": "path", + "name": "dialcode_batchcode", + "expr": "$.dialcodedata.batchcode" + }, + { + "type": "path", + "name": "dialcode_publisher", + "expr": "$.dialcodedata.publisher" + }, + { + "type": "path", + "name": "dialcode_generated_on", + "expr": "$.dialcodedata.generatedon" + }, + { + "type": "path", + "name": "dialcode_published_on", + "expr": "$.dialcodedata.publishedon" + }, + { + "type": "path", + "name": "dialcode_object_type", + "expr": "$.dialcodedata.objecttype" + }, + { + "type": "path", + "name": "device_loc_iso_state_code", + "expr": "$.devicedata.iso3166statecode" + }, + { + "type": "path", + "name": "content_play_progress", + "expr": "$.edata.summary[*].progress" + }, + { + "type": "path", + "name": "object_l2_created_for", + "expr": "$.l2data.createdfor" + }, + { + "type": "path", + "name": "object_l2_framework", + "expr": "$.l2data.framework" + }, + { + "type": "path", + "name": "object_l2_name", + "expr": "$.l2data.name" + }, + { + "type": "path", + "name": "object_l2_channel", + "expr": "$.l2data.channel" + }, + { + "type": "path", + "name": "object_l2_mimetype", + "expr": "$.l2data.mimetype" + }, + { + "type": "path", + "name": "object_l2_medium", + "expr": "$.l2data.medium[*]" + }, + { + "type": "path", + "name": "object_l2_board", + "expr": "$.l2data.board" + }, + { + "type": "path", + "name": "object_l2_contenttype", + "expr": "$.l2data.contenttype" + }, + { + "type": "path", + "name": "object_l2_gradelevel", + "expr": "$.l2data.gradelevel[*]" + }, + { + "type": "path", + "name": "object_l2_subjects", + "expr": "$.l2data.subject[*]" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "mid" + }, + { + "type": "long", + "name": "ets" + }, + { + "type": "string", + "name": "@timestamp" + }, + { + "type": "string", + "name": "actor_id" + }, + { + "type": "string", + "name": "actor_type" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "context_hub" + }, + { + "type": "string", + "name": "context_sid" + }, + { + "type": "string", + "name": "context_did" + }, + { + "name": "context_cdata_type" + }, + { + "name": "context_cdata_id" + }, + { + "type": "string", + "name": "context_rollup_l1" + }, + { + "type": "string", + "name": "context_rollup_l2" + }, + { + "type": "string", + "name": "context_rollup_l3" + }, + { + "type": "string", + "name": "context_rollup_l4" + }, + { + "name" : "content_play_progress" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_version" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "object_rollup_l2" + }, + { + "type": "string", + "name": "object_rollup_l3" + }, + { + "type": "string", + "name": "object_rollup_l4" + }, + { + "name": "tags" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "type": "string", + "name": "edata_subtype" + }, + { + "type": "string", + "name": "edata_query" + }, + { + "type": "string", + "name": "edata_mode" + }, + { + "type": "string", + "name": "edata_pageid" + }, + { + "type": "string", + "name": "edata_uri" + }, + { + "type": "string", + "name": "edata_id" + }, + { + "type": "double", + "name": "edata_duration" + }, + { + "type": "long", + "name": "edata_index" + }, + { + "type": "string", + "name": "edata_pass" + }, + { + "type": "double", + "name": "edata_score" + }, + { + "name": "edata_resvalues" + }, + { + "type": "string", + "name": "edata_item_id" + }, + { + "type": "string", + "name": "edata_item_type" + }, + { + "type": "string", + "name": "edata_item_title" + }, + { + "type": "double", + "name": "edata_item_maxscore" + }, + { + "type": "string", + "name": "edata_target_id" + }, + { + "type": "string", + "name": "edata_target_type" + }, + { + "type": "long", + "name": "edata_rating" + }, + { + "type": "string", + "name": "edata_comments" + }, + { + "type": "string", + "name": "edata_dir" + }, + { + "type": "string", + "name": "edata_items_id" + }, + { + "type": "string", + "name": "edata_items_type" + }, + { + "type": "string", + "name": "edata_items_origin_id" + }, + { + "type": "string", + "name": "edata_items_origin_type" + }, + { + "type": "string", + "name": "edata_items_to_id" + }, + { + "type": "string", + "name": "edata_items_to_type" + }, + { + "type": "string", + "name": "edata_plugin_id" + }, + { + "type": "string", + "name": "edata_plugin_ver" + }, + { + "type": "string", + "name": "edata_plugin_category" + }, + { + "name": "edata_props" + }, + { + "type": "string", + "name": "edata_state" + }, + { + "type": "string", + "name": "edata_prevstate" + }, + { + "type": "long", + "name": "edata_size" + }, + { + "name": "edata_filters_dialcodes" + }, + { + "type": "string", + "name": "edata_topn_identifier" + }, + { + "type": "string", + "name": "edata_visits_objid" + }, + { + "type": "string", + "name": "edata_visits_objtype" + }, + { + "type": "string", + "name": "edata_visits_objver" + }, + { + "type": "string", + "name": "edata_visits_index" + }, + { + "type": "double", + "name": "edata_timespent" + }, + { + "type": "long", + "name": "edata_pageviews" + }, + { + "type": "long", + "name": "edata_interactions" + }, + { + "type": "long", + "name": "edata_starttime" + }, + { + "type": "long", + "name": "edata_endtime" + }, + { + "type": "string", + "name": "device_loc_state" + }, + { + "type": "string", + "name": "device_loc_state_code" + }, + { + "type": "string", + "name": "device_loc_city" + }, + { + "type": "string", + "name": "device_loc_country_code" + }, + { + "type": "string", + "name": "device_loc_country" + }, + { + "type": "string", + "name": "device_os" + }, + { + "type": "string", + "name": "device_make" + }, + { + "type": "string", + "name": "device_id" + }, + { + "type": "long", + "name": "device_mem" + }, + { + "type": "string", + "name": "device_idisk" + }, + { + "type": "string", + "name": "device_edisk" + }, + { + "type": "string", + "name": "device_scrn" + }, + { + "type": "string", + "name": "device_camera" + }, + { + "type": "string", + "name": "device_cpu" + }, + { + "type": "long", + "name": "device_sims" + }, + { + "type": "string", + "name": "device_uaspec_agent" + }, + { + "type": "string", + "name": "device_uaspec_ver" + }, + { + "type": "string", + "name": "device_uaspec_system" + }, + { + "type": "string", + "name": "device_uaspec_platform" + }, + { + "type": "string", + "name": "device_uaspec_raw" + }, + { + "type": "long", + "name": "device_first_access" + }, + { + "type": "string", + "name": "device_loc_state_custom_code" + }, + { + "type": "string", + "name": "device_loc_state_custom_name" + }, + { + "type": "string", + "name": "device_loc_district" + }, + { + "type": "string", + "name": "user_declared_state" + }, + { + "type": "string", + "name": "user_declared_district" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_object_type" + }, + { + "type": "string", + "name": "content_type" + }, + { + "type": "string", + "name": "content_media_type" + }, + { + "name": "content_language" + }, + { + "name": "content_channel" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subjects" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "content_framework" + }, + { + "type": "string", + "name": "content_board" + }, + { + "type": "string", + "name": "content_status" + }, + { + "type": "double", + "name": "content_version" + }, + { + "type": "long", + "name": "content_last_submitted_on" + }, + { + "type": "long", + "name": "content_last_published_on" + }, + { + "type": "long", + "name": "content_last_updated_on" + }, + { + "type": "string", + "name": "content_created_by" + }, + { + "name": "content_created_for" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_object_type" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "type": "string", + "name": "collection_media_type" + }, + { + "name": "collection_language" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_channel" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_subjects" + }, + { + "type": "string", + "name": "collection_mimetype" + }, + { + "type": "string", + "name": "collection_framework" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "type": "string", + "name": "collection_status" + }, + { + "type": "double", + "name": "collection_version" + }, + { + "type": "long", + "name": "collection_last_submitted_on" + }, + { + "type": "long", + "name": "collection_last_published_on" + }, + { + "type": "long", + "name": "collection_last_updated_on" + }, + { + "type": "string", + "name": "collection_created_by" + }, + { + "name": "collection_created_for" + }, + { + "name": "user_grade_list" + }, + { + "name": "user_language_list" + }, + { + "name": "user_subject_list" + }, + { + "type": "string", + "name": "user_type" + }, + { + "name": "user_roles" + }, + { + "type": "string", + "name": "user_loc_state" + }, + { + "type": "string", + "name": "user_loc_district" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "user_loc_block" + }, + { + "type": "string", + "name": "user_loc_cluster" + }, + { + "type": "string", + "name": "user_school_name" + }, + { + "type": "string", + "name": "dialcode_channel" + }, + { + "type": "string", + "name": "dialcode_batchcode" + }, + { + "type": "string", + "name": "dialcode_publisher" + }, + { + "type": "long", + "name": "dialcode_generated_on" + }, + { + "type": "long", + "name": "dialcode_published_on" + }, + { + "type": "string", + "name": "dialcode_object_type" + }, + { + "type": "string", + "name": "device_loc_iso_state_code" + }, + { + "name": "object_l2_created_for" + }, + { + "type": "string", + "name": "object_l2_framework" + }, + { + "type": "string", + "name": "object_l2_name" + }, + { + "type": "string", + "name": "object_l2_channel" + }, + { + "type": "string", + "name": "object_l2_mimetype" + }, + { + "name": "object_l2_medium" + }, + { + "type": "string", + "name": "object_l2_board" + }, + { + "type": "string", + "name": "object_l2_contenttype" + }, + { + "name": "object_l2_gradelevel" + }, + { + "name": "object_l2_subjects" + }, + { + "type": "string", + "name": "utm_source" + }, + { + "type": "string", + "name": "utm_content" + }, + { + "type": "string", + "name": "utm_medium" + }, + { + "type": "string", + "name": "utm_campaign" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": false + }, + "transformSpec": { + "transforms": [ + { + "type": "expression", + "name": "utm_source", + "expression": "if(like(\"context_cdata_type\",'UtmSource'), context_cdata_id, '')" + }, + { + "type": "expression", + "name": "utm_content", + "expression": "if(like(\"context_cdata_type\",'UtmContent'), context_cdata_id, '')" + }, + { + "type": "expression", + "name": "utm_medium", + "expression": "if(like(\"context_cdata_type\",'UtmMedium'), context_cdata_id, '')" + }, + { + "type": "expression", + "name": "utm_campaign", + "expression": "if(like(\"context_cdata_type\",'UtmCampaign'), context_cdata_id, '')" + } + ] + } + }, + "ioConfig": { + "topic": "{{env}}.druid.events.telemetry", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_telemetry_events_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "useEarliestOffset": true, + "completionTimeout" : "PT7200S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 2000000 + } +} diff --git a/ansible/roles/druid-ingestion/templates/raw_telemetry_feedback_events b/ansible/roles/druid-ingestion/templates/raw_telemetry_feedback_events new file mode 100644 index 0000000000..2793650757 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_telemetry_feedback_events @@ -0,0 +1,864 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "telemetry-feedback-events", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "root", + "name": "mid" + }, + { + "type": "root", + "name": "ets" + }, + { + "type": "root", + "name": "@timestamp" + }, + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "actor_type", + "expr": "$.actor.type" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "context_sid", + "expr": "$.context.sid" + }, + { + "type": "path", + "name": "context_did", + "expr": "$.context.did" + }, + { + "type": "path", + "name": "context_cdata_type", + "expr": "$.context.cdata[*].type" + }, + { + "type": "path", + "name": "context_cdata_id", + "expr": "$.context.cdata[*].id" + }, + { + "type": "path", + "name": "context_rollup_l1", + "expr": "$.context.rollup.l1" + }, + { + "type": "path", + "name": "context_rollup_l2", + "expr": "$.context.rollup.l2" + }, + { + "type": "path", + "name": "context_rollup_l3", + "expr": "$.context.rollup.l3" + }, + { + "type": "path", + "name": "context_rollup_l4", + "expr": "$.context.rollup.l4" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_version", + "expr": "$.object.ver" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "object_rollup_l2", + "expr": "$.object.rollup.l2" + }, + { + "type": "path", + "name": "object_rollup_l3", + "expr": "$.object.rollup.l3" + }, + { + "type": "path", + "name": "object_rollup_l4", + "expr": "$.object.rollup.l4" + }, + { + "type": "root", + "name": "tags" + }, + { + "type": "path", + "name": "edata_rating", + "expr": "$.edata.rating" + }, + { + "type": "path", + "name": "edata_comments", + "expr": "$.edata.comments" + }, + { + "type": "path", + "name": "edata_commentid", + "expr": "$.edata.commentid" + }, + { + "type": "path", + "name": "edata_commenttxt", + "expr": "$.edata.commenttxt" + }, + { + "type": "path", + "name": "device_loc_state", + "expr": "$.devicedata.state" + }, + { + "type": "path", + "name": "device_loc_state_code", + "expr": "$.devicedata.statecode" + }, + { + "type": "path", + "name": "device_loc_city", + "expr": "$.devicedata.city" + }, + { + "type": "path", + "name": "device_loc_country_code", + "expr": "$.devicedata.countrycode" + }, + { + "type": "path", + "name": "device_loc_country", + "expr": "$.devicedata.country" + }, + { + "type": "path", + "name": "device_os", + "expr": "$.devicedata.devicespec.os" + }, + { + "type": "path", + "name": "device_make", + "expr": "$.devicedata.devicespec.make" + }, + { + "type": "path", + "name": "device_id", + "expr": "$.devicedata.devicespec.id" + }, + { + "type": "path", + "name": "device_mem", + "expr": "$.devicedata.devicespec.mem" + }, + { + "type": "path", + "name": "device_idisk", + "expr": "$.devicedata.devicespec.idisk" + }, + { + "type": "path", + "name": "device_edisk", + "expr": "$.devicedata.devicespec.edisk" + }, + { + "type": "path", + "name": "device_scrn", + "expr": "$.devicedata.devicespec.scrn" + }, + { + "type": "path", + "name": "device_camera", + "expr": "$.devicedata.devicespec.camera" + }, + { + "type": "path", + "name": "device_cpu", + "expr": "$.devicedata.devicespec.cpu" + }, + { + "type": "path", + "name": "device_sims", + "expr": "$.devicedata.devicespec.sims" + }, + { + "type": "path", + "name": "device_uaspec_agent", + "expr": "$.devicedata.uaspec.agent" + }, + { + "type": "path", + "name": "device_uaspec_ver", + "expr": "$.devicedata.uaspec.ver" + }, + { + "type": "path", + "name": "device_uaspec_system", + "expr": "$.devicedata.uaspec.system" + }, + { + "type": "path", + "name": "device_uaspec_platform", + "expr": "$.devicedata.uaspec.platform" + }, + { + "type": "path", + "name": "device_uaspec_raw", + "expr": "$.devicedata.uaspec.raw" + }, + { + "type": "path", + "name": "device_first_access", + "expr": "$.devicedata.firstaccess" + }, + { + "type": "path", + "name": "device_loc_state_custom_code", + "expr": "$.devicedata.statecustomcode" + }, + { + "type": "path", + "name": "device_loc_state_custom_name", + "expr": "$.devicedata.statecustomname" + }, + { + "type": "path", + "name": "device_loc_district", + "expr": "$.devicedata.districtcustom" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_object_type", + "expr": "$.contentdata.objecttype" + }, + { + "type": "path", + "name": "content_type", + "expr": "$.contentdata.contenttype" + }, + { + "type": "path", + "name": "content_media_type", + "expr": "$.contentdata.mediatype" + }, + { + "type": "path", + "name": "content_language", + "expr": "$.contentdata.language[*]" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subjects", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "content_framework", + "expr": "$.contentdata.framework" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_status", + "expr": "$.contentdata.status" + }, + { + "type": "path", + "name": "content_version", + "expr": "$.contentdata.pkgversion" + }, + { + "type": "path", + "name": "content_last_submitted_on", + "expr": "$.contentdata.lastsubmittedon" + }, + { + "type": "path", + "name": "content_last_published_on", + "expr": "$.contentdata.lastpublishedon" + }, + { + "type": "path", + "name": "content_last_updated_on", + "expr": "$.contentdata.lastupdatedon" + }, + { + "type": "path", + "name": "user_grade_list", + "expr": "$.userdata.gradelist[*]" + }, + { + "type": "path", + "name": "user_language_list", + "expr": "$.userdata.languagelist[*]" + }, + { + "type": "path", + "name": "user_subject_list", + "expr": "$.userdata.subjectlist[*]" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_roles", + "expr": "$.userdata.roles[*]" + }, + { + "type": "path", + "name": "user_loc_state", + "expr": "$.userdata.state" + }, + { + "type": "path", + "name": "user_loc_district", + "expr": "$.userdata.district" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "device_loc_iso_state_code", + "expr": "$.devicedata.iso3166statecode" + }, + { + "type": "path", + "name": "object_l2_created_for", + "expr": "$.l2data.createdfor" + }, + { + "type": "path", + "name": "object_l2_framework", + "expr": "$.l2data.framework" + }, + { + "type": "path", + "name": "object_l2_name", + "expr": "$.l2data.name" + }, + { + "type": "path", + "name": "object_l2_channel", + "expr": "$.l2data.channel" + }, + { + "type": "path", + "name": "object_l2_mimetype", + "expr": "$.l2data.mimetype" + }, + { + "type": "path", + "name": "object_l2_medium", + "expr": "$.l2data.medium[*]" + }, + { + "type": "path", + "name": "object_l2_board", + "expr": "$.l2data.board" + }, + { + "type": "path", + "name": "object_l2_contenttype", + "expr": "$.l2data.contenttype" + }, + { + "type": "path", + "name": "object_l2_gradelevel", + "expr": "$.l2data.gradelevel[*]" + }, + { + "type": "path", + "name": "object_l2_subjects", + "expr": "$.l2data.subject[*]" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "mid" + }, + { + "type": "long", + "name": "ets" + }, + { + "type": "string", + "name": "@timestamp" + }, + { + "type": "string", + "name": "actor_id" + }, + { + "type": "string", + "name": "actor_type" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "context_sid" + }, + { + "type": "string", + "name": "context_did" + }, + { + "name": "context_cdata_type" + }, + { + "name": "context_cdata_id" + }, + { + "type": "string", + "name": "context_rollup_l1" + }, + { + "type": "string", + "name": "context_rollup_l2" + }, + { + "type": "string", + "name": "context_rollup_l3" + }, + { + "type": "string", + "name": "context_rollup_l4" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_version" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "object_rollup_l2" + }, + { + "type": "string", + "name": "object_rollup_l3" + }, + { + "type": "string", + "name": "object_rollup_l4" + }, + { + "name": "tags" + }, + { + "type": "long", + "name": "edata_rating" + }, + { + "type": "string", + "name": "edata_comments" + }, + { + "type": "string", + "name": "edata_commentid" + }, + { + "type": "string", + "name": "edata_commenttxt" + }, + { + "type": "string", + "name": "device_loc_state" + }, + { + "type": "string", + "name": "device_loc_state_code" + }, + { + "type": "string", + "name": "device_loc_city" + }, + { + "type": "string", + "name": "device_loc_country_code" + }, + { + "type": "string", + "name": "device_loc_country" + }, + { + "type": "string", + "name": "device_os" + }, + { + "type": "string", + "name": "device_make" + }, + { + "type": "string", + "name": "device_id" + }, + { + "type": "long", + "name": "device_mem" + }, + { + "type": "string", + "name": "device_idisk" + }, + { + "type": "string", + "name": "device_edisk" + }, + { + "type": "string", + "name": "device_scrn" + }, + { + "type": "string", + "name": "device_camera" + }, + { + "type": "string", + "name": "device_cpu" + }, + { + "type": "long", + "name": "device_sims" + }, + { + "type": "string", + "name": "device_uaspec_agent" + }, + { + "type": "string", + "name": "device_uaspec_ver" + }, + { + "type": "string", + "name": "device_uaspec_system" + }, + { + "type": "string", + "name": "device_uaspec_platform" + }, + { + "type": "string", + "name": "device_uaspec_raw" + }, + { + "type": "long", + "name": "device_first_access" + }, + { + "type": "string", + "name": "device_loc_state_custom_code" + }, + { + "type": "string", + "name": "device_loc_state_custom_name" + }, + { + "type": "string", + "name": "device_loc_district" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_object_type" + }, + { + "type": "string", + "name": "content_type" + }, + { + "type": "string", + "name": "content_media_type" + }, + { + "name": "content_language" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subjects" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "content_framework" + }, + { + "type": "string", + "name": "content_board" + }, + { + "type": "string", + "name": "content_status" + }, + { + "type": "double", + "name": "content_version" + }, + { + "type": "long", + "name": "content_last_submitted_on" + }, + { + "type": "long", + "name": "content_last_published_on" + }, + { + "type": "long", + "name": "content_last_updated_on" + }, + { + "name": "user_grade_list" + }, + { + "name": "user_language_list" + }, + { + "name": "user_subject_list" + }, + { + "type": "string", + "name": "user_type" + }, + { + "name": "user_roles" + }, + { + "type": "string", + "name": "user_loc_state" + }, + { + "type": "string", + "name": "user_loc_district" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "device_loc_iso_state_code" + }, + { + "name": "object_l2_created_for" + }, + { + "type": "string", + "name": "object_l2_framework" + }, + { + "type": "string", + "name": "object_l2_name" + }, + { + "type": "string", + "name": "object_l2_channel" + }, + { + "type": "string", + "name": "object_l2_mimetype" + }, + { + "name": "object_l2_medium" + }, + { + "type": "string", + "name": "object_l2_board" + }, + { + "type": "string", + "name": "object_l2_contenttype" + }, + { + "name": "object_l2_gradelevel" + }, + { + "name": "object_l2_subjects" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [ + { + "type": "count", + "name": "count" + }, + { + "type": "longSum", + "name": "total_rating", + "fieldName": "edata_rating" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "rollup": true + }, + "transformSpec": { + "filter": { + "type": "selector", + "dimension": "eid", + "value": "FEEDBACK" + } + } + }, + "ioConfig": { + "topic": "{{env}}.druid.events.telemetry", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{raw_telemetry_feedback_events_taskcount}}, + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 5000000 + } +} diff --git a/ansible/roles/druid-ingestion/templates/rollup_course_completion_audit_syncts b/ansible/roles/druid-ingestion/templates/rollup_course_completion_audit_syncts new file mode 100644 index 0000000000..8246184b9d --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/rollup_course_completion_audit_syncts @@ -0,0 +1,605 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "course-completion-events-rollup", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "actor_type", + "expr": "$.actor.type" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_version", + "expr": "$.object.ver" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "object_rollup_l2", + "expr": "$.object.rollup.l2" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "edata_state", + "expr": "$.edata.state" + }, + { + "type": "path", + "name": "edata_prevstate", + "expr": "$.edata.prevstate" + }, + { + "type": "path", + "name": "edata_duration", + "expr": "$.edata.duration" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_object_type", + "expr": "$.contentdata.objecttype" + }, + { + "type": "path", + "name": "content_type", + "expr": "$.contentdata.contenttype" + }, + { + "type": "path", + "name": "content_language", + "expr": "$.contentdata.language[*]" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subjects", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "content_framework", + "expr": "$.contentdata.framework" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_status", + "expr": "$.contentdata.status" + }, + { + "type": "path", + "name": "content_created_by", + "expr": "$.contentdata.createdby" + }, + { + "type": "path", + "name": "content_created_for", + "expr": "$.contentdata.createdfor" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_object_type", + "expr": "$.collectiondata.objecttype" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_media_type", + "expr": "$.collectiondata.mediatype" + }, + { + "type": "path", + "name": "collection_language", + "expr": "$.collectiondata.language[*]" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "collection_subjects", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_mimetype", + "expr": "$.collectiondata.mimetype" + }, + { + "type": "path", + "name": "collection_framework", + "expr": "$.collectiondata.framework" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_status", + "expr": "$.collectiondata.status" + }, + { + "type": "path", + "name": "collection_version", + "expr": "$.collectiondata.pkgversion" + }, + { + "type": "path", + "name": "collection_created_by", + "expr": "$.collectiondata.createdby" + }, + { + "type": "path", + "name": "collection_created_for", + "expr": "$.collectiondata.createdfor" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "object_l2_created_for", + "expr": "$.l2data.createdfor" + }, + { + "type": "path", + "name": "object_l2_framework", + "expr": "$.l2data.framework" + }, + { + "type": "path", + "name": "object_l2_name", + "expr": "$.l2data.name" + }, + { + "type": "path", + "name": "object_l2_channel", + "expr": "$.l2data.channel" + }, + { + "type": "path", + "name": "object_l2_mimetype", + "expr": "$.l2data.mimetype" + }, + { + "type": "path", + "name": "object_l2_medium", + "expr": "$.l2data.medium[*]" + }, + { + "type": "path", + "name": "object_l2_board", + "expr": "$.l2data.board" + }, + { + "type": "path", + "name": "object_l2_contenttype", + "expr": "$.l2data.contenttype" + }, + { + "type": "path", + "name": "object_l2_gradelevel", + "expr": "$.l2data.gradelevel[*]" + }, + { + "type": "path", + "name": "object_l2_subjects", + "expr": "$.l2data.subject[*]" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "actor_id" + }, + { + "type": "string", + "name": "actor_type" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_version" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "object_rollup_l2" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "type": "string", + "name": "edata_state" + }, + { + "type": "string", + "name": "edata_prevstate" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_object_type" + }, + { + "type": "string", + "name": "content_type" + }, + { + "name": "content_language" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subjects" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "content_framework" + }, + { + "type": "string", + "name": "content_board" + }, + { + "type": "string", + "name": "content_status" + }, + { + "type": "string", + "name": "content_created_by" + }, + { + "name": "content_created_for" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_object_type" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "name": "collection_language" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_subjects" + }, + { + "type": "string", + "name": "collection_mimetype" + }, + { + "type": "string", + "name": "collection_framework" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "type": "string", + "name": "collection_status" + }, + { + "type": "double", + "name": "collection_version" + }, + { + "type": "string", + "name": "collection_created_by" + }, + { + "name": "collection_created_for" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "user_declared_state" + }, + { + "type": "string", + "name": "device_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "name": "object_l2_created_for" + }, + { + "type": "string", + "name": "object_l2_framework" + }, + { + "type": "string", + "name": "object_l2_name" + }, + { + "type": "string", + "name": "object_l2_channel" + }, + { + "type": "string", + "name": "object_l2_mimetype" + }, + { + "name": "object_l2_medium" + }, + { + "type": "string", + "name": "object_l2_board" + }, + { + "type": "string", + "name": "object_l2_contenttype" + }, + { + "name": "object_l2_gradelevel" + }, + { + "name": "object_l2_subjects" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "transformSpec": { + "filter": { + "type": "and", + "fields": [ + { + "type": "selector", + "dimension": "eid", + "value": "AUDIT" + }, + { + "type": "selector", + "dimension": "edata_type", + "value": "enrol-complete" + } + ] + } + }, + "metricsSpec": [ + { + "name": "total_count", + "type": "count" + }, + { + "type": "doubleSum", + "name": "total_edata_duration", + "fieldName": "edata_duration" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "rollup": true + } + }, + "ioConfig": { + "topic": "{{env}}.druid.events.telemetry", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{rollup_course_completion_audit_syncts_taskcount}}, + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": false, + "completionTimeout": "PT1800S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 5000000 + } +} diff --git a/ansible/roles/druid-ingestion/templates/rollup_error_hourly_syncts b/ansible/roles/druid-ingestion/templates/rollup_error_hourly_syncts new file mode 100644 index 0000000000..738bb5dd90 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/rollup_error_hourly_syncts @@ -0,0 +1,315 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "error-hourly-rollup-syncts", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subject", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_subject", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "dialcode_channel", + "expr": "$.dialcodedata.channel" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "edata_err", + "expr": "$.edata.err" + }, + { + "type": "path", + "name": "edata_errtype", + "expr": "$.edata.errtype" + }, + { + "type": "path", + "name": "edata_plugin_id", + "expr": "$.edata.plugin.id" + }, + { + "type": "path", + "name": "edata_plugin_ver", + "expr": "$.edata.plugin.ver" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_board" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subject" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_subject" + }, + { + "type": "string", + "name": "dialcode_channel" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "edata_err" + }, + { + "type": "string", + "name": "edata_errtype" + }, + { + "type": "string", + "name": "edata_plugin_id" + }, + { + "type": "string", + "name": "edata_plugin_ver" + } + ] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [ + { + "type": "count", + "name": "total_count" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "HOUR", + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000 + }, + "ioConfig": { + "topic": "{{env}}.druid.events.error", + "replicas": 1, + "taskCount": {{rollup_error_hourly_syncts_taskcount}}, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "useEarliestOffset": true, + "completionTimeout": "PT1800S" + } +} diff --git a/ansible/roles/druid-ingestion/templates/rollup_summary_distinct_counts b/ansible/roles/druid-ingestion/templates/rollup_summary_distinct_counts new file mode 100644 index 0000000000..108c77af35 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/rollup_summary_distinct_counts @@ -0,0 +1,375 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "summary-distinct-counts", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "syncts" + }, + { + "type": "root", + "name": "uid" + }, + { + "type": "path", + "name": "dimensions_pdata_id", + "expr": "$.dimensions.pdata.id" + }, + { + "type": "path", + "name": "dimensions_pdata_pid", + "expr": "$.dimensions.pdata.pid" + }, + { + "type": "path", + "name": "dimensions_type", + "expr": "$.dimensions.type" + }, + { + "type": "path", + "name": "dimensions_pdata_ver", + "expr": "$.dimensions.pdata.ver" + }, + { + "type": "path", + "name": "dimensions_mode", + "expr": "$.dimensions.mode" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_channel", + "expr": "$.contentdata.channel" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subject", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_created_for", + "expr": "$.contentdata.createdfor" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "dialcode_channel", + "expr": "$.dialcodedata.channel" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_channel", + "expr": "$.collectiondata.channel" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_subject", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "collection_created_for", + "expr": "$.collectiondata.createdfor" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "user_loc_block", + "expr": "$.userdata.block" + }, + { + "type": "path", + "name": "user_loc_cluster", + "expr": "$.userdata.cluster" + }, + { + "type": "path", + "name": "user_school_name", + "expr": "$.userdata.schoolname" + }, + { + "type": "path", + "name": "dimensions_did", + "expr": "$.dimensions.did" + }, + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "dimensions_pdata_id" + }, + { + "type": "string", + "name": "dimensions_pdata_pid" + }, + { + "type": "string", + "name": "dimensions_type" + }, + { + "type": "string", + "name": "dimensions_pdata_ver" + }, + { + "type": "string", + "name": "dimensions_mode" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_board" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_channel" + }, + { + "name": "content_subject" + }, + { + "name": "content_created_for" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "dialcode_channel" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "user_loc_block" + }, + { + "type": "string", + "name": "user_loc_cluster" + }, + { + "type": "string", + "name": "user_school_name" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "name": "collection_channel" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_subject" + }, + { + "type": "string", + "name": "collection_created_for" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [ + { + "type": "HLLSketchBuild", + "name": "unique_devices", + "fieldName": "dimensions_did" + }, + { + "type": "HLLSketchBuild", + "name": "unique_users", + "fieldName": "uid" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "day", + "queryGranularity": "day", + "rollup": false + } + }, + "ioConfig": { + "topic": "{{env}}.druid.events.summary", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{rollup_summary_distinct_counts_taskcount}}, + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 5000000 + } +} diff --git a/ansible/roles/druid-ingestion/templates/rollup_summary_syncts b/ansible/roles/druid-ingestion/templates/rollup_summary_syncts new file mode 100644 index 0000000000..0c54f6684d --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/rollup_summary_syncts @@ -0,0 +1,384 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "summary-rollup-syncts", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "syncts" + }, + { + "type": "path", + "name": "dimensions_pdata_id", + "expr": "$.dimensions.pdata.id" + }, + { + "type": "path", + "name": "dimensions_pdata_pid", + "expr": "$.dimensions.pdata.pid" + }, + { + "type": "path", + "name": "dimensions_type", + "expr": "$.dimensions.type" + }, + { + "type": "path", + "name": "dimensions_pdata_ver", + "expr": "$.dimensions.pdata.ver" + }, + { + "type": "path", + "name": "dimensions_mode", + "expr": "$.dimensions.mode" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_channel", + "expr": "$.contentdata.channel" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subject", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_created_for", + "expr": "$.contentdata.createdfor" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "dialcode_channel", + "expr": "$.dialcodedata.channel" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_channel", + "expr": "$.collectiondata.channel" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_subject", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "collection_keywords", + "expr": "$.collectiondata.keywords[*]" + }, + { + "type": "path", + "name": "collection_created_for", + "expr": "$.collectiondata.createdfor" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "user_loc_block", + "expr": "$.userdata.block" + }, + { + "type": "path", + "name": "user_loc_cluster", + "expr": "$.userdata.cluster" + }, + { + "type": "path", + "name": "user_school_name", + "expr": "$.userdata.schoolname" + }, + { + "type": "path", + "name": "edata_interaction_count", + "expr": "$.edata.eks.interact_events_count" + }, + { + "type": "path", + "name": "edata_time_spent", + "expr": "$.edata.eks.time_spent" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "dimensions_pdata_id" + }, + { + "type": "string", + "name": "dimensions_pdata_pid" + }, + { + "type": "string", + "name": "dimensions_type" + }, + { + "type": "string", + "name": "dimensions_pdata_ver" + }, + { + "type": "string", + "name": "dimensions_mode" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_board" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "name": "content_channel" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subject" + }, + { + "name": "content_created_for" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "dialcode_channel" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "user_loc_block" + }, + { + "type": "string", + "name": "user_loc_cluster" + }, + { + "type": "string", + "name": "user_school_name" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "name": "collection_channel" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_keywords" + }, + { + "name": "collection_subject" + }, + { + "name": "collection_created_for" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [ + { + "type": "count", + "name": "total_count" + }, + { + "type": "longSum", + "name": "total_interactions", + "fieldName": "edata_interaction_count", + "expression": null + }, + { + "type": "doubleSum", + "name": "total_time_spent", + "fieldName": "edata_time_spent", + "expression": null + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "rollup": true, + "intervals": null + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 5000000 + }, + "ioConfig": { + "topic": "{{env}}.druid.events.summary", + "replicas": 1, + "taskCount": {{rollup_summary_syncts_taskcount}}, + "taskDuration": "PT14400S", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "useEarliestOffset": false, + "completionTimeout": "PT1800S" + } +} diff --git a/ansible/roles/druid-ingestion/templates/rollup_telemetry_audit_syncts b/ansible/roles/druid-ingestion/templates/rollup_telemetry_audit_syncts new file mode 100644 index 0000000000..5ee34cfcda --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/rollup_telemetry_audit_syncts @@ -0,0 +1,674 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "audit-rollup-syncts", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "actor_type", + "expr": "$.actor.type" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "context_cdata_type", + "expr": "$.context.cdata[*].type" + }, + { + "type": "path", + "name": "context_cdata_id", + "expr": "$.context.cdata[*].id" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_version", + "expr": "$.object.ver" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "object_rollup_l2", + "expr": "$.object.rollup.l2" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "edata_state", + "expr": "$.edata.state" + }, + { + "type": "path", + "name": "edata_prevstate", + "expr": "$.edata.prevstate" + }, + { + "type": "path", + "name": "edata_duration", + "expr": "$.edata.duration" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_object_type", + "expr": "$.contentdata.objecttype" + }, + { + "type": "path", + "name": "content_type", + "expr": "$.contentdata.contenttype" + }, + { + "type": "path", + "name": "content_language", + "expr": "$.contentdata.language[*]" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subjects", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "content_framework", + "expr": "$.contentdata.framework" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_status", + "expr": "$.contentdata.status" + }, + { + "type": "path", + "name": "content_last_submitted_on", + "expr": "$.contentdata.lastsubmittedon" + }, + { + "type": "path", + "name": "content_last_published_on", + "expr": "$.contentdata.lastpublishedon" + }, + { + "type": "path", + "name": "content_last_updated_on", + "expr": "$.contentdata.lastupdatedon" + }, + { + "type": "path", + "name": "content_created_by", + "expr": "$.contentdata.createdby" + }, + { + "type": "path", + "name": "content_created_for", + "expr": "$.contentdata.createdfor" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_object_type", + "expr": "$.collectiondata.objecttype" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_media_type", + "expr": "$.collectiondata.mediatype" + }, + { + "type": "path", + "name": "collection_language", + "expr": "$.collectiondata.language[*]" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "collection_subjects", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_mimetype", + "expr": "$.collectiondata.mimetype" + }, + { + "type": "path", + "name": "collection_framework", + "expr": "$.collectiondata.framework" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_status", + "expr": "$.collectiondata.status" + }, + { + "type": "path", + "name": "collection_version", + "expr": "$.collectiondata.pkgversion" + }, + { + "type": "path", + "name": "collection_last_submitted_on", + "expr": "$.collectiondata.lastsubmittedon" + }, + { + "type": "path", + "name": "collection_last_published_on", + "expr": "$.collectiondata.lastpublishedon" + }, + { + "type": "path", + "name": "collection_last_updated_on", + "expr": "$.collectiondata.lastupdatedon" + }, + { + "type": "path", + "name": "collection_created_by", + "expr": "$.collectiondata.createdby" + }, + { + "type": "path", + "name": "collection_created_for", + "expr": "$.collectiondata.createdfor" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "device_first_access", + "expr": "$.devicedata.firstaccess" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "object_l2_created_for", + "expr": "$.l2data.createdfor" + }, + { + "type": "path", + "name": "object_l2_framework", + "expr": "$.l2data.framework" + }, + { + "type": "path", + "name": "object_l2_name", + "expr": "$.l2data.name" + }, + { + "type": "path", + "name": "object_l2_channel", + "expr": "$.l2data.channel" + }, + { + "type": "path", + "name": "object_l2_mimetype", + "expr": "$.l2data.mimetype" + }, + { + "type": "path", + "name": "object_l2_medium", + "expr": "$.l2data.medium[*]" + }, + { + "type": "path", + "name": "object_l2_board", + "expr": "$.l2data.board" + }, + { + "type": "path", + "name": "object_l2_contenttype", + "expr": "$.l2data.contenttype" + }, + { + "type": "path", + "name": "object_l2_gradelevel", + "expr": "$.l2data.gradelevel[*]" + }, + { + "type": "path", + "name": "object_l2_subjects", + "expr": "$.l2data.subject[*]" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "actor_id" + }, + { + "type": "string", + "name": "actor_type" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "name": "context_cdata_type" + }, + { + "name": "context_cdata_id" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_version" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "object_rollup_l2" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "type": "string", + "name": "edata_state" + }, + { + "type": "string", + "name": "edata_prevstate" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_object_type" + }, + { + "type": "string", + "name": "content_type" + }, + { + "name": "content_language" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subjects" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "content_framework" + }, + { + "type": "string", + "name": "content_board" + }, + { + "type": "string", + "name": "content_status" + }, + { + "type": "long", + "name": "content_last_submitted_on" + }, + { + "type": "long", + "name": "content_last_published_on" + }, + { + "type": "long", + "name": "content_last_updated_on" + }, + { + "type": "string", + "name": "content_created_by" + }, + { + "name": "content_created_for" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_object_type" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "name": "collection_language" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_subjects" + }, + { + "type": "string", + "name": "collection_mimetype" + }, + { + "type": "string", + "name": "collection_framework" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "type": "string", + "name": "collection_status" + }, + { + "type": "double", + "name": "collection_version" + }, + { + "type": "long", + "name": "collection_last_submitted_on" + }, + { + "type": "long", + "name": "collection_last_published_on" + }, + { + "type": "long", + "name": "collection_last_updated_on" + }, + { + "type": "string", + "name": "collection_created_by" + }, + { + "name": "collection_created_for" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "long", + "name": "device_first_access" + }, + { + "type": "string", + "name": "user_declared_state" + }, + { + "type": "string", + "name": "device_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "name": "object_l2_created_for" + }, + { + "type": "string", + "name": "object_l2_framework" + }, + { + "type": "string", + "name": "object_l2_name" + }, + { + "type": "string", + "name": "object_l2_channel" + }, + { + "type": "string", + "name": "object_l2_mimetype" + }, + { + "name": "object_l2_medium" + }, + { + "type": "string", + "name": "object_l2_board" + }, + { + "type": "string", + "name": "object_l2_contenttype" + }, + { + "name": "object_l2_gradelevel" + }, + { + "name": "object_l2_subjects" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "transformSpec": { + "filter": { + "type": "selector", + "dimension": "eid", + "value": "AUDIT" + } + }, + "metricsSpec": [ + { + "name": "total_count", + "type": "count" + }, + { + "type": "doubleSum", + "name": "total_edata_duration", + "fieldName": "edata_duration" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "rollup": true + } + }, + "ioConfig": { + "topic": "{{env}}.druid.events.telemetry", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{rollup_telemetry_audit_syncts_taskcount}}, + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": false, + "completionTimeout": "PT1800S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 5000000 + } +} diff --git a/ansible/roles/druid-ingestion/templates/rollup_telemetry_hourly_syncts b/ansible/roles/druid-ingestion/templates/rollup_telemetry_hourly_syncts new file mode 100644 index 0000000000..0a515b106c --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/rollup_telemetry_hourly_syncts @@ -0,0 +1,420 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "telemetry-hourly-rollup-syncts", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subject", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_subject", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "dialcode_channel", + "expr": "$.dialcodedata.channel" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "edata_item_id", + "expr": "$.edata.item.id" + }, + { + "type": "path", + "name": "edata_item_title", + "expr": "$.edata.item.title" + }, + { + "type": "path", + "name": "edata_state", + "expr": "$.edata.state" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "edata_mode", + "expr": "$.edata.mode" + }, + { + "type": "path", + "name": "edata_size", + "expr": "$.edata.size" + }, + { + "type": "path", + "name": "edata_duration", + "expr": "$.edata.duration" + }, + { + "type": "path", + "name": "edata_score", + "expr": "$.edata.score" + }, + { + "type": "path", + "name": "edata_item_maxscore", + "expr": "$.edata.item.maxscore" + }, + { + "type": "path", + "name": "edata_rating", + "expr": "$.edata.rating" + }, + { + "type": "path", + "name": "edata_timespent", + "expr": "$.edata.timespent" + }, + { + "type": "path", + "name": "edata_pageviews", + "expr": "$.edata.pageviews" + }, + { + "type": "path", + "name": "edata_interactions", + "expr": "$.edata.interactions" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_board" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subject" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_subject" + }, + { + "type": "string", + "name": "dialcode_channel" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "type": "string", + "name": "edata_mode" + }, + { + "type": "string", + "name": "edata_item_id" + }, + { + "type": "string", + "name": "edata_item_title" + }, + { + "type": "string", + "name": "edata_state" + }, + { + "type": "long", + "name": "edata_size" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [ + { + "type": "count", + "name": "total_count" + }, + { + "type": "doubleSum", + "name": "total_edata_duration", + "fieldName": "edata_duration" + }, + { + "type": "doubleSum", + "name": "total_edata_rating", + "fieldName": "edata_rating" + }, + { + "type": "doubleSum", + "name": "total_edata_score", + "fieldName": "edata_score" + }, + { + "type": "doubleSum", + "name": "total_max_score", + "fieldName": "edata_item_maxscore" + }, + { + "type": "doubleSum", + "name": "total_edata_timespent", + "fieldName": "edata_timespent" + }, + { + "type": "longSum", + "name": "total_edata_pageviews", + "fieldName": "edata_pageviews" + }, + { + "type": "longSum", + "name": "total_edata_interactions", + "fieldName": "edata_interactions" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "HOUR", + "rollup": true + }, + "transformSpec": { + "transforms": [ + { + "type": "expression", + "name": "edata_size", + "expression": "if((edata_size>0),1,0)" + } + ] + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000 + }, + "ioConfig": { + "topic": "{{env}}.druid.events.telemetry", + "replicas": 1, + "taskCount": {{rollup_telemetry_hourly_syncts_taskcount}}, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "completionTimeout": "PT1800S" + } +} diff --git a/ansible/roles/druid-ingestion/templates/rollup_telemetry_syncts b/ansible/roles/druid-ingestion/templates/rollup_telemetry_syncts new file mode 100644 index 0000000000..3fc476ff61 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/rollup_telemetry_syncts @@ -0,0 +1,414 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "telemetry-rollup-syncts", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "path", + "name": "context_channel", + "expr": "$.context.channel" + }, + { + "type": "path", + "name": "context_pdata_id", + "expr": "$.context.pdata.id" + }, + { + "type": "path", + "name": "context_pdata_pid", + "expr": "$.context.pdata.pid" + }, + { + "type": "path", + "name": "context_pdata_ver", + "expr": "$.context.pdata.ver" + }, + { + "type": "path", + "name": "object_id", + "expr": "$.object.id" + }, + { + "type": "path", + "name": "object_type", + "expr": "$.object.type" + }, + { + "type": "path", + "name": "object_rollup_l1", + "expr": "$.object.rollup.l1" + }, + { + "type": "path", + "name": "content_name", + "expr": "$.contentdata.name" + }, + { + "type": "path", + "name": "content_board", + "expr": "$.contentdata.board" + }, + { + "type": "path", + "name": "content_medium", + "expr": "$.contentdata.medium[*]" + }, + { + "type": "path", + "name": "content_gradelevel", + "expr": "$.contentdata.gradelevel[*]" + }, + { + "type": "path", + "name": "content_subject", + "expr": "$.contentdata.subject[*]" + }, + { + "type": "path", + "name": "content_mimetype", + "expr": "$.contentdata.mimetype" + }, + { + "type": "path", + "name": "derived_loc_state", + "expr": "$.derivedlocationdata.state" + }, + { + "type": "path", + "name": "derived_loc_district", + "expr": "$.derivedlocationdata.district" + }, + { + "type": "path", + "name": "derived_loc_from", + "expr": "$.derivedlocationdata.from" + }, + { + "type": "path", + "name": "collection_name", + "expr": "$.collectiondata.name" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "collection_board", + "expr": "$.collectiondata.board" + }, + { + "type": "path", + "name": "collection_medium", + "expr": "$.collectiondata.medium[*]" + }, + { + "type": "path", + "name": "collection_subject", + "expr": "$.collectiondata.subject[*]" + }, + { + "type": "path", + "name": "collection_gradelevel", + "expr": "$.collectiondata.gradelevel[*]" + }, + { + "type": "path", + "name": "dialcode_channel", + "expr": "$.dialcodedata.channel" + }, + { + "type": "path", + "name": "user_type", + "expr": "$.userdata.usertype" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "user_login_type", + "expr": "$.userdata.userlogintype" + }, + { + "type": "path", + "name": "edata_item_id", + "expr": "$.edata.item.id" + }, + { + "type": "path", + "name": "edata_item_title", + "expr": "$.edata.item.title" + }, + { + "type": "path", + "name": "edata_state", + "expr": "$.edata.state" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "edata_size", + "expr": "$.edata.size" + }, + { + "type": "path", + "name": "edata_duration", + "expr": "$.edata.duration" + }, + { + "type": "path", + "name": "edata_score", + "expr": "$.edata.score" + }, + { + "type": "path", + "name": "edata_item_maxscore", + "expr": "$.edata.item.maxscore" + }, + { + "type": "path", + "name": "edata_rating", + "expr": "$.edata.rating" + }, + { + "type": "path", + "name": "edata_timespent", + "expr": "$.edata.timespent" + }, + { + "type": "path", + "name": "edata_pageviews", + "expr": "$.edata.pageviews" + }, + { + "type": "path", + "name": "edata_interactions", + "expr": "$.edata.interactions" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "context_channel" + }, + { + "type": "string", + "name": "context_pdata_id" + }, + { + "type": "string", + "name": "context_pdata_pid" + }, + { + "type": "string", + "name": "context_pdata_ver" + }, + { + "type": "string", + "name": "object_id" + }, + { + "type": "string", + "name": "object_type" + }, + { + "type": "string", + "name": "object_rollup_l1" + }, + { + "type": "string", + "name": "content_name" + }, + { + "type": "string", + "name": "content_board" + }, + { + "name": "content_medium" + }, + { + "name": "content_gradelevel" + }, + { + "name": "content_subject" + }, + { + "type": "string", + "name": "content_mimetype" + }, + { + "type": "string", + "name": "derived_loc_state" + }, + { + "type": "string", + "name": "derived_loc_district" + }, + { + "type": "string", + "name": "derived_loc_from" + }, + { + "type": "string", + "name": "collection_name" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "type": "string", + "name": "collection_board" + }, + { + "name": "collection_medium" + }, + { + "name": "collection_gradelevel" + }, + { + "name": "collection_subject" + }, + { + "type": "string", + "name": "dialcode_channel" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "user_login_type" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "type": "string", + "name": "edata_item_id" + }, + { + "type": "string", + "name": "edata_item_title" + }, + { + "type": "string", + "name": "edata_state" + }, + { + "type": "long", + "name": "edata_size" + } + ], + "dimensionsExclusions": [] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "metricsSpec": [ + { + "type": "count", + "name": "total_count" + }, + { + "type": "doubleSum", + "name": "total_edata_duration", + "fieldName": "edata_duration" + }, + { + "type": "doubleSum", + "name": "total_edata_rating", + "fieldName": "edata_rating" + }, + { + "type": "doubleSum", + "name": "total_edata_score", + "fieldName": "edata_score" + }, + { + "type": "doubleSum", + "name": "total_max_score", + "fieldName": "edata_item_maxscore" + }, + { + "type": "doubleSum", + "name": "total_edata_timespent", + "fieldName": "edata_timespent" + }, + { + "type": "longSum", + "name": "total_edata_pageviews", + "fieldName": "edata_pageviews" + }, + { + "type": "longSum", + "name": "total_edata_interactions", + "fieldName": "edata_interactions" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "rollup": true + }, + "transformSpec": { + "filter": null, + "transforms": [ + { + "type": "expression", + "name": "edata_size", + "expression": "if((edata_size>0),1,0)" + } + ] + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 10000000 + }, + "ioConfig": { + "topic": "{{env}}.druid.events.telemetry", + "replicas": 1, + "taskCount": {{rollup_telemetry_syncts_taskcount}}, + "taskDuration": "PT14400S", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "useEarliestOffset": false, + "completionTimeout": "PT1800S" + } +} diff --git a/ansible/roles/druid-ingestion/templates/rollup_tpd_hourly_syncts b/ansible/roles/druid-ingestion/templates/rollup_tpd_hourly_syncts new file mode 100644 index 0000000000..5878dab401 --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/rollup_tpd_hourly_syncts @@ -0,0 +1,165 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "tpd-hourly-rollup-syncts", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "edata_id", + "expr": "$.edata.id" + }, + { + "type": "path", + "name": "edata_mode", + "expr": "$.edata.mode" + }, + { + "type": "path", + "name": "edata_pageid", + "expr": "$.edata.pageid" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "context_did", + "expr": "$.context.did" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "type": "string", + "name": "edata_id" + }, + { + "type": "string", + "name": "edata_mode" + }, + { + "type": "string", + "name": "edata_pageid" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "type":"string", + "name": "first_time_user" + } + ] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "transformSpec": { + "transforms": [ + { + "type": "expression", + "name": "first_time_user ", + "expression": "if(like(\"context_cdata_type\",'FirstTimeUser') && like(\"context_cdata_id\",'true') , 'true','false')" + }, + { + "type": "expression", + "name": "is_playerstart_event", + "expression": "if((like(\"context_pdata_pid\",'%contentplayer%'), if ( \"eid\" == 'START', 'true','false') ,'true')" + } + ], + "filter": { + "type": "selector", + "dimension": "is_playerstart_event", + "value": "true" + } + }, + "metricsSpec": [ + { + "name": "total_count", + "type": "count" + }, + { + "type": "HLLSketchBuild", + "name": "unique_devices", + "fieldName": "context_did" + }, + { + "type": "HLLSketchBuild", + "name": "unique_users", + "fieldName": "actor_id" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "hour", + "rollup": true + } + }, + "ioConfig": { + "topic": "{{env}}.druid.events.telemetry", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": {{rollup_tpd_hourly_taskcount}}, + "replicas": 1, + "taskDuration": "PT7200S", + "completionTimeout" : "PT3600S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 5000000 + } +} diff --git a/ansible/roles/druid-ingestion/templates/summary_index_kafka b/ansible/roles/druid-ingestion/templates/summary_index_kafka deleted file mode 100644 index 5506678c29..0000000000 --- a/ansible/roles/druid-ingestion/templates/summary_index_kafka +++ /dev/null @@ -1,1124 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "summary-events", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "flattenSpec": { - "useFieldDiscovery": false, - "fields": [ - { - "type": "root", - "name": "eid" - }, - { - "type": "root", - "name": "mid" - }, - { - "type": "root", - "name": "ver" - }, - { - "type": "root", - "name": "ets" - }, - { - "type": "root", - "name": "uid" - }, - { - "type": "root", - "name": "syncts" - }, - { - "type": "path", - "name": "context_date_range_from", - "expr": "$.context.date_range.from" - }, - { - "type": "path", - "name": "context_date_range_to", - "expr": "$.context.date_range.to" - }, - { - "type": "path", - "name": "context_cdata_type", - "expr": "$.context.cdata[*].type" - }, - { - "type": "path", - "name": "context_cdata_id", - "expr": "$.context.cdata[*].id" - }, - { - "type": "path", - "name": "context_rollup_l1", - "expr": "$.context.rollup.l1" - }, - { - "type": "path", - "name": "context_rollup_l2", - "expr": "$.context.rollup.l2" - }, - { - "type": "path", - "name": "context_rollup_l3", - "expr": "$.context.rollup.l3" - }, - { - "type": "path", - "name": "context_rollup_l4", - "expr": "$.context.rollup.l4" - }, - { - "type": "path", - "name": "dimension_channel", - "expr": "$.dimensions.channel" - }, - { - "type": "path", - "name": "dimensions_did", - "expr": "$.dimensions.did" - }, - { - "type": "path", - "name": "dimensions_pdata_id", - "expr": "$.dimensions.pdata.id" - }, - { - "type": "path", - "name": "dimensions_pdata_pid", - "expr": "$.dimensions.pdata.pid" - }, - { - "type": "path", - "name": "dimensions_pdata_ver", - "expr": "$.dimensions.pdata.ver" - }, - { - "type": "path", - "name": "dimensions_sid", - "expr": "$.dimensions.sid" - }, - { - "type": "path", - "name": "dimensions_type", - "expr": "$.dimensions.type" - }, - { - "type": "path", - "name": "dimensions_mode", - "expr": "$.dimensions.mode" - }, - { - "type": "path", - "name": "object_id", - "expr": "$.object.id" - }, - { - "type": "path", - "name": "object_type", - "expr": "$.object.type" - }, - { - "type": "path", - "name": "object_version", - "expr": "$.object.ver" - }, - { - "type": "path", - "name": "object_rollup_l1", - "expr": "$.object.rollup.l1" - }, - { - "type": "path", - "name": "object_rollup_l2", - "expr": "$.object.rollup.l2" - }, - { - "type": "path", - "name": "object_rollup_l3", - "expr": "$.object.rollup.l3" - }, - { - "type": "path", - "name": "object_rollup_l4", - "expr": "$.object.rollup.l4" - }, - { - "type": "root", - "name": "tags" - }, - { - "type": "path", - "name": "edata_time_spent", - "expr": "$.edata.eks.time_spent" - }, - { - "type": "path", - "name": "edata_time_difference", - "expr": "$.edata.eks.time_diff" - }, - { - "type": "path", - "name": "edata_interaction_count", - "expr": "$.edata.eks.interact_events_count" - }, - { - "type": "path", - "name": "edata_env_summary_env", - "expr": "$.edata.eks.env_summary[*].env" - }, - { - "type": "path", - "name": "edata_env_summary_count", - "expr": "$.edata.eks.env_summary[*].count" - }, - { - "type": "path", - "name": "edata_env_summary_time_spent", - "expr": "$.edata.eks.env_summary[*].time_spent" - }, - { - "type": "path", - "name": "edata_page_summary_id", - "expr": "$.edata.eks.page_summary[*].id" - }, - { - "type": "path", - "name": "edata_page_summary_type", - "expr": "$.edata.eks.page_summary[*].type" - }, - { - "type": "path", - "name": "edata_page_summary_env", - "expr": "$.edata.eks.page_summary[*].env" - }, - { - "type": "path", - "name": "edata_page_summary_visit_count", - "expr": "$.edata.eks.page_summary[*].visit_count" - }, - { - "type": "path", - "name": "edata_page_summary_time_spent", - "expr": "$.edata.eks.page_summary[*].time_spent" - }, - { - "type": "path", - "name": "edata_item_responses_item_id", - "expr": "$.edata.eks.item_responses[*].itemId" - }, - { - "type": "path", - "name": "edata_item_responses_time_spent", - "expr": "$.edata.eks.item_responses[*].timeSpent" - }, - { - "type": "path", - "name": "edata_item_responses_pass", - "expr": "$.edata.eks.item_responses[*].pass" - }, - { - "type": "path", - "name": "edata_item_responses_score", - "expr": "$.edata.eks.item_responses[*].score" - }, - { - "type": "path", - "name": "edata_item_responses_max_score", - "expr": "$.edata.eks.item_responses[*].maxScore" - }, - { - "type": "path", - "name": "edata_item_responses_timestamp", - "expr": "$.edata.eks.item_responses[*].time_stamp" - }, - { - "type": "path", - "name": "device_loc_state", - "expr": "$.devicedata.state" - }, - { - "type": "path", - "name": "device_loc_state_code", - "expr": "$.devicedata.statecode" - }, - { - "type": "path", - "name": "device_loc_city", - "expr": "$.devicedata.city" - }, - { - "type": "path", - "name": "device_loc_country_code", - "expr": "$.devicedata.countrycode" - }, - { - "type": "path", - "name": "device_loc_country", - "expr": "$.devicedata.country" - }, - { - "type": "path", - "name": "device_os", - "expr": "$.devicedata.devicespec.os" - }, - { - "type": "path", - "name": "device_make", - "expr": "$.devicedata.devicespec.make" - }, - { - "type": "path", - "name": "device_id", - "expr": "$.devicedata.devicespec.id" - }, - { - "type": "path", - "name": "device_mem", - "expr": "$.devicedata.devicespec.mem" - }, - { - "type": "path", - "name": "device_idisk", - "expr": "$.devicedata.devicespec.idisk" - }, - { - "type": "path", - "name": "device_edisk", - "expr": "$.devicedata.devicespec.edisk" - }, - { - "type": "path", - "name": "device_scrn", - "expr": "$.devicedata.devicespec.scrn" - }, - { - "type": "path", - "name": "device_camera", - "expr": "$.devicedata.devicespec.camera" - }, - { - "type": "path", - "name": "device_cpu", - "expr": "$.devicedata.devicespec.cpu" - }, - { - "type": "path", - "name": "device_sims", - "expr": "$.devicedata.devicespec.sims" - }, - { - "type": "path", - "name": "device_uaspec_agent", - "expr": "$.devicedata.uaspec.agent" - }, - { - "type": "path", - "name": "device_uaspec_ver", - "expr": "$.devicedata.uaspec.ver" - }, - { - "type": "path", - "name": "device_uaspec_system", - "expr": "$.devicedata.uaspec.system" - }, - { - "type": "path", - "name": "device_uaspec_platform", - "expr": "$.devicedata.uaspec.platform" - }, - { - "type": "path", - "name": "device_uaspec_raw", - "expr": "$.devicedata.uaspec.raw" - }, - { - "type": "path", - "name": "device_first_access", - "expr": "$.devicedata.firstaccess" - }, - { - "type": "path", - "name": "device_loc_state_custom_code", - "expr": "$.devicedata.statecustomcode" - }, - { - "type": "path", - "name": "device_loc_state_custom_name", - "expr": "$.devicedata.statecustomname" - }, - { - "type": "path", - "name": "device_loc_district", - "expr": "$.devicedata.districtcustom" - }, - { - "type": "path", - "name": "user_declared_state", - "expr": "$.devicedata.userdeclared.state" - }, - { - "type": "path", - "name": "user_declared_district", - "expr": "$.devicedata.userdeclared.district" - }, - { - "type": "path", - "name": "derived_loc_state", - "expr": "$.derivedlocationdata.state" - }, - { - "type": "path", - "name": "derived_loc_district", - "expr": "$.derivedlocationdata.district" - }, - { - "type": "path", - "name": "derived_loc_from", - "expr": "$.derivedlocationdata.from" - }, - { - "type": "path", - "name": "content_name", - "expr": "$.contentdata.name" - }, - { - "type": "path", - "name": "content_object_type", - "expr": "$.contentdata.objecttype" - }, - { - "type": "path", - "name": "content_type", - "expr": "$.contentdata.contenttype" - }, - { - "type": "path", - "name": "content_media_type", - "expr": "$.contentdata.mediatype" - }, - { - "type": "path", - "name": "content_language", - "expr": "$.contentdata.language[*]" - }, - { - "type": "path", - "name": "content_medium", - "expr": "$.contentdata.medium[*]" - }, - { - "type": "path", - "name": "content_gradelevel", - "expr": "$.contentdata.gradelevel[*]" - }, - { - "type": "path", - "name": "content_subjects", - "expr": "$.contentdata.subject[*]" - }, - { - "type": "path", - "name": "content_mimetype", - "expr": "$.contentdata.mimetype" - }, - { - "type": "path", - "name": "content_framework", - "expr": "$.contentdata.framework" - }, - { - "type": "path", - "name": "content_board", - "expr": "$.contentdata.board" - }, - { - "type": "path", - "name": "content_status", - "expr": "$.contentdata.status" - }, - { - "type": "path", - "name": "content_created_by", - "expr": "$.contentdata.createdby" - }, - { - "type": "path", - "name": "content_created_for", - "expr": "$.contentdata.createdfor" - }, - { - "type": "path", - "name": "content_version", - "expr": "$.contentdata.pkgversion" - }, - { - "type": "path", - "name": "content_last_submitted_on", - "expr": "$.contentdata.lastsubmittedon" - }, - { - "type": "path", - "name": "content_last_published_on", - "expr": "$.contentdata.lastpublishedon" - }, - { - "type": "path", - "name": "content_last_updated_on", - "expr": "$.contentdata.lastupdatedon" - }, - { - "type": "path", - "name": "collection_name", - "expr": "$.collectiondata.name" - }, - { - "type": "path", - "name": "collection_object_type", - "expr": "$.collectiondata.objecttype" - }, - { - "type": "path", - "name": "collection_type", - "expr": "$.collectiondata.contenttype" - }, - { - "type": "path", - "name": "collection_media_type", - "expr": "$.collectiondata.mediatype" - }, - { - "type": "path", - "name": "collection_language", - "expr": "$.collectiondata.language[*]" - }, - { - "type": "path", - "name": "collection_medium", - "expr": "$.collectiondata.medium[*]" - }, - { - "type": "path", - "name": "collection_gradelevel", - "expr": "$.collectiondata.gradelevel[*]" - }, - { - "type": "path", - "name": "collection_subjects", - "expr": "$.collectiondata.subject[*]" - }, - { - "type": "path", - "name": "collection_mimetype", - "expr": "$.collectiondata.mimetype" - }, - { - "type": "path", - "name": "collection_framework", - "expr": "$.collectiondata.framework" - }, - { - "type": "path", - "name": "collection_board", - "expr": "$.collectiondata.board" - }, - { - "type": "path", - "name": "collection_status", - "expr": "$.collectiondata.status" - }, - { - "type": "path", - "name": "collection_version", - "expr": "$.collectiondata.pkgversion" - }, - { - "type": "path", - "name": "collection_last_submitted_on", - "expr": "$.collectiondata.lastsubmittedon" - }, - { - "type": "path", - "name": "collection_last_published_on", - "expr": "$.collectiondata.lastpublishedon" - }, - { - "type": "path", - "name": "collection_last_updated_on", - "expr": "$.collectiondata.lastupdatedon" - }, - { - "type": "path", - "name": "collection_created_by", - "expr": "$.collectiondata.createdby" - }, - { - "type": "path", - "name": "collection_created_for", - "expr": "$.collectiondata.createdfor" - }, - { - "type": "path", - "name": "user_grade_list", - "expr": "$.userdata.gradelist[*]" - }, - { - "type": "path", - "name": "user_language_list", - "expr": "$.userdata.languagelist[*]" - }, - { - "type": "path", - "name": "user_subject_list", - "expr": "$.userdata.subjectlist[*]" - }, - { - "type": "path", - "name": "user_type", - "expr": "$.userdata.usertype" - }, - { - "type": "path", - "name": "user_roles", - "expr": "$.userdata.roles[*]" - }, - { - "type": "path", - "name": "user_loc_state", - "expr": "$.userdata.state" - }, - { - "type": "path", - "name": "user_loc_district", - "expr": "$.userdata.district" - }, - { - "type": "path", - "name": "user_signin_type", - "expr": "$.userdata.usersignintype" - }, - { - "type": "path", - "name": "user_login_type", - "expr": "$.userdata.userlogintype" - }, - { - "type": "path", - "name": "device_loc_iso_state_code", - "expr": "$.devicedata.iso3166statecode" - } - ] - }, - "dimensionsSpec": { - "dimensions": [ - { - "type": "string", - "name": "eid" - }, - { - "type": "string", - "name": "mid" - }, - { - "type": "string", - "name": "ver" - }, - { - "type": "long", - "name": "syncts" - }, - { - "type": "string", - "name": "uid" - }, - { - "type": "long", - "name": "context_date_range_from" - }, - { - "name": "context_cdata_type" - }, - { - "name": "context_cdata_id" - }, - { - "type": "string", - "name": "context_rollup_l1" - }, - { - "type": "string", - "name": "context_rollup_l2" - }, - { - "type": "string", - "name": "context_rollup_l3" - }, - { - "type": "string", - "name": "context_rollup_l4" - }, - { - "type": "string", - "name": "dimension_channel" - }, - { - "type": "string", - "name": "dimensions_did" - }, - { - "type": "string", - "name": "dimensions_pdata_id" - }, - { - "type": "string", - "name": "dimensions_pdata_pid" - }, - { - "type": "string", - "name": "dimensions_pdata_ver" - }, - { - "type": "string", - "name": "dimensions_sid" - }, - { - "type": "string", - "name": "dimensions_type" - }, - { - "type": "string", - "name": "dimensions_mode" - }, - { - "type": "string", - "name": "object_id" - }, - { - "type": "string", - "name": "object_type" - }, - { - "type": "string", - "name": "object_version" - }, - { - "type": "string", - "name": "object_rollup_l1" - }, - { - "type": "string", - "name": "object_rollup_l2" - }, - { - "type": "string", - "name": "object_rollup_l3" - }, - { - "type": "string", - "name": "object_rollup_l4" - }, - { - "name": "tags" - }, - { - "type": "double", - "name": "edata_time_spent" - }, - { - "type": "double", - "name": "edata_time_difference" - }, - { - "type": "long", - "name": "edata_interaction_count" - }, - { - "type": "string", - "name": "edata_env_summary_env" - }, - { - "type": "string", - "name": "edata_env_summary_count" - }, - { - "type": "string", - "name": "edata_env_summary_time_spent" - }, - { - "type": "string", - "name": "edata_page_summary_id" - }, - { - "type": "string", - "name": "edata_page_summary_type" - }, - { - "type": "string", - "name": "edata_page_summary_env" - }, - { - "type": "string", - "name": "edata_page_summary_visit_count" - }, - { - "type": "string", - "name": "edata_page_summary_time_spent" - }, - { - "type": "string", - "name": "edata_item_responses_item_id" - }, - { - "type": "string", - "name": "edata_item_responses_time_spent" - }, - { - "type": "string", - "name": "edata_item_responses_pass" - }, - { - "type": "string", - "name": "edata_item_responses_score" - }, - { - "type": "string", - "name": "edata_item_responses_max_score" - }, - { - "type": "string", - "name": "edata_item_responses_timestamp" - }, - { - "type": "string", - "name": "device_loc_state" - }, - { - "type": "string", - "name": "device_loc_state_code" - }, - { - "type": "string", - "name": "device_loc_city" - }, - { - "type": "string", - "name": "device_loc_country_code" - }, - { - "type": "string", - "name": "device_loc_country" - }, - { - "type": "string", - "name": "device_os" - }, - { - "type": "string", - "name": "device_make" - }, - { - "type": "string", - "name": "device_id" - }, - { - "type": "long", - "name": "device_mem" - }, - { - "type": "string", - "name": "device_idisk" - }, - { - "type": "string", - "name": "device_edisk" - }, - { - "type": "string", - "name": "device_scrn" - }, - { - "type": "string", - "name": "device_camera" - }, - { - "type": "string", - "name": "device_cpu" - }, - { - "type": "long", - "name": "device_sims" - }, - { - "type": "string", - "name": "device_uaspec_agent" - }, - { - "type": "string", - "name": "device_uaspec_ver" - }, - { - "type": "string", - "name": "device_uaspec_system" - }, - { - "type": "string", - "name": "device_uaspec_platform" - }, - { - "type": "string", - "name": "device_uaspec_raw" - }, - { - "type": "long", - "name": "device_first_access" - }, - { - "type": "string", - "name": "device_loc_state_custom_code" - }, - { - "type": "string", - "name": "device_loc_state_custom_name" - }, - { - "type": "string", - "name": "device_loc_district" - }, - { - "type": "string", - "name": "user_declared_state" - }, - { - "type": "string", - "name": "user_declared_district" - }, - { - "type": "string", - "name": "derived_loc_state" - }, - { - "type": "string", - "name": "derived_loc_district" - }, - { - "type": "string", - "name": "derived_loc_from" - }, - { - "type": "string", - "name": "content_name" - }, - { - "type": "string", - "name": "content_object_type" - }, - { - "type": "string", - "name": "content_type" - }, - { - "type": "string", - "name": "content_media_type" - }, - { - "name": "content_language" - }, - { - "name": "content_medium" - }, - { - "name": "content_gradelevel" - }, - { - "name": "content_subjects" - }, - { - "type": "string", - "name": "content_mimetype" - }, - { - "type": "string", - "name": "content_framework" - }, - { - "type": "string", - "name": "content_board" - }, - { - "type": "string", - "name": "content_status" - }, - { - "type": "double", - "name": "content_version" - }, - { - "type": "long", - "name": "content_last_submitted_on" - }, - { - "type": "long", - "name": "content_last_published_on" - }, - { - "type": "long", - "name": "content_last_updated_on" - }, - { - "type": "string", - "name": "content_created_by" - }, - { - "name": "content_created_for" - }, - { - "type": "string", - "name": "collection_name" - }, - { - "type": "string", - "name": "collection_object_type" - }, - { - "type": "string", - "name": "collection_type" - }, - { - "type": "string", - "name": "collection_media_type" - }, - { - "name": "collection_language" - }, - { - "name": "collection_medium" - }, - { - "name": "collection_gradelevel" - }, - { - "name": "collection_subjects" - }, - { - "type": "string", - "name": "collection_mimetype" - }, - { - "type": "string", - "name": "collection_framework" - }, - { - "type": "string", - "name": "collection_board" - }, - { - "type": "string", - "name": "collection_status" - }, - { - "type": "double", - "name": "collection_version" - }, - { - "type": "long", - "name": "collection_last_submitted_on" - }, - { - "type": "long", - "name": "collection_last_published_on" - }, - { - "type": "long", - "name": "collection_last_updated_on" - }, - { - "type": "string", - "name": "collection_created_by" - }, - { - "name": "collection_created_for" - }, - { - "name": "user_grade_list" - }, - { - "name": "user_language_list" - }, - { - "name": "user_subject_list" - }, - { - "type": "string", - "name": "user_type" - }, - { - "name": "user_roles" - }, - { - "type": "string", - "name": "user_loc_state" - }, - { - "type": "string", - "name": "user_loc_district" - }, - { - "type": "string", - "name": "user_signin_type" - }, - { - "type": "string", - "name": "user_login_type" - }, - { - "type": "string", - "name": "device_loc_iso_state_code" - } - ], - "dimensionsExclusions": [] - }, - "timestampSpec": { - "column": "syncts", - "format": "auto" - } - } - }, - "metricsSpec": [ - { - "type": "longSum", - "name": "total_interactions", - "fieldName": "edata_interaction_count" - }, - { - "type": "doubleSum", - "name": "total_time_spent", - "fieldName": "edata_time_spent" - } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "day", - "queryGranularity": "none", - "rollup": true - } - }, - "ioConfig": { - "topic": "{{env}}.events.summary", - "consumerProperties": { - "bootstrap.servers": "{{kafka_brokers}}" - }, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT14400S", - "useEarliestOffset": false - }, - "tuningConfig": { - "type": "kafka", - "reportParseExceptions": false - } -} diff --git a/ansible/roles/druid-ingestion/templates/telemetry_feedback_index_kafka b/ansible/roles/druid-ingestion/templates/telemetry_feedback_index_kafka deleted file mode 100644 index c9a86819de..0000000000 --- a/ansible/roles/druid-ingestion/templates/telemetry_feedback_index_kafka +++ /dev/null @@ -1,777 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "telemetry-feedback-events", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "flattenSpec": { - "useFieldDiscovery": false, - "fields": [ - { - "type": "root", - "name": "eid" - }, - { - "type": "root", - "name": "mid" - }, - { - "type": "root", - "name": "ets" - }, - { - "type": "root", - "name": "@timestamp" - }, - { - "type": "path", - "name": "actor_id", - "expr": "$.actor.id" - }, - { - "type": "path", - "name": "actor_type", - "expr": "$.actor.type" - }, - { - "type": "path", - "name": "context_channel", - "expr": "$.context.channel" - }, - { - "type": "path", - "name": "context_pdata_id", - "expr": "$.context.pdata.id" - }, - { - "type": "path", - "name": "context_pdata_pid", - "expr": "$.context.pdata.pid" - }, - { - "type": "path", - "name": "context_pdata_ver", - "expr": "$.context.pdata.ver" - }, - { - "type": "path", - "name": "context_env", - "expr": "$.context.env" - }, - { - "type": "path", - "name": "context_sid", - "expr": "$.context.sid" - }, - { - "type": "path", - "name": "context_did", - "expr": "$.context.did" - }, - { - "type": "path", - "name": "context_cdata_type", - "expr": "$.context.cdata[*].type" - }, - { - "type": "path", - "name": "context_cdata_id", - "expr": "$.context.cdata[*].id" - }, - { - "type": "path", - "name": "context_rollup_l1", - "expr": "$.context.rollup.l1" - }, - { - "type": "path", - "name": "context_rollup_l2", - "expr": "$.context.rollup.l2" - }, - { - "type": "path", - "name": "context_rollup_l3", - "expr": "$.context.rollup.l3" - }, - { - "type": "path", - "name": "context_rollup_l4", - "expr": "$.context.rollup.l4" - }, - { - "type": "path", - "name": "object_id", - "expr": "$.object.id" - }, - { - "type": "path", - "name": "object_type", - "expr": "$.object.type" - }, - { - "type": "path", - "name": "object_version", - "expr": "$.object.ver" - }, - { - "type": "path", - "name": "object_rollup_l1", - "expr": "$.object.rollup.l1" - }, - { - "type": "path", - "name": "object_rollup_l2", - "expr": "$.object.rollup.l2" - }, - { - "type": "path", - "name": "object_rollup_l3", - "expr": "$.object.rollup.l3" - }, - { - "type": "path", - "name": "object_rollup_l4", - "expr": "$.object.rollup.l4" - }, - { - "type": "root", - "name": "tags" - }, - { - "type": "path", - "name": "edata_rating", - "expr": "$.edata.rating" - }, - { - "type": "path", - "name": "edata_comments", - "expr": "$.edata.comments" - }, - { - "type": "path", - "name": "edata_commentid", - "expr": "$.edata.commentid" - }, - { - "type": "path", - "name": "edata_commenttxt", - "expr": "$.edata.commenttxt" - }, - { - "type": "path", - "name": "device_loc_state", - "expr": "$.devicedata.state" - }, - { - "type": "path", - "name": "device_loc_state_code", - "expr": "$.devicedata.statecode" - }, - { - "type": "path", - "name": "device_loc_city", - "expr": "$.devicedata.city" - }, - { - "type": "path", - "name": "device_loc_country_code", - "expr": "$.devicedata.countrycode" - }, - { - "type": "path", - "name": "device_loc_country", - "expr": "$.devicedata.country" - }, - { - "type": "path", - "name": "device_os", - "expr": "$.devicedata.devicespec.os" - }, - { - "type": "path", - "name": "device_make", - "expr": "$.devicedata.devicespec.make" - }, - { - "type": "path", - "name": "device_id", - "expr": "$.devicedata.devicespec.id" - }, - { - "type": "path", - "name": "device_mem", - "expr": "$.devicedata.devicespec.mem" - }, - { - "type": "path", - "name": "device_idisk", - "expr": "$.devicedata.devicespec.idisk" - }, - { - "type": "path", - "name": "device_edisk", - "expr": "$.devicedata.devicespec.edisk" - }, - { - "type": "path", - "name": "device_scrn", - "expr": "$.devicedata.devicespec.scrn" - }, - { - "type": "path", - "name": "device_camera", - "expr": "$.devicedata.devicespec.camera" - }, - { - "type": "path", - "name": "device_cpu", - "expr": "$.devicedata.devicespec.cpu" - }, - { - "type": "path", - "name": "device_sims", - "expr": "$.devicedata.devicespec.sims" - }, - { - "type": "path", - "name": "device_uaspec_agent", - "expr": "$.devicedata.uaspec.agent" - }, - { - "type": "path", - "name": "device_uaspec_ver", - "expr": "$.devicedata.uaspec.ver" - }, - { - "type": "path", - "name": "device_uaspec_system", - "expr": "$.devicedata.uaspec.system" - }, - { - "type": "path", - "name": "device_uaspec_platform", - "expr": "$.devicedata.uaspec.platform" - }, - { - "type": "path", - "name": "device_uaspec_raw", - "expr": "$.devicedata.uaspec.raw" - }, - { - "type": "path", - "name": "device_first_access", - "expr": "$.devicedata.firstaccess" - }, - { - "type": "path", - "name": "device_loc_state_custom_code", - "expr": "$.devicedata.statecustomcode" - }, - { - "type": "path", - "name": "device_loc_state_custom_name", - "expr": "$.devicedata.statecustomname" - }, - { - "type": "path", - "name": "device_loc_district", - "expr": "$.devicedata.districtcustom" - }, - { - "type": "path", - "name": "content_name", - "expr": "$.contentdata.name" - }, - { - "type": "path", - "name": "content_object_type", - "expr": "$.contentdata.objecttype" - }, - { - "type": "path", - "name": "content_type", - "expr": "$.contentdata.contenttype" - }, - { - "type": "path", - "name": "content_media_type", - "expr": "$.contentdata.mediatype" - }, - { - "type": "path", - "name": "content_language", - "expr": "$.contentdata.language[*]" - }, - { - "type": "path", - "name": "content_medium", - "expr": "$.contentdata.medium[*]" - }, - { - "type": "path", - "name": "content_gradelevel", - "expr": "$.contentdata.gradelevel[*]" - }, - { - "type": "path", - "name": "content_subjects", - "expr": "$.contentdata.subject[*]" - }, - { - "type": "path", - "name": "content_mimetype", - "expr": "$.contentdata.mimetype" - }, - { - "type": "path", - "name": "content_framework", - "expr": "$.contentdata.framework" - }, - { - "type": "path", - "name": "content_board", - "expr": "$.contentdata.board" - }, - { - "type": "path", - "name": "content_status", - "expr": "$.contentdata.status" - }, - { - "type": "path", - "name": "content_version", - "expr": "$.contentdata.pkgversion" - }, - { - "type": "path", - "name": "content_last_submitted_on", - "expr": "$.contentdata.lastsubmittedon" - }, - { - "type": "path", - "name": "content_last_published_on", - "expr": "$.contentdata.lastpublishedon" - }, - { - "type": "path", - "name": "content_last_updated_on", - "expr": "$.contentdata.lastupdatedon" - }, - { - "type": "path", - "name": "user_grade_list", - "expr": "$.userdata.gradelist[*]" - }, - { - "type": "path", - "name": "user_language_list", - "expr": "$.userdata.languagelist[*]" - }, - { - "type": "path", - "name": "user_subject_list", - "expr": "$.userdata.subjectlist[*]" - }, - { - "type": "path", - "name": "user_type", - "expr": "$.userdata.usertype" - }, - { - "type": "path", - "name": "user_roles", - "expr": "$.userdata.roles[*]" - }, - { - "type": "path", - "name": "user_loc_state", - "expr": "$.userdata.state" - }, - { - "type": "path", - "name": "user_loc_district", - "expr": "$.userdata.district" - }, - { - "type": "path", - "name": "user_signin_type", - "expr": "$.userdata.usersignintype" - }, - { - "type": "path", - "name": "user_login_type", - "expr": "$.userdata.userlogintype" - }, - { - "type": "path", - "name": "device_loc_iso_state_code", - "expr": "$.devicedata.iso3166statecode" - } - ] - }, - "dimensionsSpec": { - "dimensions": [ - { - "type": "string", - "name": "eid" - }, - { - "type": "string", - "name": "mid" - }, - { - "type": "long", - "name": "ets" - }, - { - "type": "string", - "name": "@timestamp" - }, - { - "type": "string", - "name": "actor_id" - }, - { - "type": "string", - "name": "actor_type" - }, - { - "type": "string", - "name": "context_channel" - }, - { - "type": "string", - "name": "context_pdata_id" - }, - { - "type": "string", - "name": "context_pdata_pid" - }, - { - "type": "string", - "name": "context_pdata_ver" - }, - { - "type": "string", - "name": "context_env" - }, - { - "type": "string", - "name": "context_sid" - }, - { - "type": "string", - "name": "context_did" - }, - { - "name": "context_cdata_type" - }, - { - "name": "context_cdata_id" - }, - { - "type": "string", - "name": "context_rollup_l1" - }, - { - "type": "string", - "name": "context_rollup_l2" - }, - { - "type": "string", - "name": "context_rollup_l3" - }, - { - "type": "string", - "name": "context_rollup_l4" - }, - { - "type": "string", - "name": "object_id" - }, - { - "type": "string", - "name": "object_type" - }, - { - "type": "string", - "name": "object_version" - }, - { - "type": "string", - "name": "object_rollup_l1" - }, - { - "type": "string", - "name": "object_rollup_l2" - }, - { - "type": "string", - "name": "object_rollup_l3" - }, - { - "type": "string", - "name": "object_rollup_l4" - }, - { - "name": "tags" - }, - { - "type": "long", - "name": "edata_rating" - }, - { - "type": "string", - "name": "edata_comments" - }, - { - "type": "string", - "name": "edata_commentid" - }, - { - "type": "string", - "name": "edata_commenttxt" - }, - { - "type": "string", - "name": "device_loc_state" - }, - { - "type": "string", - "name": "device_loc_state_code" - }, - { - "type": "string", - "name": "device_loc_city" - }, - { - "type": "string", - "name": "device_loc_country_code" - }, - { - "type": "string", - "name": "device_loc_country" - }, - { - "type": "string", - "name": "device_os" - }, - { - "type": "string", - "name": "device_make" - }, - { - "type": "string", - "name": "device_id" - }, - { - "type": "long", - "name": "device_mem" - }, - { - "type": "string", - "name": "device_idisk" - }, - { - "type": "string", - "name": "device_edisk" - }, - { - "type": "string", - "name": "device_scrn" - }, - { - "type": "string", - "name": "device_camera" - }, - { - "type": "string", - "name": "device_cpu" - }, - { - "type": "long", - "name": "device_sims" - }, - { - "type": "string", - "name": "device_uaspec_agent" - }, - { - "type": "string", - "name": "device_uaspec_ver" - }, - { - "type": "string", - "name": "device_uaspec_system" - }, - { - "type": "string", - "name": "device_uaspec_platform" - }, - { - "type": "string", - "name": "device_uaspec_raw" - }, - { - "type": "long", - "name": "device_first_access" - }, - { - "type": "string", - "name": "device_loc_state_custom_code" - }, - { - "type": "string", - "name": "device_loc_state_custom_name" - }, - { - "type": "string", - "name": "device_loc_district" - }, - { - "type": "string", - "name": "content_name" - }, - { - "type": "string", - "name": "content_object_type" - }, - { - "type": "string", - "name": "content_type" - }, - { - "type": "string", - "name": "content_media_type" - }, - { - "name": "content_language" - }, - { - "name": "content_medium" - }, - { - "name": "content_gradelevel" - }, - { - "name": "content_subjects" - }, - { - "type": "string", - "name": "content_mimetype" - }, - { - "type": "string", - "name": "content_framework" - }, - { - "type": "string", - "name": "content_board" - }, - { - "type": "string", - "name": "content_status" - }, - { - "type": "double", - "name": "content_version" - }, - { - "type": "long", - "name": "content_last_submitted_on" - }, - { - "type": "long", - "name": "content_last_published_on" - }, - { - "type": "long", - "name": "content_last_updated_on" - }, - { - "name": "user_grade_list" - }, - { - "name": "user_language_list" - }, - { - "name": "user_subject_list" - }, - { - "type": "string", - "name": "user_type" - }, - { - "name": "user_roles" - }, - { - "type": "string", - "name": "user_loc_state" - }, - { - "type": "string", - "name": "user_loc_district" - }, - { - "type": "string", - "name": "user_signin_type" - }, - { - "type": "string", - "name": "user_login_type" - }, - { - "type": "string", - "name": "device_loc_iso_state_code" - } - ], - "dimensionsExclusions": [] - }, - "timestampSpec": { - "column": "syncts", - "format": "auto" - } - } - }, - "metricsSpec": [ - { - "type": "count", - "name": "count" - }, - { - "type": "longSum", - "name": "total_rating", - "fieldName": "edata_rating" - } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "day", - "queryGranularity": "none", - "rollup": true - }, - "transformSpec": { - "filter": { - "type": "selector", - "dimension": "eid", - "value": "FEEDBACK" - } - } - }, - "ioConfig": { - "topic": "{{env}}.events.telemetry", - "consumerProperties": { - "bootstrap.servers": "{{kafka_brokers}}" - }, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT14400S", - "useEarliestOffset": false - }, - "tuningConfig": { - "type": "kafka", - "reportParseExceptions": false - } -} diff --git a/ansible/roles/druid-ingestion/templates/telemetry_index_kafka b/ansible/roles/druid-ingestion/templates/telemetry_index_kafka deleted file mode 100644 index daf253e287..0000000000 --- a/ansible/roles/druid-ingestion/templates/telemetry_index_kafka +++ /dev/null @@ -1,1362 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "telemetry-events", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "flattenSpec": { - "useFieldDiscovery": false, - "fields": [ - { - "type": "root", - "name": "eid" - }, - { - "type": "root", - "name": "mid" - }, - { - "type": "root", - "name": "syncts" - }, - { - "type": "root", - "name": "@timestamp" - }, - { - "type": "path", - "name": "actor_id", - "expr": "$.actor.id" - }, - { - "type": "path", - "name": "actor_type", - "expr": "$.actor.type" - }, - { - "type": "path", - "name": "context_channel", - "expr": "$.context.channel" - }, - { - "type": "path", - "name": "context_pdata_id", - "expr": "$.context.pdata.id" - }, - { - "type": "path", - "name": "context_pdata_pid", - "expr": "$.context.pdata.pid" - }, - { - "type": "path", - "name": "context_pdata_ver", - "expr": "$.context.pdata.ver" - }, - { - "type": "path", - "name": "context_env", - "expr": "$.context.env" - }, - { - "type": "path", - "name": "context_sid", - "expr": "$.context.sid" - }, - { - "type": "path", - "name": "context_did", - "expr": "$.context.did" - }, - { - "type": "path", - "name": "context_cdata_type", - "expr": "$.context.cdata[*].type" - }, - { - "type": "path", - "name": "context_cdata_id", - "expr": "$.context.cdata[*].id" - }, - { - "type": "path", - "name": "context_rollup_l1", - "expr": "$.context.rollup.l1" - }, - { - "type": "path", - "name": "context_rollup_l2", - "expr": "$.context.rollup.l2" - }, - { - "type": "path", - "name": "context_rollup_l3", - "expr": "$.context.rollup.l3" - }, - { - "type": "path", - "name": "context_rollup_l4", - "expr": "$.context.rollup.l4" - }, - { - "type": "path", - "name": "content_play_progress", - "expr": "$.edata.summary[*].progress" - }, - { - "type": "path", - "name": "object_id", - "expr": "$.object.id" - }, - { - "type": "path", - "name": "object_type", - "expr": "$.object.type" - }, - { - "type": "path", - "name": "object_version", - "expr": "$.object.ver" - }, - { - "type": "path", - "name": "object_rollup_l1", - "expr": "$.object.rollup.l1" - }, - { - "type": "path", - "name": "object_rollup_l2", - "expr": "$.object.rollup.l2" - }, - { - "type": "path", - "name": "object_rollup_l3", - "expr": "$.object.rollup.l3" - }, - { - "type": "path", - "name": "object_rollup_l4", - "expr": "$.object.rollup.l4" - }, - { - "type": "root", - "name": "tags" - }, - { - "type": "path", - "name": "edata_type", - "expr": "$.edata.type" - }, - { - "type": "path", - "name": "edata_subtype", - "expr": "$.edata.subtype" - }, - { - "type": "path", - "name": "edata_mode", - "expr": "$.edata.mode" - }, - { - "type": "path", - "name": "edata_pageid", - "expr": "$.edata.pageid" - }, - { - "type": "path", - "name": "edata_uri", - "expr": "$.edata.uri" - }, - { - "type": "path", - "name": "edata_id", - "expr": "$.edata.id" - }, - { - "type": "path", - "name": "edata_duration", - "expr": "$.edata.duration" - }, - { - "type": "path", - "name": "edata_index", - "expr": "$.edata.index" - }, - { - "type": "path", - "name": "edata_pass", - "expr": "$.edata.pass" - }, - { - "type": "path", - "name": "edata_score", - "expr": "$.edata.score" - }, - { - "type": "path", - "name": "edata_resvalues", - "expr": "$.edata.resvalues[*]" - }, - { - "type": "path", - "name": "edata_item_id", - "expr": "$.edata.item.id" - }, - { - "type": "path", - "name": "edata_item_title", - "expr": "$.edata.item.title" - }, - { - "type": "path", - "name": "edata_item_maxscore", - "expr": "$.edata.item.maxscore" - }, - { - "type": "path", - "name": "edata_target_id", - "expr": "$.edata.target.id" - }, - { - "type": "path", - "name": "edata_target_type", - "expr": "$.edata.target.type" - }, - { - "type": "path", - "name": "edata_rating", - "expr": "$.edata.rating" - }, - { - "type": "path", - "name": "edata_comments", - "expr": "$.edata.comments" - }, - { - "type": "path", - "name": "edata_commentid", - "expr": "$.edata.commentid" - }, - { - "type": "path", - "name": "edata_commenttxt", - "expr": "$.edata.commenttxt" - }, - { - "type": "path", - "name": "edata_dir", - "expr": "$.edata.dir" - }, - { - "type": "path", - "name": "edata_items_id", - "expr": "$.edata.items[*].id" - }, - { - "type": "path", - "name": "edata_items_type", - "expr": "$.edata.items[*].type" - }, - { - "type": "path", - "name": "edata_items_origin_id", - "expr": "$.edata.items[*].origin.id" - }, - { - "type": "path", - "name": "edata_items_origin_type", - "expr": "$.edata.items[*].origin.type" - }, - { - "type": "path", - "name": "edata_items_to_id", - "expr": "$.edata.items[*].to.id" - }, - { - "type": "path", - "name": "edata_items_to_type", - "expr": "$.edata.items[*].to.type" - }, - { - "type": "path", - "name": "edata_plugin_id", - "expr": "$.edata.plugin.id" - }, - { - "type": "path", - "name": "edata_plugin_ver", - "expr": "$.edata.plugin.ver" - }, - { - "type": "path", - "name": "edata_plugin_category", - "expr": "$.edata.plugin.category" - }, - { - "type": "path", - "name": "edata_props", - "expr": "$.edata.props[*]" - }, - { - "type": "path", - "name": "edata_state", - "expr": "$.edata.state" - }, - { - "type": "path", - "name": "edata_prevstate", - "expr": "$.edata.prevstate" - }, - { - "type": "path", - "name": "edata_size", - "expr": "$.edata.size" - }, - { - "type": "path", - "name": "edata_filters_dialcodes", - "expr": "$.edata.filters.dialcodes" - }, - { - "type": "path", - "name": "edata_topn_identifier", - "expr": "$.edata.topn[*].identifier" - }, - { - "type": "path", - "name": "edata_visits_objid", - "expr": "$.edata.visits[*].objid" - }, - { - "type": "path", - "name": "edata_visits_objtype", - "expr": "$.edata.visits[*].objtype" - }, - { - "type": "path", - "name": "edata_visits_objver", - "expr": "$.edata.visits[*].objver" - }, - { - "type": "path", - "name": "edata_visits_index", - "expr": "$.edata.visits[*].index" - }, - { - "type": "path", - "name": "device_loc_state", - "expr": "$.devicedata.state" - }, - { - "type": "path", - "name": "device_loc_state_code", - "expr": "$.devicedata.statecode" - }, - { - "type": "path", - "name": "device_loc_city", - "expr": "$.devicedata.city" - }, - { - "type": "path", - "name": "device_loc_country_code", - "expr": "$.devicedata.countrycode" - }, - { - "type": "path", - "name": "device_loc_country", - "expr": "$.devicedata.country" - }, - { - "type": "path", - "name": "device_os", - "expr": "$.devicedata.devicespec.os" - }, - { - "type": "path", - "name": "device_make", - "expr": "$.devicedata.devicespec.make" - }, - { - "type": "path", - "name": "device_id", - "expr": "$.devicedata.devicespec.id" - }, - { - "type": "path", - "name": "device_mem", - "expr": "$.devicedata.devicespec.mem" - }, - { - "type": "path", - "name": "device_idisk", - "expr": "$.devicedata.devicespec.idisk" - }, - { - "type": "path", - "name": "device_edisk", - "expr": "$.devicedata.devicespec.edisk" - }, - { - "type": "path", - "name": "device_scrn", - "expr": "$.devicedata.devicespec.scrn" - }, - { - "type": "path", - "name": "device_camera", - "expr": "$.devicedata.devicespec.camera" - }, - { - "type": "path", - "name": "device_cpu", - "expr": "$.devicedata.devicespec.cpu" - }, - { - "type": "path", - "name": "device_sims", - "expr": "$.devicedata.devicespec.sims" - }, - { - "type": "path", - "name": "device_uaspec_agent", - "expr": "$.devicedata.uaspec.agent" - }, - { - "type": "path", - "name": "device_uaspec_ver", - "expr": "$.devicedata.uaspec.ver" - }, - { - "type": "path", - "name": "device_uaspec_system", - "expr": "$.devicedata.uaspec.system" - }, - { - "type": "path", - "name": "device_uaspec_platform", - "expr": "$.devicedata.uaspec.platform" - }, - { - "type": "path", - "name": "device_uaspec_raw", - "expr": "$.devicedata.uaspec.raw" - }, - { - "type": "path", - "name": "device_first_access", - "expr": "$.devicedata.firstaccess" - }, - { - "type": "path", - "name": "device_loc_state_custom_code", - "expr": "$.devicedata.statecustomcode" - }, - { - "type": "path", - "name": "device_loc_state_custom_name", - "expr": "$.devicedata.statecustomname" - }, - { - "type": "path", - "name": "device_loc_district", - "expr": "$.devicedata.districtcustom" - }, - { - "type": "path", - "name": "user_declared_state", - "expr": "$.devicedata.userdeclared.state" - }, - { - "type": "path", - "name": "user_declared_district", - "expr": "$.devicedata.userdeclared.district" - }, - { - "type": "path", - "name": "derived_loc_state", - "expr": "$.derivedlocationdata.state" - }, - { - "type": "path", - "name": "derived_loc_district", - "expr": "$.derivedlocationdata.district" - }, - { - "type": "path", - "name": "derived_loc_from", - "expr": "$.derivedlocationdata.from" - }, - { - "type": "path", - "name": "content_name", - "expr": "$.contentdata.name" - }, - { - "type": "path", - "name": "content_object_type", - "expr": "$.contentdata.objecttype" - }, - { - "type": "path", - "name": "content_type", - "expr": "$.contentdata.contenttype" - }, - { - "type": "path", - "name": "content_media_type", - "expr": "$.contentdata.mediatype" - }, - { - "type": "path", - "name": "content_language", - "expr": "$.contentdata.language[*]" - }, - { - "type": "path", - "name": "content_medium", - "expr": "$.contentdata.medium[*]" - }, - { - "type": "path", - "name": "content_gradelevel", - "expr": "$.contentdata.gradelevel[*]" - }, - { - "type": "path", - "name": "content_subjects", - "expr": "$.contentdata.subject[*]" - }, - { - "type": "path", - "name": "content_mimetype", - "expr": "$.contentdata.mimetype" - }, - { - "type": "path", - "name": "content_framework", - "expr": "$.contentdata.framework" - }, - { - "type": "path", - "name": "content_board", - "expr": "$.contentdata.board" - }, - { - "type": "path", - "name": "content_status", - "expr": "$.contentdata.status" - }, - { - "type": "path", - "name": "content_version", - "expr": "$.contentdata.pkgversion" - }, - { - "type": "path", - "name": "content_last_submitted_on", - "expr": "$.contentdata.lastsubmittedon" - }, - { - "type": "path", - "name": "content_last_published_on", - "expr": "$.contentdata.lastpublishedon" - }, - { - "type": "path", - "name": "content_last_updated_on", - "expr": "$.contentdata.lastupdatedon" - }, - { - "type": "path", - "name": "content_created_by", - "expr": "$.contentdata.createdby" - }, - { - "type": "path", - "name": "content_created_for", - "expr": "$.contentdata.createdfor" - }, - { - "type": "path", - "name": "collection_name", - "expr": "$.collectiondata.name" - }, - { - "type": "path", - "name": "collection_object_type", - "expr": "$.collectiondata.objecttype" - }, - { - "type": "path", - "name": "collection_type", - "expr": "$.collectiondata.contenttype" - }, - { - "type": "path", - "name": "collection_media_type", - "expr": "$.collectiondata.mediatype" - }, - { - "type": "path", - "name": "collection_language", - "expr": "$.collectiondata.language[*]" - }, - { - "type": "path", - "name": "collection_medium", - "expr": "$.collectiondata.medium[*]" - }, - { - "type": "path", - "name": "collection_gradelevel", - "expr": "$.collectiondata.gradelevel[*]" - }, - { - "type": "path", - "name": "collection_subjects", - "expr": "$.collectiondata.subject[*]" - }, - { - "type": "path", - "name": "collection_mimetype", - "expr": "$.collectiondata.mimetype" - }, - { - "type": "path", - "name": "collection_framework", - "expr": "$.collectiondata.framework" - }, - { - "type": "path", - "name": "collection_board", - "expr": "$.collectiondata.board" - }, - { - "type": "path", - "name": "collection_status", - "expr": "$.collectiondata.status" - }, - { - "type": "path", - "name": "collection_version", - "expr": "$.collectiondata.pkgversion" - }, - { - "type": "path", - "name": "collection_last_submitted_on", - "expr": "$.collectiondata.lastsubmittedon" - }, - { - "type": "path", - "name": "collection_last_published_on", - "expr": "$.collectiondata.lastpublishedon" - }, - { - "type": "path", - "name": "collection_last_updated_on", - "expr": "$.collectiondata.lastupdatedon" - }, - { - "type": "path", - "name": "collection_created_by", - "expr": "$.collectiondata.createdby" - }, - { - "type": "path", - "name": "collection_created_for", - "expr": "$.collectiondata.createdfor" - }, - { - "type": "path", - "name": "user_grade_list", - "expr": "$.userdata.gradelist[*]" - }, - { - "type": "path", - "name": "user_language_list", - "expr": "$.userdata.languagelist[*]" - }, - { - "type": "path", - "name": "user_subject_list", - "expr": "$.userdata.subjectlist[*]" - }, - { - "type": "path", - "name": "user_type", - "expr": "$.userdata.usertype" - }, - { - "type": "path", - "name": "user_roles", - "expr": "$.userdata.roles[*]" - }, - { - "type": "path", - "name": "user_loc_state", - "expr": "$.userdata.state" - }, - { - "type": "path", - "name": "user_loc_district", - "expr": "$.userdata.district" - }, - { - "type": "path", - "name": "user_signin_type", - "expr": "$.userdata.usersignintype" - }, - { - "type": "path", - "name": "user_login_type", - "expr": "$.userdata.userlogintype" - }, - { - "type": "path", - "name": "dialcode_channel", - "expr": "$.dialcodedata.channel" - }, - { - "type": "path", - "name": "dialcode_batchcode", - "expr": "$.dialcodedata.batchcode" - }, - { - "type": "path", - "name": "dialcode_publisher", - "expr": "$.dialcodedata.publisher" - }, - { - "type": "path", - "name": "dialcode_generated_on", - "expr": "$.dialcodedata.generatedon" - }, - { - "type": "path", - "name": "dialcode_published_on", - "expr": "$.dialcodedata.publishedon" - }, - { - "type": "path", - "name": "dialcode_object_type", - "expr": "$.dialcodedata.objecttype" - }, - { - "type": "path", - "name": "device_loc_iso_state_code", - "expr": "$.devicedata.iso3166statecode" - } - ] - }, - "dimensionsSpec": { - "dimensions": [ - { - "type": "string", - "name": "eid" - }, - { - "type": "string", - "name": "mid" - }, - { - "type": "long", - "name": "syncts" - }, - { - "type": "string", - "name": "@timestamp" - }, - { - "type": "string", - "name": "actor_id" - }, - { - "type": "string", - "name": "actor_type" - }, - { - "type": "string", - "name": "context_channel" - }, - { - "type": "string", - "name": "context_pdata_id" - }, - { - "type": "string", - "name": "context_pdata_pid" - }, - { - "type": "string", - "name": "context_pdata_ver" - }, - { - "type": "string", - "name": "context_env" - }, - { - "type": "string", - "name": "context_sid" - }, - { - "type": "string", - "name": "context_did" - }, - { - "name": "context_cdata_type" - }, - { - "name": "context_cdata_id" - }, - { - "type": "string", - "name": "context_rollup_l1" - }, - { - "type": "string", - "name": "context_rollup_l2" - }, - { - "type": "string", - "name": "context_rollup_l3" - }, - { - "type": "string", - "name": "context_rollup_l4" - }, - { - "type": "string", - "name": "object_id" - }, - { - "type": "string", - "name": "object_type" - }, - { - "type": "string", - "name": "object_version" - }, - { - "type": "string", - "name": "object_rollup_l1" - }, - { - "type": "string", - "name": "object_rollup_l2" - }, - { - "type": "string", - "name": "object_rollup_l3" - }, - { - "type": "string", - "name": "object_rollup_l4" - }, - { - "name": "tags" - }, - { - "type": "string", - "name": "edata_type" - }, - { - "type": "string", - "name": "edata_subtype" - }, - { - "type": "string", - "name": "edata_mode" - }, - { - "type": "string", - "name": "edata_pageid" - }, - { - "type": "string", - "name": "edata_uri" - }, - { - "type": "string", - "name": "edata_id" - }, - { - "type": "double", - "name": "edata_duration" - }, - { - "type": "long", - "name": "edata_index" - }, - { - "type": "string", - "name": "edata_pass" - }, - { - "type": "double", - "name": "edata_score" - }, - { - "name": "edata_resvalues" - }, - { - "type": "string", - "name": "edata_item_id" - }, - { - "type": "string", - "name": "edata_item_title" - }, - { - "type": "double", - "name": "edata_item_maxscore" - }, - { - "type": "string", - "name": "edata_target_id" - }, - { - "type": "string", - "name": "edata_target_type" - }, - { - "type": "long", - "name": "edata_rating" - }, - { - "type": "string", - "name": "edata_comments" - }, - { - "type": "string", - "name": "edata_commentid" - }, - { - "type": "string", - "name": "edata_commenttxt" - }, - { - "type": "string", - "name": "edata_dir" - }, - { - "type": "string", - "name": "edata_items_id" - }, - { - "type": "string", - "name": "edata_items_type" - }, - { - "type": "string", - "name": "edata_items_origin_id" - }, - { - "type": "string", - "name": "edata_items_origin_type" - }, - { - "type": "string", - "name": "edata_items_to_id" - }, - { - "type": "string", - "name": "edata_items_to_type" - }, - { - "type": "string", - "name": "edata_plugin_id" - }, - { - "type": "string", - "name": "edata_plugin_ver" - }, - { - "type": "string", - "name": "edata_plugin_category" - }, - { - "name": "edata_props" - }, - { - "type": "string", - "name": "edata_state" - }, - { - "type": "string", - "name": "edata_prevstate" - }, - { - "type": "long", - "name": "edata_size" - }, - { - "name": "edata_filters_dialcodes" - }, - { - "type": "string", - "name": "edata_topn_identifier" - }, - { - "type": "string", - "name": "edata_visits_objid" - }, - { - "type": "string", - "name": "edata_visits_objtype" - }, - { - "type": "string", - "name": "edata_visits_objver" - }, - { - "type": "string", - "name": "edata_visits_index" - }, - { - "type": "string", - "name": "device_loc_state" - }, - { - "type": "string", - "name": "device_loc_state_code" - }, - { - "type": "string", - "name": "device_loc_city" - }, - { - "type": "string", - "name": "device_loc_country_code" - }, - { - "type": "string", - "name": "device_loc_country" - }, - { - "type": "string", - "name": "device_os" - }, - { - "type": "string", - "name": "device_make" - }, - { - "type": "string", - "name": "device_id" - }, - { - "type": "long", - "name": "device_mem" - }, - { - "type": "string", - "name": "device_idisk" - }, - { - "type": "string", - "name": "device_edisk" - }, - { - "type": "string", - "name": "device_scrn" - }, - { - "type": "string", - "name": "device_camera" - }, - { - "type": "string", - "name": "device_cpu" - }, - { - "type": "long", - "name": "device_sims" - }, - { - "type": "string", - "name": "device_uaspec_agent" - }, - { - "type": "string", - "name": "device_uaspec_ver" - }, - { - "type": "string", - "name": "device_uaspec_system" - }, - { - "type": "string", - "name": "device_uaspec_platform" - }, - { - "type": "string", - "name": "device_uaspec_raw" - }, - { - "type": "long", - "name": "device_first_access" - }, - { - "type": "string", - "name": "device_loc_state_custom_code" - }, - { - "type": "string", - "name": "device_loc_state_custom_name" - }, - { - "type": "string", - "name": "device_loc_district" - }, - { - "type": "string", - "name": "user_declared_state" - }, - { - "type": "string", - "name": "user_declared_district" - }, - { - "type": "string", - "name": "derived_loc_state" - }, - { - "type": "string", - "name": "derived_loc_district" - }, - { - "type": "string", - "name": "derived_loc_from" - }, - { - "type": "string", - "name": "content_name" - }, - { - "type": "string", - "name": "content_object_type" - }, - { - "type": "string", - "name": "content_type" - }, - { - "type": "string", - "name": "content_media_type" - }, - { - "name": "content_language" - }, - { - "name": "content_medium" - }, - { - "name": "content_gradelevel" - }, - { - "name": "content_subjects" - }, - { - "type": "string", - "name": "content_mimetype" - }, - { - "type": "string", - "name": "content_framework" - }, - { - "type": "string", - "name": "content_board" - }, - { - "type": "string", - "name": "content_status" - }, - { - "type": "double", - "name": "content_version" - }, - { - "type": "long", - "name": "content_last_submitted_on" - }, - { - "type": "long", - "name": "content_last_published_on" - }, - { - "type": "long", - "name": "content_last_updated_on" - }, - { - "type": "string", - "name": "content_created_by" - }, - { - "name": "content_created_for" - }, - { - "name" : "content_play_progress" - }, - { - "type": "string", - "name": "collection_name" - }, - { - "type": "string", - "name": "collection_object_type" - }, - { - "type": "string", - "name": "collection_type" - }, - { - "type": "string", - "name": "collection_media_type" - }, - { - "name": "collection_language" - }, - { - "name": "collection_medium" - }, - { - "name": "collection_gradelevel" - }, - { - "name": "collection_subjects" - }, - { - "type": "string", - "name": "collection_mimetype" - }, - { - "type": "string", - "name": "collection_framework" - }, - { - "type": "string", - "name": "collection_board" - }, - { - "type": "string", - "name": "collection_status" - }, - { - "type": "double", - "name": "collection_version" - }, - { - "type": "long", - "name": "collection_last_submitted_on" - }, - { - "type": "long", - "name": "collection_last_published_on" - }, - { - "type": "long", - "name": "collection_last_updated_on" - }, - { - "type": "string", - "name": "collection_created_by" - }, - { - "name": "collection_created_for" - }, - { - "name": "user_grade_list" - }, - { - "name": "user_language_list" - }, - { - "name": "user_subject_list" - }, - { - "type": "string", - "name": "user_type" - }, - { - "name": "user_roles" - }, - { - "type": "string", - "name": "user_loc_state" - }, - { - "type": "string", - "name": "user_loc_district" - }, - { - "type": "string", - "name": "user_signin_type" - }, - { - "type": "string", - "name": "user_login_type" - }, - { - "type": "string", - "name": "dialcode_channel" - }, - { - "type": "string", - "name": "dialcode_batchcode" - }, - { - "type": "string", - "name": "dialcode_publisher" - }, - { - "type": "long", - "name": "dialcode_generated_on" - }, - { - "type": "long", - "name": "dialcode_published_on" - }, - { - "type": "string", - "name": "dialcode_object_type" - }, - { - "type": "string", - "name": "device_loc_iso_state_code" - } - ], - "dimensionsExclusions": [] - }, - "timestampSpec": { - "column": "ets", - "format": "auto" - } - } - }, - "metricsSpec": [], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "day", - "queryGranularity": "none", - "rollup": false - } - }, - "ioConfig": { - "topic": "{{env}}.events.telemetry", - "consumerProperties": { - "bootstrap.servers": "{{kafka_brokers}}" - }, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT14400S", - "useEarliestOffset": false - }, - "tuningConfig": { - "type": "kafka", - "reportParseExceptions": false - } -} diff --git a/ansible/roles/druid-ingestion/templates/telemetry_index_syncts_kafka b/ansible/roles/druid-ingestion/templates/telemetry_index_syncts_kafka deleted file mode 100644 index 04266a46f0..0000000000 --- a/ansible/roles/druid-ingestion/templates/telemetry_index_syncts_kafka +++ /dev/null @@ -1,1355 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "telemetry-events", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "flattenSpec": { - "useFieldDiscovery": false, - "fields": [ - { - "type": "root", - "name": "eid" - }, - { - "type": "root", - "name": "mid" - }, - { - "type": "root", - "name": "ets" - }, - { - "type": "root", - "name": "@timestamp" - }, - { - "type": "path", - "name": "actor_id", - "expr": "$.actor.id" - }, - { - "type": "path", - "name": "actor_type", - "expr": "$.actor.type" - }, - { - "type": "path", - "name": "context_channel", - "expr": "$.context.channel" - }, - { - "type": "path", - "name": "context_pdata_id", - "expr": "$.context.pdata.id" - }, - { - "type": "path", - "name": "context_pdata_pid", - "expr": "$.context.pdata.pid" - }, - { - "type": "path", - "name": "context_pdata_ver", - "expr": "$.context.pdata.ver" - }, - { - "type": "path", - "name": "context_env", - "expr": "$.context.env" - }, - { - "type": "path", - "name": "context_sid", - "expr": "$.context.sid" - }, - { - "type": "path", - "name": "context_did", - "expr": "$.context.did" - }, - { - "type": "path", - "name": "context_cdata_type", - "expr": "$.context.cdata[*].type" - }, - { - "type": "path", - "name": "context_cdata_id", - "expr": "$.context.cdata[*].id" - }, - { - "type": "path", - "name": "context_rollup_l1", - "expr": "$.context.rollup.l1" - }, - { - "type": "path", - "name": "context_rollup_l2", - "expr": "$.context.rollup.l2" - }, - { - "type": "path", - "name": "context_rollup_l3", - "expr": "$.context.rollup.l3" - }, - { - "type": "path", - "name": "context_rollup_l4", - "expr": "$.context.rollup.l4" - }, - { - "type": "path", - "name": "object_id", - "expr": "$.object.id" - }, - { - "type": "path", - "name": "object_type", - "expr": "$.object.type" - }, - { - "type": "path", - "name": "object_version", - "expr": "$.object.ver" - }, - { - "type": "path", - "name": "object_rollup_l1", - "expr": "$.object.rollup.l1" - }, - { - "type": "path", - "name": "object_rollup_l2", - "expr": "$.object.rollup.l2" - }, - { - "type": "path", - "name": "object_rollup_l3", - "expr": "$.object.rollup.l3" - }, - { - "type": "path", - "name": "object_rollup_l4", - "expr": "$.object.rollup.l4" - }, - { - "type": "root", - "name": "tags" - }, - { - "type": "path", - "name": "edata_type", - "expr": "$.edata.type" - }, - { - "type": "path", - "name": "edata_subtype", - "expr": "$.edata.subtype" - }, - { - "type": "path", - "name": "edata_mode", - "expr": "$.edata.mode" - }, - { - "type": "path", - "name": "edata_pageid", - "expr": "$.edata.pageid" - }, - { - "type": "path", - "name": "edata_uri", - "expr": "$.edata.uri" - }, - { - "type": "path", - "name": "edata_id", - "expr": "$.edata.id" - }, - { - "type": "path", - "name": "edata_duration", - "expr": "$.edata.duration" - }, - { - "type": "path", - "name": "edata_index", - "expr": "$.edata.index" - }, - { - "type": "path", - "name": "edata_pass", - "expr": "$.edata.pass" - }, - { - "type": "path", - "name": "edata_score", - "expr": "$.edata.score" - }, - { - "type": "path", - "name": "edata_resvalues", - "expr": "$.edata.resvalues[*]" - }, - { - "type": "path", - "name": "edata_item_id", - "expr": "$.edata.item.id" - }, - { - "type": "path", - "name": "edata_item_title", - "expr": "$.edata.item.title" - }, - { - "type": "path", - "name": "edata_item_maxscore", - "expr": "$.edata.item.maxscore" - }, - { - "type": "path", - "name": "edata_target_id", - "expr": "$.edata.target.id" - }, - { - "type": "path", - "name": "edata_target_type", - "expr": "$.edata.target.type" - }, - { - "type": "path", - "name": "edata_rating", - "expr": "$.edata.rating" - }, - { - "type": "path", - "name": "edata_comments", - "expr": "$.edata.comments" - }, - { - "type": "path", - "name": "edata_commentid", - "expr": "$.edata.commentid" - }, - { - "type": "path", - "name": "edata_commenttxt", - "expr": "$.edata.commenttxt" - }, - { - "type": "path", - "name": "edata_dir", - "expr": "$.edata.dir" - }, - { - "type": "path", - "name": "edata_items_id", - "expr": "$.edata.items[*].id" - }, - { - "type": "path", - "name": "edata_items_type", - "expr": "$.edata.items[*].type" - }, - { - "type": "path", - "name": "edata_items_origin_id", - "expr": "$.edata.items[*].origin.id" - }, - { - "type": "path", - "name": "edata_items_origin_type", - "expr": "$.edata.items[*].origin.type" - }, - { - "type": "path", - "name": "edata_items_to_id", - "expr": "$.edata.items[*].to.id" - }, - { - "type": "path", - "name": "edata_items_to_type", - "expr": "$.edata.items[*].to.type" - }, - { - "type": "path", - "name": "edata_plugin_id", - "expr": "$.edata.plugin.id" - }, - { - "type": "path", - "name": "edata_plugin_ver", - "expr": "$.edata.plugin.ver" - }, - { - "type": "path", - "name": "edata_plugin_category", - "expr": "$.edata.plugin.category" - }, - { - "type": "path", - "name": "edata_props", - "expr": "$.edata.props[*]" - }, - { - "type": "path", - "name": "edata_state", - "expr": "$.edata.state" - }, - { - "type": "path", - "name": "edata_prevstate", - "expr": "$.edata.prevstate" - }, - { - "type": "path", - "name": "edata_size", - "expr": "$.edata.size" - }, - { - "type": "path", - "name": "edata_filters_dialcodes", - "expr": "$.edata.filters.dialcodes" - }, - { - "type": "path", - "name": "edata_topn_identifier", - "expr": "$.edata.topn[*].identifier" - }, - { - "type": "path", - "name": "edata_visits_objid", - "expr": "$.edata.visits[*].objid" - }, - { - "type": "path", - "name": "edata_visits_objtype", - "expr": "$.edata.visits[*].objtype" - }, - { - "type": "path", - "name": "edata_visits_objver", - "expr": "$.edata.visits[*].objver" - }, - { - "type": "path", - "name": "edata_visits_index", - "expr": "$.edata.visits[*].index" - }, - { - "type": "path", - "name": "device_loc_state", - "expr": "$.devicedata.state" - }, - { - "type": "path", - "name": "device_loc_state_code", - "expr": "$.devicedata.statecode" - }, - { - "type": "path", - "name": "device_loc_city", - "expr": "$.devicedata.city" - }, - { - "type": "path", - "name": "device_loc_country_code", - "expr": "$.devicedata.countrycode" - }, - { - "type": "path", - "name": "device_loc_country", - "expr": "$.devicedata.country" - }, - { - "type": "path", - "name": "device_os", - "expr": "$.devicedata.devicespec.os" - }, - { - "type": "path", - "name": "device_make", - "expr": "$.devicedata.devicespec.make" - }, - { - "type": "path", - "name": "device_id", - "expr": "$.devicedata.devicespec.id" - }, - { - "type": "path", - "name": "device_mem", - "expr": "$.devicedata.devicespec.mem" - }, - { - "type": "path", - "name": "device_idisk", - "expr": "$.devicedata.devicespec.idisk" - }, - { - "type": "path", - "name": "device_edisk", - "expr": "$.devicedata.devicespec.edisk" - }, - { - "type": "path", - "name": "device_scrn", - "expr": "$.devicedata.devicespec.scrn" - }, - { - "type": "path", - "name": "device_camera", - "expr": "$.devicedata.devicespec.camera" - }, - { - "type": "path", - "name": "device_cpu", - "expr": "$.devicedata.devicespec.cpu" - }, - { - "type": "path", - "name": "device_sims", - "expr": "$.devicedata.devicespec.sims" - }, - { - "type": "path", - "name": "device_uaspec_agent", - "expr": "$.devicedata.uaspec.agent" - }, - { - "type": "path", - "name": "device_uaspec_ver", - "expr": "$.devicedata.uaspec.ver" - }, - { - "type": "path", - "name": "device_uaspec_system", - "expr": "$.devicedata.uaspec.system" - }, - { - "type": "path", - "name": "device_uaspec_platform", - "expr": "$.devicedata.uaspec.platform" - }, - { - "type": "path", - "name": "device_uaspec_raw", - "expr": "$.devicedata.uaspec.raw" - }, - { - "type": "path", - "name": "device_first_access", - "expr": "$.devicedata.firstaccess" - }, - { - "type": "path", - "name": "device_loc_state_custom_code", - "expr": "$.devicedata.statecustomcode" - }, - { - "type": "path", - "name": "device_loc_state_custom_name", - "expr": "$.devicedata.statecustomname" - }, - { - "type": "path", - "name": "device_loc_district", - "expr": "$.devicedata.districtcustom" - }, - { - "type": "path", - "name": "user_declared_state", - "expr": "$.devicedata.userdeclared.state" - }, - { - "type": "path", - "name": "user_declared_district", - "expr": "$.devicedata.userdeclared.district" - }, - { - "type": "path", - "name": "derived_loc_state", - "expr": "$.derivedlocationdata.state" - }, - { - "type": "path", - "name": "derived_loc_district", - "expr": "$.derivedlocationdata.district" - }, - { - "type": "path", - "name": "derived_loc_from", - "expr": "$.derivedlocationdata.from" - }, - { - "type": "path", - "name": "content_name", - "expr": "$.contentdata.name" - }, - { - "type": "path", - "name": "content_object_type", - "expr": "$.contentdata.objecttype" - }, - { - "type": "path", - "name": "content_type", - "expr": "$.contentdata.contenttype" - }, - { - "type": "path", - "name": "content_media_type", - "expr": "$.contentdata.mediatype" - }, - { - "type": "path", - "name": "content_language", - "expr": "$.contentdata.language[*]" - }, - { - "type": "path", - "name": "content_medium", - "expr": "$.contentdata.medium[*]" - }, - { - "type": "path", - "name": "content_gradelevel", - "expr": "$.contentdata.gradelevel[*]" - }, - { - "type": "path", - "name": "content_subjects", - "expr": "$.contentdata.subject[*]" - }, - { - "type": "path", - "name": "content_mimetype", - "expr": "$.contentdata.mimetype" - }, - { - "type": "path", - "name": "content_framework", - "expr": "$.contentdata.framework" - }, - { - "type": "path", - "name": "content_board", - "expr": "$.contentdata.board" - }, - { - "type": "path", - "name": "content_status", - "expr": "$.contentdata.status" - }, - { - "type": "path", - "name": "content_version", - "expr": "$.contentdata.pkgversion" - }, - { - "type": "path", - "name": "content_last_submitted_on", - "expr": "$.contentdata.lastsubmittedon" - }, - { - "type": "path", - "name": "content_last_published_on", - "expr": "$.contentdata.lastpublishedon" - }, - { - "type": "path", - "name": "content_last_updated_on", - "expr": "$.contentdata.lastupdatedon" - }, - { - "type": "path", - "name": "content_created_by", - "expr": "$.contentdata.createdby" - }, - { - "type": "path", - "name": "content_created_for", - "expr": "$.contentdata.createdfor" - }, - { - "type": "path", - "name": "collection_name", - "expr": "$.collectiondata.name" - }, - { - "type": "path", - "name": "collection_object_type", - "expr": "$.collectiondata.objecttype" - }, - { - "type": "path", - "name": "collection_type", - "expr": "$.collectiondata.contenttype" - }, - { - "type": "path", - "name": "collection_media_type", - "expr": "$.collectiondata.mediatype" - }, - { - "type": "path", - "name": "collection_language", - "expr": "$.collectiondata.language[*]" - }, - { - "type": "path", - "name": "collection_medium", - "expr": "$.collectiondata.medium[*]" - }, - { - "type": "path", - "name": "collection_gradelevel", - "expr": "$.collectiondata.gradelevel[*]" - }, - { - "type": "path", - "name": "collection_subjects", - "expr": "$.collectiondata.subject[*]" - }, - { - "type": "path", - "name": "collection_mimetype", - "expr": "$.collectiondata.mimetype" - }, - { - "type": "path", - "name": "collection_framework", - "expr": "$.collectiondata.framework" - }, - { - "type": "path", - "name": "collection_board", - "expr": "$.collectiondata.board" - }, - { - "type": "path", - "name": "collection_status", - "expr": "$.collectiondata.status" - }, - { - "type": "path", - "name": "collection_version", - "expr": "$.collectiondata.pkgversion" - }, - { - "type": "path", - "name": "collection_last_submitted_on", - "expr": "$.collectiondata.lastsubmittedon" - }, - { - "type": "path", - "name": "collection_last_published_on", - "expr": "$.collectiondata.lastpublishedon" - }, - { - "type": "path", - "name": "collection_last_updated_on", - "expr": "$.collectiondata.lastupdatedon" - }, - { - "type": "path", - "name": "collection_created_by", - "expr": "$.collectiondata.createdby" - }, - { - "type": "path", - "name": "collection_created_for", - "expr": "$.collectiondata.createdfor" - }, - { - "type": "path", - "name": "user_grade_list", - "expr": "$.userdata.gradelist[*]" - }, - { - "type": "path", - "name": "user_language_list", - "expr": "$.userdata.languagelist[*]" - }, - { - "type": "path", - "name": "user_subject_list", - "expr": "$.userdata.subjectlist[*]" - }, - { - "type": "path", - "name": "user_type", - "expr": "$.userdata.usertype" - }, - { - "type": "path", - "name": "user_roles", - "expr": "$.userdata.roles[*]" - }, - { - "type": "path", - "name": "user_loc_state", - "expr": "$.userdata.state" - }, - { - "type": "path", - "name": "user_loc_district", - "expr": "$.userdata.district" - }, - { - "type": "path", - "name": "user_signin_type", - "expr": "$.userdata.usersignintype" - }, - { - "type": "path", - "name": "user_login_type", - "expr": "$.userdata.userlogintype" - }, - { - "type": "path", - "name": "dialcode_channel", - "expr": "$.dialcodedata.channel" - }, - { - "type": "path", - "name": "dialcode_batchcode", - "expr": "$.dialcodedata.batchcode" - }, - { - "type": "path", - "name": "dialcode_publisher", - "expr": "$.dialcodedata.publisher" - }, - { - "type": "path", - "name": "dialcode_generated_on", - "expr": "$.dialcodedata.generatedon" - }, - { - "type": "path", - "name": "dialcode_published_on", - "expr": "$.dialcodedata.publishedon" - }, - { - "type": "path", - "name": "dialcode_object_type", - "expr": "$.dialcodedata.objecttype" - }, - { - "type": "path", - "name": "device_loc_iso_state_code", - "expr": "$.devicedata.iso3166statecode" - } - ] - }, - "dimensionsSpec": { - "dimensions": [ - { - "type": "string", - "name": "eid" - }, - { - "type": "string", - "name": "mid" - }, - { - "type": "long", - "name": "ets" - }, - { - "type": "string", - "name": "@timestamp" - }, - { - "type": "string", - "name": "actor_id" - }, - { - "type": "string", - "name": "actor_type" - }, - { - "type": "string", - "name": "context_channel" - }, - { - "type": "string", - "name": "context_pdata_id" - }, - { - "type": "string", - "name": "context_pdata_pid" - }, - { - "type": "string", - "name": "context_pdata_ver" - }, - { - "type": "string", - "name": "context_env" - }, - { - "type": "string", - "name": "context_sid" - }, - { - "type": "string", - "name": "context_did" - }, - { - "name": "context_cdata_type" - }, - { - "name": "context_cdata_id" - }, - { - "type": "string", - "name": "context_rollup_l1" - }, - { - "type": "string", - "name": "context_rollup_l2" - }, - { - "type": "string", - "name": "context_rollup_l3" - }, - { - "type": "string", - "name": "context_rollup_l4" - }, - { - "type": "string", - "name": "object_id" - }, - { - "type": "string", - "name": "object_type" - }, - { - "type": "string", - "name": "object_version" - }, - { - "type": "string", - "name": "object_rollup_l1" - }, - { - "type": "string", - "name": "object_rollup_l2" - }, - { - "type": "string", - "name": "object_rollup_l3" - }, - { - "type": "string", - "name": "object_rollup_l4" - }, - { - "name": "tags" - }, - { - "type": "string", - "name": "edata_type" - }, - { - "type": "string", - "name": "edata_subtype" - }, - { - "type": "string", - "name": "edata_mode" - }, - { - "type": "string", - "name": "edata_pageid" - }, - { - "type": "string", - "name": "edata_uri" - }, - { - "type": "string", - "name": "edata_id" - }, - { - "type": "double", - "name": "edata_duration" - }, - { - "type": "long", - "name": "edata_index" - }, - { - "type": "string", - "name": "edata_pass" - }, - { - "type": "double", - "name": "edata_score" - }, - { - "name": "edata_resvalues" - }, - { - "type": "string", - "name": "edata_item_id" - }, - { - "type": "string", - "name": "edata_item_title" - }, - { - "type": "double", - "name": "edata_item_maxscore" - }, - { - "type": "string", - "name": "edata_target_id" - }, - { - "type": "string", - "name": "edata_target_type" - }, - { - "type": "long", - "name": "edata_rating" - }, - { - "type": "string", - "name": "edata_comments" - }, - { - "type": "string", - "name": "edata_commentid" - }, - { - "type": "string", - "name": "edata_commenttxt" - }, - { - "type": "string", - "name": "edata_dir" - }, - { - "type": "string", - "name": "edata_items_id" - }, - { - "type": "string", - "name": "edata_items_type" - }, - { - "type": "string", - "name": "edata_items_origin_id" - }, - { - "type": "string", - "name": "edata_items_origin_type" - }, - { - "type": "string", - "name": "edata_items_to_id" - }, - { - "type": "string", - "name": "edata_items_to_type" - }, - { - "type": "string", - "name": "edata_plugin_id" - }, - { - "type": "string", - "name": "edata_plugin_ver" - }, - { - "type": "string", - "name": "edata_plugin_category" - }, - { - "name": "edata_props" - }, - { - "type": "string", - "name": "edata_state" - }, - { - "type": "string", - "name": "edata_prevstate" - }, - { - "type": "long", - "name": "edata_size" - }, - { - "name": "edata_filters_dialcodes" - }, - { - "type": "string", - "name": "edata_topn_identifier" - }, - { - "type": "string", - "name": "edata_visits_objid" - }, - { - "type": "string", - "name": "edata_visits_objtype" - }, - { - "type": "string", - "name": "edata_visits_objver" - }, - { - "type": "string", - "name": "edata_visits_index" - }, - { - "type": "string", - "name": "device_loc_state" - }, - { - "type": "string", - "name": "device_loc_state_code" - }, - { - "type": "string", - "name": "device_loc_city" - }, - { - "type": "string", - "name": "device_loc_country_code" - }, - { - "type": "string", - "name": "device_loc_country" - }, - { - "type": "string", - "name": "device_os" - }, - { - "type": "string", - "name": "device_make" - }, - { - "type": "string", - "name": "device_id" - }, - { - "type": "long", - "name": "device_mem" - }, - { - "type": "string", - "name": "device_idisk" - }, - { - "type": "string", - "name": "device_edisk" - }, - { - "type": "string", - "name": "device_scrn" - }, - { - "type": "string", - "name": "device_camera" - }, - { - "type": "string", - "name": "device_cpu" - }, - { - "type": "long", - "name": "device_sims" - }, - { - "type": "string", - "name": "device_uaspec_agent" - }, - { - "type": "string", - "name": "device_uaspec_ver" - }, - { - "type": "string", - "name": "device_uaspec_system" - }, - { - "type": "string", - "name": "device_uaspec_platform" - }, - { - "type": "string", - "name": "device_uaspec_raw" - }, - { - "type": "long", - "name": "device_first_access" - }, - { - "type": "string", - "name": "device_loc_state_custom_code" - }, - { - "type": "string", - "name": "device_loc_state_custom_name" - }, - { - "type": "string", - "name": "device_loc_district" - }, - { - "type": "string", - "name": "user_declared_state" - }, - { - "type": "string", - "name": "user_declared_district" - }, - { - "type": "string", - "name": "derived_loc_state" - }, - { - "type": "string", - "name": "derived_loc_district" - }, - { - "type": "string", - "name": "derived_loc_from" - }, - { - "type": "string", - "name": "content_name" - }, - { - "type": "string", - "name": "content_object_type" - }, - { - "type": "string", - "name": "content_type" - }, - { - "type": "string", - "name": "content_media_type" - }, - { - "name": "content_language" - }, - { - "name": "content_medium" - }, - { - "name": "content_gradelevel" - }, - { - "name": "content_subjects" - }, - { - "type": "string", - "name": "content_mimetype" - }, - { - "type": "string", - "name": "content_framework" - }, - { - "type": "string", - "name": "content_board" - }, - { - "type": "string", - "name": "content_status" - }, - { - "type": "double", - "name": "content_version" - }, - { - "type": "long", - "name": "content_last_submitted_on" - }, - { - "type": "long", - "name": "content_last_published_on" - }, - { - "type": "long", - "name": "content_last_updated_on" - }, - { - "type": "string", - "name": "content_created_by" - }, - { - "name": "content_created_for" - }, - { - "type": "string", - "name": "collection_name" - }, - { - "type": "string", - "name": "collection_object_type" - }, - { - "type": "string", - "name": "collection_type" - }, - { - "type": "string", - "name": "collection_media_type" - }, - { - "name": "collection_language" - }, - { - "name": "collection_medium" - }, - { - "name": "collection_gradelevel" - }, - { - "name": "collection_subjects" - }, - { - "type": "string", - "name": "collection_mimetype" - }, - { - "type": "string", - "name": "collection_framework" - }, - { - "type": "string", - "name": "collection_board" - }, - { - "type": "string", - "name": "collection_status" - }, - { - "type": "double", - "name": "collection_version" - }, - { - "type": "long", - "name": "collection_last_submitted_on" - }, - { - "type": "long", - "name": "collection_last_published_on" - }, - { - "type": "long", - "name": "collection_last_updated_on" - }, - { - "type": "string", - "name": "collection_created_by" - }, - { - "name": "collection_created_for" - }, - { - "name": "user_grade_list" - }, - { - "name": "user_language_list" - }, - { - "name": "user_subject_list" - }, - { - "type": "string", - "name": "user_type" - }, - { - "name": "user_roles" - }, - { - "type": "string", - "name": "user_loc_state" - }, - { - "type": "string", - "name": "user_loc_district" - }, - { - "type": "string", - "name": "user_signin_type" - }, - { - "type": "string", - "name": "user_login_type" - }, - { - "type": "string", - "name": "dialcode_channel" - }, - { - "type": "string", - "name": "dialcode_batchcode" - }, - { - "type": "string", - "name": "dialcode_publisher" - }, - { - "type": "long", - "name": "dialcode_generated_on" - }, - { - "type": "long", - "name": "dialcode_published_on" - }, - { - "type": "string", - "name": "dialcode_object_type" - }, - { - "type": "string", - "name": "device_loc_iso_state_code" - } - ], - "dimensionsExclusions": [] - }, - "timestampSpec": { - "column": "syncts", - "format": "auto" - } - } - }, - "metricsSpec": [], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "day", - "queryGranularity": "none", - "rollup": false - } - }, - "ioConfig": { - "topic": "{{env}}.events.telemetry", - "consumerProperties": { - "bootstrap.servers": "{{kafka_brokers}}" - }, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT14400S", - "useEarliestOffset": false, - "completionTimeout" : "PT2H" - }, - "tuningConfig": { - "type": "kafka", - "reportParseExceptions": false - } -} diff --git a/ansible/roles/druid-summary-monthly-ingestion/defaults/main.yml b/ansible/roles/druid-summary-monthly-ingestion/defaults/main.yml new file mode 100644 index 0000000000..549881ea6d --- /dev/null +++ b/ansible/roles/druid-summary-monthly-ingestion/defaults/main.yml @@ -0,0 +1,14 @@ +remote_user: druid +druid_overlord_host: "{{ groups['rollup-overlord'][0] }}" +druid_overlord_port: 8090 +druid_home: /home/druid + +summary_monthly: + base_data_source_name: "summary-rollup-syncts" + new_data_source_name: "summary-monthly-rollup-syncts" + no_batch_parallel_tasks: 1 + +supervisor_summary_spec_url: "http://{{ druid_overlord_host }}:{{ druid_overlord_port }}/druid/indexer/v1/supervisor/{{ summary_monthly.base_data_source_name }}" + +druid_schema: + - summary_monthly_batch diff --git a/ansible/roles/druid-summary-monthly-ingestion/tasks/main.yml b/ansible/roles/druid-summary-monthly-ingestion/tasks/main.yml new file mode 100644 index 0000000000..908e7bc245 --- /dev/null +++ b/ansible/roles/druid-summary-monthly-ingestion/tasks/main.yml @@ -0,0 +1,31 @@ +- name: Get the summary supervisor spec + uri: + url: "{{ supervisor_summary_spec_url }}" + method: GET + return_content: yes + register : supervisor_spec + +- set_fact: + dimensions_spec: "{{ supervisor_spec.json.dataSchema.parser.parseSpec.dimensionsSpec | to_json }}" + +- name : Retrive Interval Month Range + shell : echo $(date -d"1 $(date -d'last month' +%b)" +%F)/$(date -d"-0 day 1 $(date +%b)" +%F ) + register : output + +- set_fact: + summary_monthly_interval: "{{ output.stdout }}" + +- name: Copy the schema file + template: src="{{item}}" dest="{{druid_home}}/{{item}}.json" + become_user: "{{remote_user}}" + with_items: "{{druid_schema}}" + +- name: Start batch ingestion task + shell: 'curl -X POST -H "Content-Type:application/json" -d @{{druid_home}}/{{item}}.json http://{{druid_overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/task' + with_items: "{{druid_schema}}" + become_user: "{{remote_user}}" + + + + + diff --git a/ansible/roles/druid-summary-monthly-ingestion/templates/summary_monthly_batch b/ansible/roles/druid-summary-monthly-ingestion/templates/summary_monthly_batch new file mode 100644 index 0000000000..e2049e23f4 --- /dev/null +++ b/ansible/roles/druid-summary-monthly-ingestion/templates/summary_monthly_batch @@ -0,0 +1,57 @@ +{ + "type": "index_parallel", + "spec": { + "ioConfig": { + "type": "index_parallel", + "inputSource": { + "type": "druid", + "dataSource": "{{ summary_monthly.base_data_source_name }}", + "interval": "{{ summary_monthly_interval }}", + "metrics": [ + "total_count", + "total_interactions", + "total_timespent" + ] + }, + "appendToExisting": false + }, + "tuningConfig": { + "type": "index_parallel", + "targetPartitionSize": 5000000, + "maxRowsInMemory": 500000, + "forceExtendableShardSpecs": true, + "maxNumConcurrentSubTasks": "{{ summary_monthly.no_batch_parallel_tasks }}" + }, + "dataSchema": { + "dataSource": "{{ summary_monthly.new_data_source_name }}", + "dimensionsSpec": {{ dimensions_spec }}, + "timestampSpec": { + "column": "syncts", + "format": "auto" + }, + "metricsSpec": [ + { + "type": "longSum", + "name": "total_count", + "fieldName": "total_count" + }, + { + "type": "longSum", + "name": "total_interactions", + "fieldName": "total_interactions" + }, + { + "type": "doubleSum", + "name": "total_time_spent", + "fieldName": "total_time_spent" + } + ] + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "month", + "queryGranularity": "none", + "rollup": true + } + } + } +} diff --git a/ansible/roles/ep_logstash/defaults/main.yml b/ansible/roles/ep_logstash/defaults/main.yml index 0df77cdccc..ad18f4e8d7 100644 --- a/ansible/roles/ep_logstash/defaults/main.yml +++ b/ansible/roles/ep_logstash/defaults/main.yml @@ -11,10 +11,13 @@ logstash_conf: es_mapping_file: - "telemetry_mapping.json" - "crash-log-mapping.json" + - "backend_telemetry_mapping.json" kafka_topic_prefix: "{{ env }}" error_es_index: "telemetry-error-events" log_es_index: "telemetry-log-events" telemetry_mapping_file_path: "{{ logstash_home }}/mappings/telemetry_mapping.json" +backend_log_es_index: "dp-backend-events" +backend_telemetry_mapping_file_path: "{{ logstash_home }}/mappings/backend_telemetry_mapping.json" apt_cache_valid_time: 3600 zookeepers: "{{ groups['processing-cluster-zookeepers']|join(':2181,') }}:2181" @@ -31,15 +34,10 @@ private_exhaust_topic: "telemetry.private_exhaust" public_exhaust_topic: "telemetry.public_exhaust" public_data_exhaust_dir_prefix: 4208ab995984d222b59299e5103d350a842d8d41 -#Device register API metrics -device_register_api_metrics_log_path: "/mount/data/analytics/logs/api-service/api-metrics.log" -api_job_log_path: "/mount/data/analytics/logs/api-service/joblog.log" -device_log_path: "/mount/data/analytics/logs/api-service/device-profile.log" - # client API error log -crash_log_path: "/mount/data/analytics/logs/api-service/crash-logs.log" logger_es: "{{ groups['log-es']|join(':9200,') }}" logstash_version: '6.x' crash_log_mapping_file_path: "{{ logstash_home }}/mappings/crash-log-mapping.json" crash_log_index : "crash-logs" +send_logs_to_graylog: false diff --git a/ansible/roles/ep_logstash/tasks/common.yml b/ansible/roles/ep_logstash/tasks/common.yml index 33cc5c022f..6ab285a8c5 100644 --- a/ansible/roles/ep_logstash/tasks/common.yml +++ b/ansible/roles/ep_logstash/tasks/common.yml @@ -74,11 +74,11 @@ become: yes ignore_errors: yes - - name: Copy logstash init.d file copy: src: logstash.sh - dest: /etc/init.d/logstash mode=755 + dest: /etc/init.d/logstash + mode: 755 become: true - name: Install plugins diff --git a/ansible/roles/ep_logstash/templates/logstash.conf.indexer.device_api_metrics b/ansible/roles/ep_logstash/templates/logstash.conf.indexer.device_api_metrics deleted file mode 100644 index aea3fa75f2..0000000000 --- a/ansible/roles/ep_logstash/templates/logstash.conf.indexer.device_api_metrics +++ /dev/null @@ -1,69 +0,0 @@ -input { - file { - path => "{{device_register_api_metrics_log_path}}" - start_position => "end" - sincedb_write_interval => 10 - type => "device_register_api_metrics" - } - file { - path => "{{api_job_log_path}}" - start_position => "end" - sincedb_write_interval => 10 - type => "api_job_logs" - } - file { - path => "{{crash_log_path}}" - codec => "json" - start_position => "end" - sincedb_write_interval => 10 - type => "crash_logs" - } - file { - path => "{{device_log_path}}" - start_position => "end" - sincedb_write_interval => 10 - type => "device_logs" - } -} -output { - if [type] == "device_logs" { - kafka { - bootstrap_servers => "{{kafka_brokers}}" - topic_id => "{{kafka_topic_prefix}}.events.deviceprofile" - codec => plain { - format => "%{message}" - } - } - } - - if [type] == "device_register_api_metrics" { - kafka { - bootstrap_servers => "{{kafka_brokers}}" - topic_id => "{{kafka_topic_prefix}}.pipeline_metrics" - codec => line { - format => "%{message}" - } - } - } - - if [type] == "api_job_logs" { - kafka { - bootstrap_servers => "{{kafka_brokers}}" - topic_id => "{{kafka_topic_prefix}}.telemetry.log" - codec => line { - format => "%{message}" - } - } - } - - if [type] == "crash_logs" { - elasticsearch { - hosts => ["{{logger_es}}"] - template => "{{crash_log_mapping_file_path}}" - template_name => "crash-log" - index => "{{crash_log_index}}" - manage_template => true - document_type => "logs" - } - } -} diff --git a/ansible/roles/ep_logstash/templates/logstash.conf.indexer.log_error.events b/ansible/roles/ep_logstash/templates/logstash.conf.indexer.log_error.events index 949f3f68de..b1fa9718f3 100644 --- a/ansible/roles/ep_logstash/templates/logstash.conf.indexer.log_error.events +++ b/ansible/roles/ep_logstash/templates/logstash.conf.indexer.log_error.events @@ -1,11 +1,12 @@ +{% if not send_logs_to_graylog %} # Logstash configuration # Kafka(telemetry.log) -> Logstash -> Elasticsearch pipeline index error type events into(telemetry-error-events) and log type events into(telemetry-log-index). input { kafka { bootstrap_servers => "{{bootstrap_server}}" - topics => ["{{kafka_topic_prefix}}.telemetry.log"] - group_id => "{{kafka_topic_prefix}}.telemetry.log" + topics => ["{{kafka_topic_prefix}}.druid.events.log", "{{kafka_topic_prefix}}.druid.events.error"] + group_id => "{{kafka_topic_prefix}}.telemetry-log.indexer" auto_offset_reset => "latest" codec => "json" consumer_threads => 1 @@ -34,9 +35,16 @@ output { manage_template => true document_type => "events" } + }else if "JOB_" in [eid] { + elasticsearch { + codec => "json" + hosts => ["{{logger_es}}"] + template => "{{backend_telemetry_mapping_file_path}}" + template_name => "dp-backend" + index => "{{backend_log_es_index}}-%{+YYYY.MM.ww}" + manage_template => true + document_type => "events" + } } } - - - - +{% endif %} \ No newline at end of file diff --git a/ansible/roles/ep_logstash/templates/mapping/backend_telemetry_mapping.json b/ansible/roles/ep_logstash/templates/mapping/backend_telemetry_mapping.json new file mode 100644 index 0000000000..e376c0d6b0 --- /dev/null +++ b/ansible/roles/ep_logstash/templates/mapping/backend_telemetry_mapping.json @@ -0,0 +1,138 @@ +{ + "template" : "backend", + "index_patterns" : "backend-*", + "settings" : { + "number_of_shards" : 5 + }, + "mappings" : { + "events" : { + "dynamic": false, + "properties": { + "@timestamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "@version": { + "type": "keyword" + }, + "eid": { + "type": "keyword" + }, + "ets": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "mid": { + "type": "keyword" + }, + "ts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "ver": { + "type": "keyword" + }, + "actor": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "edata": { + "properties": { + "data": { + "type": "object", + "properties": { + "config": { + "properties": { + "search": { + "type": "object", + }, + "model": { + "type": "keyword" + }, + "modelParams": { + "type": "object", + }, + "output": { + "type": "object" + }, + "parallelization": { + "type": "long" + }, + "appName": { + "type": "keyword" + }, + "deviceMapping": { + "type": "boolean" + } + } + }, + "model": { + "type": "keyword" + }, + "date": { + "type": "keyword" + }, + "inputEvents": { + "type": "long" + }, + "outputEvents": { + "type": "long" + }, + "timeTaken": { + "type": "double" + } + } + }, + "level": { + "type": "keyword" + }, + "message": { + "type": "keyword" + }, + "class": { + "type": "keyword" + }, + "status": { + "type": "keyword" + } + } + }, + "context": { + "properties": { + "channel": { + "type": "keyword" + }, + "env": { + "type": "keyword" + }, + "sid": { + "type": "keyword" + }, + "did": { + "type": "keyword" + }, + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "pid": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/es6/LICENSE b/ansible/roles/es6/LICENSE deleted file mode 100644 index 0455cacdf6..0000000000 --- a/ansible/roles/es6/LICENSE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright (c) 2012-2016 Elasticsearch - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/ansible/roles/es6/README.md b/ansible/roles/es6/README.md deleted file mode 100644 index 85f97acf43..0000000000 --- a/ansible/roles/es6/README.md +++ /dev/null @@ -1,415 +0,0 @@ -# ansible-elasticsearch -[![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/) - -**THIS ROLE IS FOR 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.** - -Ansible role for 6.x/5.x Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: - -* Ubuntu 14.04/16.04 -* Debian 8 -* Centos 7 - -The latest Elasticsearch versions of 6.x are actively tested. **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.** - -##### Dependency -This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine. - -## Usage - -Create your Ansible playbook with your own tasks, and include the role elasticsearch. You will have to have this repository accessible within the context of playbook, e.g. - -e.g. - -``` -cd /my/repos/ -git clone https://github.com/elastic/ansible-elasticsearch.git -cd /my/ansible/playbook -mkdir -p roles -ln -s /my/repos/ansible-elasticsearch ./roles/elasticsearch -``` - -Then create your playbook yaml adding the role elasticsearch. By default, the user is only required to specify a unique es_instance_name per role application. This should be unique per node. -The application of the elasticsearch role results in the installation of a node on a host. - -The simplest configuration therefore consists of: - -``` -- name: Simple Example - hosts: localhost - roles: - - { role: elasticsearch, es_instance_name: "node1" } - vars: -``` - -The above installs a single node 'node1' on the hosts 'localhost'. - -This role also uses [Ansible tags](http://docs.ansible.com/ansible/playbooks_tags.html). Run your playbook with the `--list-tasks` flag for more information. - -### Basic Elasticsearch Configuration - -All Elasticsearch configuration parameters are supported. This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file. -The use of a map ensures the Ansible playbook does not need to be updated to reflect new/deprecated/plugin configuration parameters. - -In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation. These can be found in the role's defaults/main.yml file. - -The following illustrates applying configuration parameters to an Elasticsearch instance. By default, Elasticsearch 5.1.2is installed. - -``` -- name: Elasticsearch with custom configuration - hosts: localhost - roles: - #expand to all available parameters - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", - es_config: { - node.name: "node1", - cluster.name: "custom-cluster", - discovery.zen.ping.unicast.hosts: "localhost:9301", - http.port: 9201, - transport.tcp.port: 9301, - node.data: false, - node.master: true, - bootstrap.memory_lock: true, - } - } - vars: - es_scripts: false - es_templates: false - es_version_lock: false - es_heap_size: 1g - es_api_port: 9201 -``` - -Whilst the role installs Elasticsearch with the default configuration parameters, the following should be configured to ensure a cluster successfully forms: - -* ```es_config['http.port']``` - the http port for the node -* ```es_config['transport.tcp.port']``` - the transport port for the node -* ```es_config['discovery.zen.ping.unicast.hosts']``` - the unicast discovery list, in the comma separated format ```":,:"``` (typically the clusters dedicated masters) -* ```es_config['network.host']``` - sets both network.bind_host and network.publish_host to the same host value. The network.bind_host setting allows to control the host different network components will bind on. - -The network.publish_host setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. - -See https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html for further details on default binding behaviour and available options. -The role makes no attempt to enforce the setting of these are requires users to specify them appropriately. IT is recommended master nodes are listed and thus deployed first where possible. - -A more complex example: - -``` -- name: Elasticsearch with custom configuration - hosts: localhost - roles: - #expand to all available parameters - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", - es_config: { - node.name: "node1", - cluster.name: "custom-cluster", - discovery.zen.ping.unicast.hosts: "localhost:9301", - http.port: 9201, - transport.tcp.port: 9301, - node.data: false, - node.master: true, - bootstrap.memory_lock: true, - } - } - vars: - es_scripts: false - es_templates: false - es_version_lock: false - es_heap_size: 1g - es_start_service: false - es_plugins_reinstall: false - es_api_port: 9201 - es_plugins: - - plugin: ingest-geoip - proxy_host: proxy.example.com - proxy_port: 8080 -``` - -#### Important Note - -**The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates and to check the NODE IS ACTIVE. These default to "localhost" and 9200 respectively. -If the node is deployed to bind on either a different host or port, these must be changed.** - -### Multi Node Server Installations - -The application of the elasticsearch role results in the installation of a node on a host. Specifying the role multiple times for a host therefore results in the installation of multiple nodes for the host. - -An example of a two server deployment is shown below. The first server holds the master and is thus declared first. Whilst not mandatory, this is recommended in any multi node cluster configuration. The second server hosts two data nodes. - -**Note the structure of the below playbook for the data nodes. Whilst a more succinct structures are possible which allow the same role to be applied to a host multiple times, we have found the below structure to be the most reliable with respect to var behaviour. This is the tested approach.** - -``` -- hosts: master_nodes - roles: - - { role: elasticsearch, es_instance_name: "node1", es_heap_size: "1g", - es_config: { - cluster.name: "test-cluster", - discovery.zen.ping.unicast.hosts: "elastic02:9300", - http.port: 9200, - transport.tcp.port: 9300, - node.data: false, - node.master: true, - bootstrap.memory_lock: false, - } - } - vars: - es_scripts: false - es_templates: false - es_version_lock: false - ansible_user: ansible - es_plugins: - - plugin: ingest-geoip - - - -- hosts: data_nodes - roles: - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch", - es_config: { - discovery.zen.ping.unicast.hosts: "elastic02:9300", - http.port: 9200, - transport.tcp.port: 9300, - node.data: true, - node.master: false, - bootstrap.memory_lock: false, - cluster.name: "test-cluster" - } - } - vars: - es_scripts: false - es_templates: false - es_version_lock: false - ansible_user: ansible - es_api_port: 9200 - es_plugins: - - plugin: ingest-geoip - - -- hosts: data_nodes - roles: - - { role: elasticsearch, es_instance_name: "node2", es_api_port:9201, - es_config: { - discovery.zen.ping.unicast.hosts: "elastic02:9300", - http.port: 9201, - transport.tcp.port: 9301, - node.data: true, - node.master: false, - bootstrap.memory_lock: false, - cluster.name: "test-cluster", - } - } - vars: - es_scripts: false - es_templates: false - es_version_lock: false - es_api_port: 9201 - ansible_user: ansible - es_plugins: - - plugin: ingest-geoip - -``` - -Parameters can additionally be assigned to hosts using the inventory file if desired. - -Make sure your hosts are defined in your ```inventory``` file with the appropriate ```ansible_ssh_host```, ```ansible_ssh_user``` and ```ansible_ssh_private_key_file``` values. - -Then run it: - -``` -ansible-playbook -i hosts ./your-playbook.yml -``` - -### Installing X-Pack Features - -X-Pack features, such as Security, are supported. This feature is currently experimental. To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`. - -The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security","ml"] - -The following additional parameters allow X-Pack to be configured: - -* ```es_message_auth_file``` System Key field to allow message authentication. This file should be placed in the 'files' directory. -* ```es_xpack_custom_url``` Url from which X-Pack can be downloaded. This can be used for installations in isolated environments where the elastic.co repo is not accessible. e.g. ```es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-5.5.1.zip"``` -* ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html) - - -``` -es_role_mapping: - power_user: - - "cn=admins,dc=example,dc=com" - user: - - "cn=users,dc=example,dc=com" - - "cn=admins,dc=example,dc=com" -``` - -* ```es_users``` - Users can be declared here as yml. Two sub keys 'native' and 'file' determine the realm under which realm the user is created. Beneath each of these keys users should be declared as yml entries. e.g. - -``` -es_users: - native: - kibana4_server: - password: changeMe - roles: - - kibana4_server - file: - es_admin: - password: changeMe - roles: - - admin - testUser: - password: changeMeAlso! - roles: - - power_user - - user -``` - - -* ```es_roles``` - Elasticsearch roles can be declared here as yml. Two sub keys 'native' and 'file' determine how the role is created i.e. either through a file or http(native) call. Beneath each key list the roles with appropriate permissions, using the file based format described [here] (https://www.elastic.co/guide/en/x-pack/current/file-realm.html) e.g. - -``` -es_roles: - file: - admin: - cluster: - - all - indices: - - names: '*' - privileges: - - all - power_user: - cluster: - - monitor - indices: - - names: '*' - privileges: - - all - user: - indices: - - names: '*' - privileges: - - read - kibana4_server: - cluster: - - monitor - indices: - - names: '.kibana' - privileges: - - all - native: - logstash: - cluster: - - manage_index_templates - indices: - - names: 'logstash-*' - privileges: - - write - - delete - - create_index -``` - -* ```es_xpack_license``` - X-Pack license. The license is a json blob. Set the variable directly (possibly protected by Ansible vault) or from a file in the Ansible project on the control machine via a lookup: - -``` -es_xpack_license: "{{ lookup('file', playbook_dir + '/files/' + es_cluster_name + '/license.json') }}" -``` - -X-Pack configuration parameters can be added to the elasticsearch.yml file using the normal `es_config` parameter. - -For a full example see [here](https://github.com/elastic/ansible-elasticsearch/blob/master/test/integration/xpack.yml) - -#### Important Note for Native Realm Configuration - -In order for native users and roles to be configured, the role calls the Elasticsearch API. Given security is installed this requires definition of two parameters: - -* ```es_api_basic_auth_username``` - admin username -* ```es_api_basic_auth_password``` - admin password - -These can either be set to a user declared in the file based realm, with admin permissions, or the default "elastic" superuser (default password is changeme). - - -### Additional Configuration - -In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include: - -* ```es_major_version``` Should be consistent with es_version. For versions >= 5.0 and < 6.0 this must be "5.x". For versions >= 6.0 this must be "6.x". -* ```es_version``` (e.g. "6.1.2"). -* ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". -* ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200** -* ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin. -* ```es_api_basic_auth_password``` The password associated with the user declared in `es_api_basic_auth_username` -* ```es_start_service``` (true (default) or false) -* ```es_plugins_reinstall``` (true or false (default) ) -* ```es_plugins``` an array of plugin definitions e.g.: -```yml - es_plugins: - - plugin: ingest-geoip -``` -* ```es_allow_downgrades``` For development purposes only. (true or false (default) ) -* ```es_java_install``` If set to false, Java will not be installed. (true (default) or false) -* ```update_java``` Updates Java to the latest version. (true or false (default)) -* ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Defaults to 262144. -* ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536. -* ```es_max_threads``` the maximum number of threads the process can start. Defaults to 2048 (the minimum required by elasticsearch). -* ```es_debian_startup_timeout``` how long Debian-family SysV init scripts wait for the service to start, in seconds. Defaults to 10 seconds. - -Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full url. This approach should NOT be used for the X-Pack plugin. See X-Pack below for details here. - -If installing Monitoring or Alerting, ensure the license plugin is also specified. Security configuration is currently not supported but planned for later versions. - -* ```es_user``` - defaults to elasticsearch. -* ```es_group``` - defaults to elasticsearch. -* ```es_user_id``` - default is undefined. -* ```es_group_id``` - default is undefined. - -Both ```es_user_id``` and ```es_group_id``` must be set for the user and group ids to be set. - -By default, each node on a host will be installed to use unique pid, plugin, work, data and log directories. These directories are created, using the instance and host name, beneath default locations ] -controlled by the following parameters: - -* ```es_pid_dir``` - defaults to "/var/run/elasticsearch". -* ```es_data_dirs``` - defaults to "/var/lib/elasticsearch". This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2" -* ```es_log_dir``` - defaults to "/var/log/elasticsearch". -* ```es_restart_on_change``` - defaults to true. If false, changes will not result in Elasticsearch being restarted. -* ```es_plugins_reinstall``` - defaults to false. If true, all currently installed plugins will be removed from a node. Listed plugins will then be re-installed. - -This role ships with sample scripts and templates located in the [files/scripts/](files/scripts) and [files/templates/](files/templates) directories, respectively. These variables are used with the Ansible [with_fileglob](http://docs.ansible.com/ansible/playbooks_loops.html#id4) loop. When setting the globs, be sure to use an absolute path. -* ```es_scripts_fileglob``` - defaults to `/files/scripts/`. -* ```es_templates_fileglob``` - defaults to `/files/templates/`. - -### Proxy - -To define proxy globaly, set the following variables: - -* ```es_proxy_host``` - global proxy host -* ```es_proxy_port``` - global proxy port - -To define proxy only for a particular plugin during its installation: - -``` - es_plugins: - - plugin: ingest-geoip - proxy_host: proxy.example.com - proxy_port: 8080 -``` - -> For plugins installation, proxy_host and proxy_port are used first if they are defined and fallback to the global proxy settings if not. The same values are currently used for both the http and https proxy settings. - -## Notes - -* The role assumes the user/group exists on the server. The elasticsearch packages create the default elasticsearch user. If this needs to be changed, ensure the user exists. -* The playbook relies on the inventory_name of each host to ensure its directories are unique -* Changing an instance_name for a role application will result in the installation of a new component. The previous component will remain. -* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test only the latest version of 6.x on -all supported platforms. -* The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and -Elasticsearch restarted where required. -* Systemd is used for Ubuntu versions >= 15, Debian >=8, Centos >=7. All other versions use init for service scripts. -* In order to run x-pack tests a license file with security enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. - -## IMPORTANT NOTES RE PLUGIN MANAGEMENT - -* If the ES version is changed, all plugins will be removed. Those listed in the playbook will be re-installed. This is behaviour is required in ES 6.x. -* If no plugins are listed in the playbook for a node, all currently installed plugins will be removed. -* The role supports automatic detection of differences between installed and listed plugins - installing those listed but not installed, and removing those installed but not listed. Should users wish to re-install plugins they should set es_plugins_reinstall to true. This will cause all currently installed plugins to be removed and those listed to be installed. - -## Questions on Usage - -We welcome questions on how to use the role. However, in order to keep the github issues list focused on "issues" we ask the community to raise questions at https://discuss.elastic.co/c/elasticsearch. This is monitored by the maintainers. diff --git a/ansible/roles/es6/ansible.cfg b/ansible/roles/es6/ansible.cfg deleted file mode 100644 index d9a8c50195..0000000000 --- a/ansible/roles/es6/ansible.cfg +++ /dev/null @@ -1 +0,0 @@ -[defaults] \ No newline at end of file diff --git a/ansible/roles/es6/defaults/main.yml b/ansible/roles/es6/defaults/main.yml deleted file mode 100644 index f0ce74ac3d..0000000000 --- a/ansible/roles/es6/defaults/main.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -es_major_version: "6.x" -es_version: "6.2.3" -es_version_lock: false -es_use_repository: true -es_templates_fileglob: "files/templates/*.json" -es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" -es_apt_url: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main" -es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main" -es_start_service: true -es_java_install: true -update_java: false -es_restart_on_change: true -es_plugins_reinstall: false -es_scripts: false -es_templates: true -es_user: elasticsearch -es_group: elasticsearch -es_config_log4j2: log4j2.properties.j2 -#Need to provide default directories -es_pid_dir: "/var/run/elasticsearch" -es_data_dirs: "/var/lib/elasticsearch" -es_log_dir: "/var/log/elasticsearch" -es_max_open_files: 65536 -es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}" -es_max_map_count: 262144 -es_allow_downgrades: false -es_enable_xpack: false -es_xpack_features: ["alerting","monitoring","graph","ml"] -#These are used for internal operations performed by ansible. -#They do not affect the current configuration -es_api_host: "localhost" -es_api_port: 9200 -es_debian_startup_timeout: 10 - -# Since ansible 2.2 the following variables need to be defined -# to allow the role to be conditionally played with a when condition. -pid_dir: '' -log_dir: '' -conf_dir: '' -data_dirs: '' -# JVM custom parameters -es_jvm_custom_parameters: '' diff --git a/ansible/roles/es6/files/logging/log4j2.properties.custom.j2 b/ansible/roles/es6/files/logging/log4j2.properties.custom.j2 deleted file mode 100644 index 9a2a60f885..0000000000 --- a/ansible/roles/es6/files/logging/log4j2.properties.custom.j2 +++ /dev/null @@ -1,76 +0,0 @@ -#CUSTOM LOG4J FILE - -status = error - -# log action execution errors for easier debugging -logger.action.name = org.elasticsearch.action -logger.action.level = info - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - -appender.rolling.type = RollingFile -appender.rolling.name = rolling -appender.rolling.fileName = ${sys:es.logs}.log -appender.rolling.layout.type = PatternLayout -appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n -appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log -appender.rolling.policies.type = Policies -appender.rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.rolling.policies.time.interval = 1 -appender.rolling.policies.time.modulate = true - -rootLogger.level = debug -rootLogger.appenderRef.console.ref = console -rootLogger.appenderRef.rolling.ref = rolling - -appender.deprecation_rolling.type = RollingFile -appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log -appender.deprecation_rolling.layout.type = PatternLayout -appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n -appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz -appender.deprecation_rolling.policies.type = Policies -appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.deprecation_rolling.policies.size.size = 10mb -appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy -appender.deprecation_rolling.strategy.max = 4 - -logger.deprecation.name = org.elasticsearch.deprecation -logger.deprecation.level = debug -logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.additivity = false - -appender.index_search_slowlog_rolling.type = RollingFile -appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log -appender.index_search_slowlog_rolling.layout.type = PatternLayout -appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n -appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log -appender.index_search_slowlog_rolling.policies.type = Policies -appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.index_search_slowlog_rolling.policies.time.interval = 1 -appender.index_search_slowlog_rolling.policies.time.modulate = true - -logger.index_search_slowlog_rolling.name = index.search.slowlog -logger.index_search_slowlog_rolling.level = debug -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling -logger.index_search_slowlog_rolling.additivity = false - -appender.index_indexing_slowlog_rolling.type = RollingFile -appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log -appender.index_indexing_slowlog_rolling.layout.type = PatternLayout -appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n -appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log -appender.index_indexing_slowlog_rolling.policies.type = Policies -appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.index_indexing_slowlog_rolling.policies.time.interval = 1 -appender.index_indexing_slowlog_rolling.policies.time.modulate = true - -logger.index_indexing_slowlog.name = index.indexing.slowlog.index -logger.index_indexing_slowlog.level = debug -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling -logger.index_indexing_slowlog.additivity = false diff --git a/ansible/roles/es6/files/mappings/learning.json b/ansible/roles/es6/files/mappings/learning.json deleted file mode 100644 index c5991a3d59..0000000000 --- a/ansible/roles/es6/files/mappings/learning.json +++ /dev/null @@ -1,783 +0,0 @@ -{ - "index_patterns": "learning-*", - "settings": { - "number_of_shards": 5 - }, - "mappings": { - "events_v1": { - "dynamic": false, - "properties": { - "eid": { - "type": "keyword" - }, - "ets": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "mid": { - "type": "keyword" - }, - "uid": { - "type": "keyword" - }, - "ts": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "syncts": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "ver": { - "type": "keyword" - }, - "context":{ - "properties": { - "pdata": { - "properties": { - "id": { - "type": "keyword" - }, - "model": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "granularity": { - "type": "keyword" - }, - "date_range": { - "properties": { - "from": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "to": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - } - } - } - } - }, - "dimensions": { - "properties": { - "did": { - "type": "keyword" - }, - "pdata": { - "properties": { - "id": { - "type": "keyword" - }, - "pid": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "gdata": { - "properties": { - "id": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "sid": { - "type": "keyword" - }, - "channel": { - "type": "keyword" - }, - "group_user": { - "type": "boolean" - }, - "anonymous_user": { - "type": "boolean" - }, - "uid": { - "type": "keyword" - }, - "ss_mid": { - "type": "keyword" - }, - "tag": { - "type": "keyword" - }, - "period": { - "type": "integer" - }, - "content_id": { - "type": "keyword" - }, - "item_id": { - "type": "keyword" - }, - "author_id": { - "type": "keyword" - }, - "loc": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "mode": { - "type": "keyword" - }, - "context_rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - }, - "object_rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - } - } - }, - "edata": { - "properties": { - "eks": { - "properties": { - "total_users_count": { - "type": "long" - }, - "total_devices_count": { - "type": "long" - }, - "total_content_count": { - "type": "long" - }, - "content_ids": { - "type": "keyword" - }, - "device_ids": { - "type": "keyword" - }, - "user_ids": { - "type": "keyword" - }, - "contents": { - "type": "keyword" - }, - "unique_users": { - "type": "keyword" - }, - "avg_ts_session": { - "type": "double" - }, - "total_sessions": { - "type": "long" - }, - "avg_interactions_min": { - "type": "double" - }, - "total_interactions": { - "type": "long" - }, - "avg_pageviews": { - "type": "double" - }, - "total_ts": { - "type": "double" - }, - "time_diff": { - "type": "double" - }, - "time_spent": { - "type": "double" - }, - "interact_events_per_min": { - "type": "double" - }, - "interact_events_count": { - "type": "long" - }, - "total_pageviews_count": { - "type": "long" - }, - "page_views_count": { - "type": "long" - }, - "ce_visits": { - "type": "long" - }, - "start_time": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "end_time": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "syncDate": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "timeSpent": { - "type": "double" - }, - "interactEventsPerMin": { - "type": "double" - }, - "mimeType": { - "type": "keyword" - }, - "contentType": { - "type": "keyword" - }, - "timeDiff": { - "type": "double" - }, - "mode": { - "type": "keyword" - }, - "telemetryVersion": { - "type": "keyword" - }, - "noOfInteractEvents": { - "type": "long" - }, - "interruptTime": { - "type": "double" - }, - "contentCount": { - "type": "long" - }, - "content": { - "type": "keyword" - }, - "time_stamp": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "first_visit": { - "type": "boolean" - }, - "noOfAttempts": { - "type": "integer" - }, - "m_side_loads": { - "type": "integer" - }, - "m_downloads": { - "type": "integer" - }, - "m_avg_rating": { - "type": "double" - }, - "m_ratings": { - "type": "nested", - "properties": { - "rating": { - "type": "integer" - }, - "time": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - } - } - }, - "m_comments": { - "type": "object" - }, - "ce_total_ts": { - "type": "double" - }, - "ce_percent_sessions": { - "type": "double" - }, - "ce_visits_count": { - "type": "long" - }, - "ce_total_visits": { - "type": "long" - }, - "ce_percent_ts": { - "type": "double" - }, - "avg_ts": { - "type": "double" - }, - "inc_res_count": { - "type": "long" - }, - "incorrect_res": { - "type": "nested", - "properties": { - "resp": { - "type": "keyword" - }, - "mmc": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "total_count": { - "type": "long" - }, - "correct_res_count": { - "type": "long" - }, - "correct_res": { - "type": "keyword" - }, - "itemId": { - "type": "keyword" - }, - "exTimeSpent": { - "type": "double" - }, - "score": { - "type": "integer" - }, - "pass": { - "type": "keyword" - }, - "qtitle": { - "type": "keyword" - }, - "qdesc": { - "type": "keyword" - }, - "new_user_count": { - "type": "long" - }, - "anon_total_sessions": { - "type": "long" - }, - "anon_total_ts": { - "type": "double" - }, - "anon_avg_ts_session": { - "type": "double" - }, - "percent_new_users_count": { - "type": "double" - }, - "ce_total_sessions": { - "type": "long" - }, - "unique_users_count": { - "type": "long" - }, - "stageVisitCount": { - "type": "long" - }, - "interactEventsCount": { - "type": "long" - }, - "interactEvents": { - "type": "nested", - "properties": { - "type": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - } - } - }, - "stageId": { - "type": "keyword" - }, - "visitCount": { - "type": "long" - }, - "num_downloads": { - "type": "long" - }, - "num_sideloads": { - "type": "long" - }, - "avg_depth": { - "type": "double" - }, - "publish_pipeline_summary": { - "type": "nested", - "properties": { - "type": { - "type": "keyword" - }, - "state": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "events_summary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "eventsSummary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "screenSummary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "timeSpent": { - "type": "double" - }, - "visitCount": { - "type": "long" - } - } - }, - "activitySummary": { - "type": "nested", - "properties": { - "actType": { - "type": "keyword" - }, - "timeSpent": { - "type": "double" - }, - "count": { - "type": "long" - } - } - }, - "page_summary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "env": { - "type": "keyword" - }, - "time_spent": { - "type": "double" - }, - "visit_count": { - "type": "long" - } - } - }, - "env_summary": { - "type": "nested", - "properties": { - "env": { - "type": "keyword" - }, - "time_spent": { - "type": "double" - }, - "count": { - "type": "long" - } - } - }, - "itemResponses": { - "type": "nested", - "properties": { - "itemId": { - "type": "keyword" - }, - "timeSpent": { - "type": "double" - }, - "exTimeSpent": { - "type": "double" - }, - "score": { - "type": "integer" - }, - "time_stamp": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "pass": { - "type": "keyword" - }, - "qtitle": { - "type": "keyword" - }, - "qdesc": { - "type": "keyword" - } - } - } - } - } - } - }, - "etags": { - "properties": { - "app": { - "type": "keyword" - }, - "partner": { - "type": "keyword" - }, - "dims": { - "type": "keyword" - } - } - }, - "geoip": { - "properties": { - "location": { - "type": "geo_point" - } - }, - "type": "object" - }, - "contentdata": { - "properties": { - "ageGroup": { - "type": "keyword" - }, - "author": { - "type": "keyword" - }, - "audience": { - "type": "keyword" - }, - "code": { - "type": "keyword" - }, - "collaborators": { - "type": "keyword" - }, - "collections": { - "type": "keyword" - }, - "concepts": { - "type": "keyword" - }, - "contentType": { - "type": "keyword" - }, - "curriculum": { - "type": "keyword" - }, - "developer": { - "type": "keyword" - }, - "domain": { - "type": "keyword" - }, - "downloadUrl": { - "type": "keyword" - }, - "downloads": { - "type": "long" - }, - "edition": { - "type": "keyword" - }, - "genre": { - "type": "keyword" - }, - "gradeLevel": { - "type": "keyword" - }, - "keywords": { - "type": "keyword" - }, - "me_totalDevices": { - "type": "long" - }, - "me_totalDownloads": { - "type": "long" - }, - "me_totalInteractions": { - "type": "long" - }, - "me_totalRatings": { - "type": "long" - }, - "me_totalSessionsCount": { - "type": "long" - }, - "me_totalSideloads": { - "type": "long" - }, - "me_totalTimespent": { - "type": "long" - }, - "me_totalUsage": { - "type": "long" - }, - "medium": { - "type": "keyword" - }, - "methods": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "owner": { - "type": "keyword" - }, - "popularity": { - "type": "long" - }, - "portalOwner": { - "type": "keyword" - }, - "publication": { - "type": "keyword" - }, - "publisher": { - "type": "keyword" - }, - "rating": { - "type": "long" - }, - "size": { - "type": "long" - }, - "source": { - "type": "keyword" - }, - "status": { - "type": "keyword" - }, - "subject": { - "type": "keyword" - }, - "templateType": { - "type": "keyword" - }, - "theme": { - "type": "keyword" - }, - "words": { - "type": "keyword" - } - } - }, - "itemdata": { - "properties": { - "concepts": { - "type": "keyword" - }, - "createdBy": { - "type": "keyword" - }, - "createdOn": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "keywords": { - "type": "keyword" - }, - "language": { - "type": "keyword" - }, - "lastUpdatedBy": { - "type": "keyword" - }, - "lastUpdatedOn": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "media": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "num_answers": { - "type": "long" - }, - "owner": { - "type": "keyword" - }, - "qlevel": { - "type": "keyword" - }, - "question": { - "type": "keyword" - }, - "source": { - "type": "keyword" - }, - "status": { - "type": "keyword" - }, - "template": { - "type": "keyword" - }, - "title": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "version": { - "type": "long" - } - } - }, - "@version": { - "type": "keyword" - } - } - } - } -} \ No newline at end of file diff --git a/ansible/roles/es6/files/scripts/calculate-score.groovy b/ansible/roles/es6/files/scripts/calculate-score.groovy deleted file mode 100644 index 442c25ca6c..0000000000 --- a/ansible/roles/es6/files/scripts/calculate-score.groovy +++ /dev/null @@ -1 +0,0 @@ -log(_score * 2) + my_modifier \ No newline at end of file diff --git a/ansible/roles/es6/files/system_key b/ansible/roles/es6/files/system_key deleted file mode 100644 index 91962910d2..0000000000 Binary files a/ansible/roles/es6/files/system_key and /dev/null differ diff --git a/ansible/roles/es6/files/templates/backend.json b/ansible/roles/es6/files/templates/backend.json deleted file mode 100644 index 07161ca348..0000000000 --- a/ansible/roles/es6/files/templates/backend.json +++ /dev/null @@ -1,411 +0,0 @@ -{ - "index_patterns" : "backend-*", - "settings" : { - "number_of_shards" : 5 - }, - "mappings" : { - "events" : { - "dynamic": false, - "properties": { - "@timestamp": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "@version": { - "type": "keyword" - }, - "eid": { - "type": "keyword" - }, - "ets": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "mid": { - "type": "keyword" - }, - "ts": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "ver": { - "type": "keyword" - }, - "actor": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - }, - "edata": { - "properties": { - "comments": { - "type": "keyword" - }, - "correlationid": { - "type": "keyword" - }, - "duration": { - "type": "double" - }, - "data": { - "type": "keyword" - }, - "dir": { - "type": "keyword" - }, - "errtype": { - "type": "keyword" - }, - "id": { - "type": "keyword" - }, - "level": { - "type": "keyword" - }, - "loc": { - "type": "keyword" - }, - "message": { - "type": "keyword" - }, - "mode": { - "type": "keyword" - }, - "pass": { - "type": "keyword" - }, - "prevstate": { - "type": "keyword" - }, - "pageid": { - "type": "keyword" - }, - "query": { - "type": "keyword" - }, - "rating": { - "type": "double" - }, - "score": { - "type": "double" - }, - "size": { - "type": "double" - }, - "state": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "uri": { - "type": "keyword" - }, - "items": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "origin": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - } - } - }, - "item": { - "properties": { - "id": { - "type": "keyword" - }, - "maxscore": { - "type": "long" - }, - "exlength": { - "type": "long" - }, - "uri": { - "type": "keyword" - }, - "desc": { - "type": "keyword" - }, - "title": { - "type": "keyword" - } - } - }, - "target": { - "properties": { - "id": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "category": { - "type": "keyword" - }, - "parent": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - } - } - }, - "visits": { - "type": "nested", - "properties": { - "objid": { - "type": "keyword" - }, - "objtype": { - "type": "keyword" - } - } - }, - "plugin": { - "properties": { - "id": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "category": { - "type": "keyword" - } - } - }, - "object": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "uaspec": { - "properties": { - "agent": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "system": { - "type": "keyword" - }, - "platform": { - "type": "keyword" - }, - "raw": { - "type": "keyword" - } - } - }, - "dspec": { - "properties": { - "camera": { - "type": "keyword" - }, - "cpu": { - "type": "keyword" - }, - "edisk": { - "type": "double" - }, - "id": { - "type": "keyword" - }, - "idisk": { - "type": "double" - }, - "make": { - "type": "keyword" - }, - "mem": { - "type": "double" - }, - "os": { - "type": "keyword" - }, - "scrn": { - "type": "double" - }, - "sims": { - "type": "double" - } - } - } - } - }, - "context": { - "properties": { - "channel": { - "type": "keyword" - }, - "env": { - "type": "keyword" - }, - "sid": { - "type": "keyword" - }, - "did": { - "type": "keyword" - }, - "pdata": { - "properties": { - "id": { - "type": "keyword" - }, - "pid": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - }, - "cdata": { - "type": "nested", - "properties": { - "type": { - "type": "keyword" - }, - "id": { - "type": "keyword" - } - } - } - } - }, - "object": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - }, - "parentid": { - "type": "keyword" - }, - "parenttype": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - } - } - }, - "metadata": { - "properties": { - "source": { - "type": "keyword" - }, - "index_name": { - "type": "keyword" - }, - "index_type": { - "type": "keyword" - }, - "source_eid": { - "type": "keyword" - }, - "source_mid": { - "type": "keyword" - }, - "pump": { - "type": "keyword" - } - } - }, - "flags": { - "properties": { - "v2_converted": { - "type": "boolean" - }, - "dd_processed": { - "type": "boolean" - }, - "tv_processed": { - "type": "boolean" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/ansible/roles/es6/files/templates/compositeindex.json b/ansible/roles/es6/files/templates/compositeindex.json deleted file mode 100644 index d19a189948..0000000000 --- a/ansible/roles/es6/files/templates/compositeindex.json +++ /dev/null @@ -1,5122 +0,0 @@ -{ - "index_patterns": "compositesearch", - "settings": { - "index": { - "max_ngram_diff" : "29", - "analysis": { - "filter": { - "mynGram": { - "token_chars": [ - "letter", - "digit", - "whitespace", - "punctuation", - "symbol" - ], - "min_gram": "1", - "type": "nGram", - "max_gram": "30" - } - }, - "analyzer": { - "cs_index_analyzer": { - "filter": [ - "lowercase", - "mynGram" - ], - "type": "custom", - "tokenizer": "standard" - }, - "keylower": { - "filter": "lowercase", - "tokenizer": "keyword" - }, - "cs_search_analyzer": { - "filter": [ - "standard", - "lowercase" - ], - "type": "custom", - "tokenizer": "standard" - } - } - } - }, - "number_of_shards": 5 - }, - "mappings": { - "cs": { - "dynamic_templates": [ - { - "longs": { - "mapping": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "match_mapping_type": "long" - } - }, - { - "booleans": { - "mapping": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "match_mapping_type": "boolean" - } - }, - { - "doubles": { - "mapping": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "match_mapping_type": "double" - } - }, - { - "dates": { - "mapping": { - "type": "date", - "fields": { - "raw": { - "type": "date" - } - } - }, - "match_mapping_type": "date" - } - }, - { - "strings": { - "mapping": { - "type": "text", - "copy_to": "all_fields", - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - } - }, - "match_mapping_type": "string" - } - }, - { - "nested": { - "mapping": { - "type": "nested", - "fields": { - "type": "nested" - } - }, - "match_mapping_type": "object" - } - } - ], - "properties": { - "Audio": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "IL_FUNC_OBJECT_TYPE": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "IL_INDEXABLE_METADATA_KEY": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "IL_IN_RELATIONS_KEY": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "IL_NON_INDEXABLE_METADATA_KEY": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "IL_OUT_RELATIONS_KEY": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "IL_REQUIRED_PROPERTIES": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "IL_SYSTEM_TAGS_KEY": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "IL_SYS_NODE_TYPE": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "IL_UNIQUE_ID": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "Link": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "RhymingSound": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "SET_OBJECT_TYPE_KEY": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "SET_TYPE": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "SYS_INTERNAL_LAST_UPDATED_ON": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "TemplateType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "actions": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "activityType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "activity_class": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "ageGroup": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "all_fields": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "allowupdate_status": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "altIsoSymbol": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "answer": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "antonyms": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "apiId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "appIcon": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "appIconLabel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "appId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "applicability": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "artifactUrl": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "attribution": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "attributions": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "audience": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "author": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "authoringScore": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "averageComplexity": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "avgGamingTime": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "badgeAssertions": { - "properties": { - "assertionId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "badgeClassId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "badgeClassName": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "createdTS": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "issuerId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - } - } - }, - "baseUrl": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "bloomsTaxonomyLevel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "board": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "body": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "c_null_open_batch_count": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "c_null_private_batch_count": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "c_test_name_open_batch_count": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "cases": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "categories": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "category": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "categoryinstances": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "channel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "channels": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "chapterName": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "chapterNumber": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "childNodes": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "children": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "class": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "code": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "collaborators": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "collections": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "communication_scheme": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "compatibilityLevel": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "conceptData": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "conceptIds": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "concepts": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "consumerGroups": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "consumerId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "contentDisposition": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "contentEncoding": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "contentFilter": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "contentMeta": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "contentType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "contentTypesCount": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "contents": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "converse": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "copyType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "copyright": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "cost": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "count_grade_1": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_grade_2": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_grade_3": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_CC": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_DEM": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_ECH": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_INJ": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_INTF": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_JJ": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_NEG": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_NN": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_NST": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_PRP": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_PSP": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_QC": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_QF": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_QO": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_RB": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_RDP": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_RP": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_VAUX": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_VM": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_pos_WQ": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_source_Karnataka_Govt_Textbooks": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_source_Rajasthan_Textbooks": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "count_textbooks": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "createdBy": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "createdFor": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "createdOn": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "creator": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "curriculum": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "defaultFramework": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "defaultRes": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "description": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "developer": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "dialcodes": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "dimensions": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "domain": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "downloadUrl": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "duration": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "edition": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "editorState": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "ekstepWordnet": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "endPoint": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "endsWithAkshara": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "example": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "exampleSentences": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "externalValidation": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "faculty": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "failPopup": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "feedback": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "fields": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "filter": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "flagReasons": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "flaggedBy": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "flags": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "follows": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "format": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "foundation": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "framework": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "frameworkId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "frameworks": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "fw_hierarchy": { - "type": "text", - "index": false - }, - "gender": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "genders": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "genieScore": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "genre": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "gloss": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "grade": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "gradeLevel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "graph_id": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "grayScaleAppIcon": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "hasAntonyms": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "hasConnotative": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "hasSynonyms": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "hints": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "holonyms": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "hypernyms": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "hyponyms": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "i18n": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "idealScreenDensity": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "idealScreenSize": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "identifier": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "illustrator": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "illustrators": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "image": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "imageCredits": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "index": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "indowordnetId": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "inflections": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "instruction": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "instructions": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "interactivityLevel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "internalValidation": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "ipaSymbol": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "isConnotative": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "isLoanWord": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "isPhrase": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "isoCode": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "isoSymbol": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "itemType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "item_sets": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "items": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "keywords": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "keywords_bkp": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "label": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "langid": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "language": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "languageCode": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "languageId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "languageLevel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "lastFlaggedOn": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "lastPublishDate": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "lastPublishedBy": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "lastPublishedOn": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "lastSubmittedOn": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "lastUpdatedBy": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "lastUpdatedOn": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "launchUrl": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "leafNodesCount": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "learnerLevel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "learningObjective": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "lemma": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "level": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "lhs_options": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "license": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "limit": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "liveWords": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "material": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "maxChainLength": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "maxWords": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "max_score": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_audiosCount": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_averageInteractionsPerMin": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_averageRating": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_averageSessionsPerDevice": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_averageTimespentPerSession": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_avgCreationTsPerSession": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_creationSessions": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_creationTimespent": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_imagesCount": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_timespentDraft": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_timespentReview": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_totalComments": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_totalDevices": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_totalDownloads": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_totalInteractions": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_totalRatings": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_totalSessionsCount": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_totalSideloads": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "me_totalTimespent": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "me_videosCount": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "meaning": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "media": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "mediaType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "mediatype": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "medium": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "meronyms": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "method": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "methods": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "mimeType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "mimeTypesCount": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "minChainLength": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "model": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "model_sample": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "morphology": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "name": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "nodeType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "node_id": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "notes": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "num_answers": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "objectType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "objects": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "objectsUsed": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "occurrenceCount": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "oldContentBody": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "optStatus": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "options": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "organization": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "origin": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "orthographic_complexity": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "os": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "osId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "owner": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "pageNumber": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "parent": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "partial_scoring": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "person": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "phonologic_complexity": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "pictures": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "pkgVersion": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "plurality": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "popularity": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "portalOwner": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "pos": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "posTags": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "pos_categories": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "posterImage": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "pragma": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "prevState": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "previewUrl": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "primaryMeaning": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "primaryMeaningId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "product": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "pronunciations": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "publication": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "publishError": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "publish_type": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "publisher": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "purpose": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "qid": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "qlevel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "qtype": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "question": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "questionImage": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "question_audio": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "question_count": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "question_image": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "references": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "renderingHints": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "resourceType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "resources": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "responses": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "rhs_options": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "rownum": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "ruleId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "ruleObjectType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "ruleScript": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "s3Key": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "sYS_INTERNAL_LAST_UPDATED_ON": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "sampleUsages": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "savingPopup": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "screenshots": { - "type": "text", - "index": false - }, - "semanticVersion": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "showNotification": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "shuffle": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "size": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "skills": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "softConstraints": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "soundCredits": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "source": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "sourceTypes": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "sources": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "startWordsSize": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "startsWithAkshara": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "state": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "status": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "strict_sequencing": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "subject": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "sublevel": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "successPopup": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "syllableCount": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "syllableNotation": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "syllables": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "synonyms": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "synsetCount": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "synsets": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "tags": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "tempData": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "template": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "templateName": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "templateType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "template_id": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "terms": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "text": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "textbookName": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "theme": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "themes": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "thumbnail": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "title": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "toc_url": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "tools": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "totalGamingTime": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "total_items": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "translations": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "ttl": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "tutor": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "type": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "unicode": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "unicodeNotation": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "usedByContent": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "used_for": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "usesContent": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "variants": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "varna": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "version": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "versionCheckMode": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "versionKey": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "visibility": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "voiceCredits": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "vowel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "weightages": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "wordImages": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "wordListIds": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "wordLists": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "word_complexity": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "words": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "wordsets": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "workers": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - }, - "year": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "cs_index_analyzer", - "search_analyzer": "cs_search_analyzer" - } - } - } - } -} diff --git a/ansible/roles/es6/files/templates/dialcode.json b/ansible/roles/es6/files/templates/dialcode.json deleted file mode 100644 index 12e600ad94..0000000000 --- a/ansible/roles/es6/files/templates/dialcode.json +++ /dev/null @@ -1,269 +0,0 @@ -{ - "index_patterns": "dialcode", - "settings": { - "index": { - "analysis": { - "filter": { - "mynGram": { - "token_chars": [ - "letter", - "digit", - "whitespace", - "punctuation", - "symbol" - ], - "min_gram": "1", - "type": "nGram", - "max_gram": "20" - } - }, - "analyzer": { - "dc_search_analyzer": { - "filter": [ - "standard", - "lowercase" - ], - "type": "custom", - "tokenizer": "standard" - }, - "dc_index_analyzer": { - "filter": [ - "lowercase", - "mynGram" - ], - "type": "custom", - "tokenizer": "standard" - }, - "keylower": { - "filter": "lowercase", - "tokenizer": "keyword" - } - } - } - }, - "number_of_shards": 5 - }, - "mappings": { - "dc": { - "dynamic_templates": [ - { - "longs": { - "mapping": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "match_mapping_type": "long" - } - }, - { - "booleans": { - "mapping": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "match_mapping_type": "boolean" - } - }, - { - "doubles": { - "mapping": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "match_mapping_type": "double" - } - }, - { - "dates": { - "mapping": { - "type": "date", - "fields": { - "raw": { - "type": "date" - } - } - }, - "match_mapping_type": "date" - } - }, - { - "strings": { - "mapping": { - "type": "text", - "copy_to": "all_fields", - "analyzer": "dc_index_analyzer", - "search_analyzer": "dc_search_analyzer", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - } - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "all_fields": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "analyzer": "dc_index_analyzer", - "search_analyzer": "dc_search_analyzer" - }, - "batchCode": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "dc_index_analyzer", - "search_analyzer": "dc_search_analyzer" - }, - "batchcode": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "dc_index_analyzer", - "search_analyzer": "dc_search_analyzer" - }, - "channel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "dc_index_analyzer", - "search_analyzer": "dc_search_analyzer" - }, - "dialcode_index": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "generated_on": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "identifier": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "dc_index_analyzer", - "search_analyzer": "dc_search_analyzer" - }, - "objectType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "dc_index_analyzer", - "search_analyzer": "dc_search_analyzer" - }, - "published_on": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "publisher": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "dc_index_analyzer", - "search_analyzer": "dc_search_analyzer" - }, - "status": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "dc_index_analyzer", - "search_analyzer": "dc_search_analyzer" - } - } - } - } -} \ No newline at end of file diff --git a/ansible/roles/es6/files/templates/experimentindex.json b/ansible/roles/es6/files/templates/experimentindex.json deleted file mode 100644 index cda5da556a..0000000000 --- a/ansible/roles/es6/files/templates/experimentindex.json +++ /dev/null @@ -1,185 +0,0 @@ -{ - "index_patterns": "experiment", - "settings": { - "index": { - "number_of_shards": "5", - "analysis": { - "analyzer": { - "exp_index_analyzer": { - "filter": [ - "lowercase" - ], - "type": "custom", - "tokenizer": "standard" - }, - "keylower": { - "filter": "lowercase", - "type": "custom", - "tokenizer": "keyword" - }, - "exp_search_analyzer": { - "filter": [ - "lowercase", - "standard" - ], - "type": "custom", - "tokenizer": "standard" - } - } - }, - "number_of_replicas": "1" - } - }, - "mappings" : { - "exp" : { - "dynamic" : "false", - "properties" : { - "all_fields" : { - "type" : "text", - "fields" : { - "raw" : { - "type" : "text", - "analyzer" : "keylower" - } - }, - "analyzer" : "exp_index_analyzer", - "search_analyzer" : "exp_search_analyzer" - }, - "id" : { - "type" : "text", - "fields" : { - "raw" : { - "type" : "text", - "analyzer" : "keylower", - "fielddata" : true - } - }, - "copy_to" : [ - "all_fields" - ], - "analyzer" : "exp_index_analyzer", - "search_analyzer" : "exp_search_analyzer", - "fielddata" : true - }, - "ver" : { - "type" : "text", - "fields" : { - "raw" : { - "type" : "text", - "analyzer" : "keylower", - "fielddata" : true - } - }, - "copy_to" : [ - "all_fields" - ], - "analyzer" : "exp_index_analyzer", - "search_analyzer" : "exp_search_analyzer", - "fielddata" : true - }, - "expType" : { - "type" : "text", - "fields" : { - "raw" : { - "type" : "text", - "analyzer" : "keylower", - "fielddata" : true - } - }, - "copy_to" : [ - "all_fields" - ], - "analyzer" : "exp_index_analyzer", - "search_analyzer" : "exp_search_analyzer", - "fielddata" : true - }, - "name" : { - "type" : "text", - "fields" : { - "raw" : { - "type" : "text", - "analyzer" : "keylower", - "fielddata" : true - } - }, - "copy_to" : [ - "all_fields" - ], - "analyzer" : "exp_index_analyzer", - "search_analyzer" : "exp_search_analyzer", - "fielddata" : true - }, - "userId" : { - "type" : "keyword" - }, - "userIdMod" : { - "type" : "long" - }, - "extData" : { - "type" : "text", - "fields" : { - "raw" : { - "type" : "text", - "analyzer" : "keylower", - "fielddata" : true - } - }, - "copy_to" : [ - "all_fields" - ], - "analyzer" : "exp_index_analyzer", - "search_analyzer" : "exp_search_analyzer", - "fielddata" : true - }, - "createdOn" : { - "type" : "date", - "fields" : { - "raw" : { - "type" : "date", - "format": "date_hour_minute_second" - } - } - }, - "startDate" : { - "type" : "date", - "fields" : { - "raw" : { - "type" : "date", - "format": "date_hour_minute_second" - } - } - }, - "endDate" : { - "type" : "date", - "fields" : { - "raw" : { - "type" : "date", - "format": "date_hour_minute_second" - } - } - }, - "lastUpdatedOn" : { - "type" : "date", - "fields" : { - "raw" : { - "type" : "date", - "format": "date_hour_minute_second" - } - } - }, - "deviceId" : { - "type" : "keyword" - }, - "deviceIdMod" : { - "type" : "long" - }, - "key" : { - "type" : "keyword" - }, - "url" : { - "type" : "keyword" - } - } - } - } -} \ No newline at end of file diff --git a/ansible/roles/es6/files/templates/failed-telemetry.json b/ansible/roles/es6/files/templates/failed-telemetry.json deleted file mode 100644 index 5d474e7231..0000000000 --- a/ansible/roles/es6/files/templates/failed-telemetry.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "index_patterns" : "failed-telemetry-*", - "settings" : { - "number_of_shards" : 5 - }, - "mappings" : { - "events" : { - "dynamic": false, - "properties": { - "@timestamp": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "@version": { - "type": "keyword" - }, - "channel": { - "type": "keyword" - }, - "eid": { - "type": "keyword" - }, - "ets": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "mid": { - "type": "keyword" - }, - "ts": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "metadata": { - "properties": { - "source": { - "type": "keyword" - }, - "index_name": { - "type": "keyword" - }, - "index_type": { - "type": "keyword" - } - } - }, - "context": { - "properties": { - "channel": { - "type": "keyword" - }, - "env": { - "type": "keyword" - }, - "sid": { - "type": "keyword" - }, - "did": { - "type": "keyword" - }, - "pdata": { - "properties": { - "id": { - "type": "keyword" - }, - "pid": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - }, - "cdata": { - "type": "nested", - "properties": { - "type": { - "type": "keyword" - }, - "id": { - "type": "keyword" - } - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/ansible/roles/es6/files/templates/lp_audit_log.json b/ansible/roles/es6/files/templates/lp_audit_log.json deleted file mode 100644 index b33cf788b0..0000000000 --- a/ansible/roles/es6/files/templates/lp_audit_log.json +++ /dev/null @@ -1,314 +0,0 @@ -{ - "index_patterns": "lp_audit_log", - "settings": { - "index": { - "analysis": { - "filter": { - "mynGram": { - "token_chars": [ - "letter", - "digit", - "whitespace", - "punctuation", - "symbol" - ], - "min_gram": "1", - "type": "nGram", - "max_gram": "20" - } - }, - "analyzer": { - "ah_search_analyzer": { - "filter": [ - "standard", - "lowercase" - ], - "type": "custom", - "tokenizer": "standard" - }, - "keylower": { - "filter": "lowercase", - "tokenizer": "keyword" - }, - "ah_index_analyzer": { - "filter": [ - "lowercase", - "mynGram" - ], - "type": "custom", - "tokenizer": "standard" - } - } - } - }, - "number_of_shards": 5 - }, - "mappings": { - "ah": { - "dynamic_templates": [ - { - "longs": { - "mapping": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "match_mapping_type": "long" - } - }, - { - "booleans": { - "mapping": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "match_mapping_type": "boolean" - } - }, - { - "doubles": { - "mapping": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "match_mapping_type": "double" - } - }, - { - "dates": { - "mapping": { - "type": "date", - "fields": { - "raw": { - "type": "date" - } - } - }, - "match_mapping_type": "date" - } - }, - { - "strings": { - "mapping": { - "type": "text", - "copy_to": "all_fields", - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - } - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "@timestamp": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "@version": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "all_fields": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "audit_id": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "createdOn": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "graphId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "label": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "logRecord": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "objectId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "objectType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "operation": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "requestId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "summary": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - }, - "userId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "ah_index_analyzer", - "search_analyzer": "ah_search_analyzer" - } - } - } - } -} diff --git a/ansible/roles/es6/files/templates/suggestionindex.json b/ansible/roles/es6/files/templates/suggestionindex.json deleted file mode 100644 index cb32559b1b..0000000000 --- a/ansible/roles/es6/files/templates/suggestionindex.json +++ /dev/null @@ -1,784 +0,0 @@ -{ - "index_patterns": "lp_suggestion_index_v1", - "settings": { - "index": { - "analysis": { - "filter": { - "mynGram": { - "token_chars": [ - "letter", - "digit", - "whitespace", - "punctuation", - "symbol" - ], - "min_gram": "1", - "type": "nGram", - "max_gram": "20" - } - }, - "analyzer": { - "keylower": { - "filter": "lowercase", - "tokenizer": "keyword" - }, - "sg_index_analyzer": { - "filter": [ - "lowercase", - "mynGram" - ], - "type": "custom", - "tokenizer": "standard" - }, - "sg_search_analyzer": { - "filter": [ - "standard", - "lowercase" - ], - "type": "custom", - "tokenizer": "standard" - } - } - } - }, - "number_of_shards": 5 - }, - "aliases": { - "lp_suggestion_index_alias": {} - }, - "mappings": { - "sg": { - "dynamic_templates": [ - { - "longs": { - "mapping": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "match_mapping_type": "long" - } - }, - { - "booleans": { - "mapping": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "match_mapping_type": "boolean" - } - }, - { - "doubles": { - "mapping": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "match_mapping_type": "double" - } - }, - { - "dates": { - "mapping": { - "type": "date", - "fields": { - "raw": { - "type": "date" - } - } - }, - "match_mapping_type": "date" - } - }, - { - "strings": { - "mapping": { - "copy_to": "all_fields", - "search_analyzer": "sg_search_analyzer", - "analyzer": "sg_index_analyzer", - "type": "text", - "fields": { - "raw": { - "analyzer": "keylower", - "type": "text", - "fielddata": true - } - } - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "all_fields": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "command": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "comment": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "comments": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "createdOn": { - "type": "date", - "fields": { - "raw": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - }, - "format": "strict_date_optional_time||epoch_millis" - }, - "es_metadata_id": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "lastUpdatedBy": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "objectId": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "objectType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "params": { - "properties": { - "Language": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "ageGroup": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "ageGrups": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "attributions": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "audience": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "code": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "concepts": { - "properties": { - "description": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "identifier": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "name": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "objectType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "relation": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "relationName": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - } - } - }, - "description": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "domain": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "domains": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "genre": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "gradeLevel": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "language": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "medium": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "methods": { - "properties": { - "description": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "identifier": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "name": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "objectType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "relation": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "relationName": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - } - } - }, - "name": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "subject": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "tags": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "templateType": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - } - } - }, - "request": { - "properties": { - "content": { - "properties": { - "comments": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "status": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - } - } - } - } - }, - "status": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "suggestedBy": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - }, - "suggestion_id": { - "type": "text", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - }, - "copy_to": [ - "all_fields" - ], - "analyzer": "sg_index_analyzer", - "search_analyzer": "sg_search_analyzer" - } - } - } - } -} \ No newline at end of file diff --git a/ansible/roles/es6/files/templates/summary.json b/ansible/roles/es6/files/templates/summary.json deleted file mode 100644 index 016f450b7e..0000000000 --- a/ansible/roles/es6/files/templates/summary.json +++ /dev/null @@ -1,802 +0,0 @@ -{ - "index_patterns": "summary-*", - "settings": { - "number_of_shards": 5 - }, - "mappings": { - "events": { - "dynamic": false, - "properties": { - "eid": { - "type": "keyword" - }, - "ets": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "mid": { - "type": "keyword" - }, - "uid": { - "type": "keyword" - }, - "ts": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "syncts": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "ver": { - "type": "keyword" - }, - "channel": { - "type": "keyword" - }, - "context":{ - "properties": { - "pdata": { - "properties": { - "id": { - "type": "keyword" - }, - "model": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "granularity": { - "type": "keyword" - }, - "date_range": { - "properties": { - "from": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "to": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - } - } - } - } - }, - "dimensions": { - "properties": { - "did": { - "type": "keyword" - }, - "pdata": { - "properties": { - "id": { - "type": "keyword" - }, - "pid": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "gdata": { - "properties": { - "id": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "sid": { - "type": "keyword" - }, - "channel": { - "type": "keyword" - }, - "group_user": { - "type": "boolean" - }, - "anonymous_user": { - "type": "boolean" - }, - "uid": { - "type": "keyword" - }, - "ss_mid": { - "type": "keyword" - }, - "tag": { - "type": "keyword" - }, - "period": { - "type": "integer" - }, - "content_id": { - "type": "keyword" - }, - "item_id": { - "type": "keyword" - }, - "author_id": { - "type": "keyword" - }, - "loc": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "mode": { - "type": "keyword" - }, - "context_rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - }, - "object_rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - } - } - }, - "edata": { - "properties": { - "eks": { - "properties": { - "total_users_count": { - "type": "long" - }, - "total_devices_count": { - "type": "long" - }, - "total_content_count": { - "type": "long" - }, - "content_ids": { - "type": "keyword" - }, - "device_ids": { - "type": "keyword" - }, - "user_ids": { - "type": "keyword" - }, - "contents": { - "type": "keyword" - }, - "unique_users": { - "type": "keyword" - }, - "avg_ts_session": { - "type": "double" - }, - "total_sessions": { - "type": "long" - }, - "avg_interactions_min": { - "type": "double" - }, - "total_interactions": { - "type": "long" - }, - "avg_pageviews": { - "type": "double" - }, - "total_ts": { - "type": "double" - }, - "time_diff": { - "type": "double" - }, - "time_spent": { - "type": "double" - }, - "interact_events_per_min": { - "type": "double" - }, - "interact_events_count": { - "type": "long" - }, - "total_pageviews_count": { - "type": "long" - }, - "page_views_count": { - "type": "long" - }, - "ce_visits": { - "type": "long" - }, - "start_time": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "end_time": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "syncDate": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "timeSpent": { - "type": "double" - }, - "interactEventsPerMin": { - "type": "double" - }, - "mimeType": { - "type": "keyword" - }, - "contentType": { - "type": "keyword" - }, - "timeDiff": { - "type": "double" - }, - "mode": { - "type": "keyword" - }, - "telemetryVersion": { - "type": "keyword" - }, - "noOfInteractEvents": { - "type": "long" - }, - "interruptTime": { - "type": "double" - }, - "contentCount": { - "type": "long" - }, - "content": { - "type": "keyword" - }, - "time_stamp": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "first_visit": { - "type": "boolean" - }, - "noOfAttempts": { - "type": "integer" - }, - "m_side_loads": { - "type": "integer" - }, - "m_downloads": { - "type": "integer" - }, - "m_avg_rating": { - "type": "double" - }, - "m_ratings": { - "type": "nested", - "properties": { - "rating": { - "type": "integer" - }, - "time": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - } - } - }, - "m_comments": { - "type": "object" - }, - "ce_total_ts": { - "type": "double" - }, - "ce_percent_sessions": { - "type": "double" - }, - "ce_visits_count": { - "type": "long" - }, - "ce_total_visits": { - "type": "long" - }, - "ce_percent_ts": { - "type": "double" - }, - "avg_ts": { - "type": "double" - }, - "inc_res_count": { - "type": "long" - }, - "incorrect_res": { - "type": "nested", - "properties": { - "resp": { - "type": "keyword" - }, - "mmc": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "total_count": { - "type": "long" - }, - "correct_res_count": { - "type": "long" - }, - "correct_res": { - "type": "keyword" - }, - "itemId": { - "type": "keyword" - }, - "exTimeSpent": { - "type": "double" - }, - "score": { - "type": "integer" - }, - "pass": { - "type": "keyword" - }, - "qtitle": { - "type": "keyword" - }, - "qdesc": { - "type": "keyword" - }, - "new_user_count": { - "type": "long" - }, - "anon_total_sessions": { - "type": "long" - }, - "anon_total_ts": { - "type": "double" - }, - "anon_avg_ts_session": { - "type": "double" - }, - "percent_new_users_count": { - "type": "double" - }, - "ce_total_sessions": { - "type": "long" - }, - "unique_users_count": { - "type": "long" - }, - "stageVisitCount": { - "type": "long" - }, - "interactEventsCount": { - "type": "long" - }, - "interactEvents": { - "type": "nested", - "properties": { - "type": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - } - } - }, - "stageId": { - "type": "keyword" - }, - "visitCount": { - "type": "long" - }, - "num_downloads": { - "type": "long" - }, - "num_sideloads": { - "type": "long" - }, - "avg_depth": { - "type": "double" - }, - "publish_pipeline_summary": { - "type": "nested", - "properties": { - "type": { - "type": "keyword" - }, - "state": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "events_summary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "eventsSummary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "screenSummary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "timeSpent": { - "type": "double" - }, - "visitCount": { - "type": "long" - } - } - }, - "activitySummary": { - "type": "nested", - "properties": { - "actType": { - "type": "keyword" - }, - "timeSpent": { - "type": "double" - }, - "count": { - "type": "long" - } - } - }, - "page_summary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "env": { - "type": "keyword" - }, - "time_spent": { - "type": "double" - }, - "visit_count": { - "type": "long" - } - } - }, - "env_summary": { - "type": "nested", - "properties": { - "env": { - "type": "keyword" - }, - "time_spent": { - "type": "double" - }, - "count": { - "type": "long" - } - } - }, - "itemResponses": { - "type": "nested", - "properties": { - "itemId": { - "type": "keyword" - }, - "timeSpent": { - "type": "double" - }, - "exTimeSpent": { - "type": "double" - }, - "score": { - "type": "integer" - }, - "time_stamp": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "pass": { - "type": "keyword" - }, - "qtitle": { - "type": "keyword" - }, - "qdesc": { - "type": "keyword" - } - } - } - } - } - } - }, - "etags": { - "properties": { - "app": { - "type": "keyword" - }, - "partner": { - "type": "keyword" - }, - "dims": { - "type": "keyword" - } - } - }, - "geoip": { - "properties": { - "location": { - "type": "geo_point" - } - }, - "type": "object" - }, - "contentdata": { - "properties": { - "ageGroup": { - "type": "keyword" - }, - "author": { - "type": "keyword" - }, - "audience": { - "type": "keyword" - }, - "code": { - "type": "keyword" - }, - "collaborators": { - "type": "keyword" - }, - "collections": { - "type": "keyword" - }, - "concepts": { - "type": "keyword" - }, - "contentType": { - "type": "keyword" - }, - "curriculum": { - "type": "keyword" - }, - "developer": { - "type": "keyword" - }, - "domain": { - "type": "keyword" - }, - "downloadUrl": { - "type": "keyword" - }, - "downloads": { - "type": "long" - }, - "edition": { - "type": "keyword" - }, - "genre": { - "type": "keyword" - }, - "gradeLevel": { - "type": "keyword" - }, - "keywords": { - "type": "keyword" - }, - "me_totalDevices": { - "type": "long" - }, - "me_totalDownloads": { - "type": "long" - }, - "me_totalInteractions": { - "type": "long" - }, - "me_totalRatings": { - "type": "long" - }, - "me_totalSessionsCount": { - "type": "long" - }, - "me_totalSideloads": { - "type": "long" - }, - "me_totalTimespent": { - "type": "long" - }, - "me_totalUsage": { - "type": "long" - }, - "medium": { - "type": "keyword" - }, - "methods": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "owner": { - "type": "keyword" - }, - "popularity": { - "type": "long" - }, - "portalOwner": { - "type": "keyword" - }, - "publication": { - "type": "keyword" - }, - "publisher": { - "type": "keyword" - }, - "rating": { - "type": "long" - }, - "size": { - "type": "long" - }, - "source": { - "type": "keyword" - }, - "status": { - "type": "keyword" - }, - "subject": { - "type": "keyword" - }, - "templateType": { - "type": "keyword" - }, - "theme": { - "type": "keyword" - }, - "words": { - "type": "keyword" - } - } - }, - "itemdata": { - "properties": { - "concepts": { - "type": "keyword" - }, - "createdBy": { - "type": "keyword" - }, - "createdOn": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "keywords": { - "type": "keyword" - }, - "language": { - "type": "keyword" - }, - "lastUpdatedBy": { - "type": "keyword" - }, - "lastUpdatedOn": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "media": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "num_answers": { - "type": "long" - }, - "owner": { - "type": "keyword" - }, - "qlevel": { - "type": "keyword" - }, - "question": { - "type": "keyword" - }, - "source": { - "type": "keyword" - }, - "status": { - "type": "keyword" - }, - "template": { - "type": "keyword" - }, - "title": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "version": { - "type": "long" - } - } - }, - "ldata": { - "properties": { - "country": { - "type": "keyword" - }, - "state": { - "type": "keyword" - }, - "district": { - "type": "keyword" - }, - "city": { - "type": "keyword" - } - } - }, - "@version": { - "type": "keyword" - } - } - } - } -} diff --git a/ansible/roles/es6/files/templates/summary_cumulative.json b/ansible/roles/es6/files/templates/summary_cumulative.json deleted file mode 100644 index 99e01b8bdf..0000000000 --- a/ansible/roles/es6/files/templates/summary_cumulative.json +++ /dev/null @@ -1,802 +0,0 @@ -{ - "index_patterns": "summary-cumulative-*", - "settings": { - "number_of_shards": 5 - }, - "mappings": { - "events": { - "dynamic": false, - "properties": { - "eid": { - "type": "keyword" - }, - "ets": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "mid": { - "type": "keyword" - }, - "uid": { - "type": "keyword" - }, - "ts": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "syncts": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "ver": { - "type": "keyword" - }, - "channel": { - "type": "keyword" - }, - "context":{ - "properties": { - "pdata": { - "properties": { - "id": { - "type": "keyword" - }, - "model": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "granularity": { - "type": "keyword" - }, - "date_range": { - "properties": { - "from": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "to": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - } - } - } - } - }, - "dimensions": { - "properties": { - "did": { - "type": "keyword" - }, - "pdata": { - "properties": { - "id": { - "type": "keyword" - }, - "pid": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "gdata": { - "properties": { - "id": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "sid": { - "type": "keyword" - }, - "channel": { - "type": "keyword" - }, - "group_user": { - "type": "boolean" - }, - "anonymous_user": { - "type": "boolean" - }, - "uid": { - "type": "keyword" - }, - "ss_mid": { - "type": "keyword" - }, - "tag": { - "type": "keyword" - }, - "period": { - "type": "integer" - }, - "content_id": { - "type": "keyword" - }, - "item_id": { - "type": "keyword" - }, - "author_id": { - "type": "keyword" - }, - "loc": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "mode": { - "type": "keyword" - }, - "context_rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - }, - "object_rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - } - } - }, - "edata": { - "properties": { - "eks": { - "properties": { - "total_users_count": { - "type": "long" - }, - "total_devices_count": { - "type": "long" - }, - "total_content_count": { - "type": "long" - }, - "content_ids": { - "type": "keyword" - }, - "device_ids": { - "type": "keyword" - }, - "user_ids": { - "type": "keyword" - }, - "contents": { - "type": "keyword" - }, - "unique_users": { - "type": "keyword" - }, - "avg_ts_session": { - "type": "double" - }, - "total_sessions": { - "type": "long" - }, - "avg_interactions_min": { - "type": "double" - }, - "total_interactions": { - "type": "long" - }, - "avg_pageviews": { - "type": "double" - }, - "total_ts": { - "type": "double" - }, - "time_diff": { - "type": "double" - }, - "time_spent": { - "type": "double" - }, - "interact_events_per_min": { - "type": "double" - }, - "interact_events_count": { - "type": "long" - }, - "total_pageviews_count": { - "type": "long" - }, - "page_views_count": { - "type": "long" - }, - "ce_visits": { - "type": "long" - }, - "start_time": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "end_time": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "syncDate": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "timeSpent": { - "type": "double" - }, - "interactEventsPerMin": { - "type": "double" - }, - "mimeType": { - "type": "keyword" - }, - "contentType": { - "type": "keyword" - }, - "timeDiff": { - "type": "double" - }, - "mode": { - "type": "keyword" - }, - "telemetryVersion": { - "type": "keyword" - }, - "noOfInteractEvents": { - "type": "long" - }, - "interruptTime": { - "type": "double" - }, - "contentCount": { - "type": "long" - }, - "content": { - "type": "keyword" - }, - "time_stamp": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "first_visit": { - "type": "boolean" - }, - "noOfAttempts": { - "type": "integer" - }, - "m_side_loads": { - "type": "integer" - }, - "m_downloads": { - "type": "integer" - }, - "m_avg_rating": { - "type": "double" - }, - "m_ratings": { - "type": "nested", - "properties": { - "rating": { - "type": "integer" - }, - "time": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - } - } - }, - "m_comments": { - "type": "object" - }, - "ce_total_ts": { - "type": "double" - }, - "ce_percent_sessions": { - "type": "double" - }, - "ce_visits_count": { - "type": "long" - }, - "ce_total_visits": { - "type": "long" - }, - "ce_percent_ts": { - "type": "double" - }, - "avg_ts": { - "type": "double" - }, - "inc_res_count": { - "type": "long" - }, - "incorrect_res": { - "type": "nested", - "properties": { - "resp": { - "type": "keyword" - }, - "mmc": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "total_count": { - "type": "long" - }, - "correct_res_count": { - "type": "long" - }, - "correct_res": { - "type": "keyword" - }, - "itemId": { - "type": "keyword" - }, - "exTimeSpent": { - "type": "double" - }, - "score": { - "type": "integer" - }, - "pass": { - "type": "keyword" - }, - "qtitle": { - "type": "keyword" - }, - "qdesc": { - "type": "keyword" - }, - "new_user_count": { - "type": "long" - }, - "anon_total_sessions": { - "type": "long" - }, - "anon_total_ts": { - "type": "double" - }, - "anon_avg_ts_session": { - "type": "double" - }, - "percent_new_users_count": { - "type": "double" - }, - "ce_total_sessions": { - "type": "long" - }, - "unique_users_count": { - "type": "long" - }, - "stageVisitCount": { - "type": "long" - }, - "interactEventsCount": { - "type": "long" - }, - "interactEvents": { - "type": "nested", - "properties": { - "type": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - } - } - }, - "stageId": { - "type": "keyword" - }, - "visitCount": { - "type": "long" - }, - "num_downloads": { - "type": "long" - }, - "num_sideloads": { - "type": "long" - }, - "avg_depth": { - "type": "double" - }, - "publish_pipeline_summary": { - "type": "nested", - "properties": { - "type": { - "type": "keyword" - }, - "state": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "events_summary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "eventsSummary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "count": { - "type": "long" - } - } - }, - "screenSummary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "timeSpent": { - "type": "double" - }, - "visitCount": { - "type": "long" - } - } - }, - "activitySummary": { - "type": "nested", - "properties": { - "actType": { - "type": "keyword" - }, - "timeSpent": { - "type": "double" - }, - "count": { - "type": "long" - } - } - }, - "page_summary": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "env": { - "type": "keyword" - }, - "time_spent": { - "type": "double" - }, - "visit_count": { - "type": "long" - } - } - }, - "env_summary": { - "type": "nested", - "properties": { - "env": { - "type": "keyword" - }, - "time_spent": { - "type": "double" - }, - "count": { - "type": "long" - } - } - }, - "itemResponses": { - "type": "nested", - "properties": { - "itemId": { - "type": "keyword" - }, - "timeSpent": { - "type": "double" - }, - "exTimeSpent": { - "type": "double" - }, - "score": { - "type": "integer" - }, - "time_stamp": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "pass": { - "type": "keyword" - }, - "qtitle": { - "type": "keyword" - }, - "qdesc": { - "type": "keyword" - } - } - } - } - } - } - }, - "etags": { - "properties": { - "app": { - "type": "keyword" - }, - "partner": { - "type": "keyword" - }, - "dims": { - "type": "keyword" - } - } - }, - "geoip": { - "properties": { - "location": { - "type": "geo_point" - } - }, - "type": "object" - }, - "contentdata": { - "properties": { - "ageGroup": { - "type": "keyword" - }, - "author": { - "type": "keyword" - }, - "audience": { - "type": "keyword" - }, - "code": { - "type": "keyword" - }, - "collaborators": { - "type": "keyword" - }, - "collections": { - "type": "keyword" - }, - "concepts": { - "type": "keyword" - }, - "contentType": { - "type": "keyword" - }, - "curriculum": { - "type": "keyword" - }, - "developer": { - "type": "keyword" - }, - "domain": { - "type": "keyword" - }, - "downloadUrl": { - "type": "keyword" - }, - "downloads": { - "type": "long" - }, - "edition": { - "type": "keyword" - }, - "genre": { - "type": "keyword" - }, - "gradeLevel": { - "type": "keyword" - }, - "keywords": { - "type": "keyword" - }, - "me_totalDevices": { - "type": "long" - }, - "me_totalDownloads": { - "type": "long" - }, - "me_totalInteractions": { - "type": "long" - }, - "me_totalRatings": { - "type": "long" - }, - "me_totalSessionsCount": { - "type": "long" - }, - "me_totalSideloads": { - "type": "long" - }, - "me_totalTimespent": { - "type": "long" - }, - "me_totalUsage": { - "type": "long" - }, - "medium": { - "type": "keyword" - }, - "methods": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "owner": { - "type": "keyword" - }, - "popularity": { - "type": "long" - }, - "portalOwner": { - "type": "keyword" - }, - "publication": { - "type": "keyword" - }, - "publisher": { - "type": "keyword" - }, - "rating": { - "type": "long" - }, - "size": { - "type": "long" - }, - "source": { - "type": "keyword" - }, - "status": { - "type": "keyword" - }, - "subject": { - "type": "keyword" - }, - "templateType": { - "type": "keyword" - }, - "theme": { - "type": "keyword" - }, - "words": { - "type": "keyword" - } - } - }, - "itemdata": { - "properties": { - "concepts": { - "type": "keyword" - }, - "createdBy": { - "type": "keyword" - }, - "createdOn": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "keywords": { - "type": "keyword" - }, - "language": { - "type": "keyword" - }, - "lastUpdatedBy": { - "type": "keyword" - }, - "lastUpdatedOn": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "media": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "num_answers": { - "type": "long" - }, - "owner": { - "type": "keyword" - }, - "qlevel": { - "type": "keyword" - }, - "question": { - "type": "keyword" - }, - "source": { - "type": "keyword" - }, - "status": { - "type": "keyword" - }, - "template": { - "type": "keyword" - }, - "title": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "version": { - "type": "long" - } - } - }, - "ldata": { - "properties": { - "country": { - "type": "keyword" - }, - "state": { - "type": "keyword" - }, - "district": { - "type": "keyword" - }, - "city": { - "type": "keyword" - } - } - }, - "@version": { - "type": "keyword" - } - } - } - } -} diff --git a/ansible/roles/es6/files/templates/telemetry.json b/ansible/roles/es6/files/templates/telemetry.json deleted file mode 100644 index afe9522328..0000000000 --- a/ansible/roles/es6/files/templates/telemetry.json +++ /dev/null @@ -1,430 +0,0 @@ -{ - "index_patterns" : "telemetry-*", - "settings" : { - "number_of_shards" : 5 - }, - "mappings" : { - "events" : { - "dynamic": false, - "properties": { - "@timestamp": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "@version": { - "type": "keyword" - }, - "eid": { - "type": "keyword" - }, - "ets": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "mid": { - "type": "keyword" - }, - "ts": { - "format": "strict_date_optional_time||epoch_millis", - "type": "date" - }, - "ver": { - "type": "keyword" - }, - "actor": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - }, - "edata": { - "properties": { - "comments": { - "type": "keyword" - }, - "correlationid": { - "type": "keyword" - }, - "duration": { - "type": "double" - }, - "data": { - "type": "keyword" - }, - "dir": { - "type": "keyword" - }, - "errtype": { - "type": "keyword" - }, - "id": { - "type": "keyword" - }, - "level": { - "type": "keyword" - }, - "loc": { - "type": "keyword" - }, - "message": { - "type": "keyword" - }, - "mode": { - "type": "keyword" - }, - "pass": { - "type": "keyword" - }, - "prevstate": { - "type": "keyword" - }, - "pageid": { - "type": "keyword" - }, - "query": { - "type": "keyword" - }, - "rating": { - "type": "double" - }, - "score": { - "type": "double" - }, - "size": { - "type": "double" - }, - "state": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "uri": { - "type": "keyword" - }, - "items": { - "type": "nested", - "properties": { - "id": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "origin": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - } - } - }, - "item": { - "properties": { - "id": { - "type": "keyword" - }, - "maxscore": { - "type": "long" - }, - "exlength": { - "type": "long" - }, - "uri": { - "type": "keyword" - }, - "desc": { - "type": "keyword" - }, - "title": { - "type": "keyword" - } - } - }, - "target": { - "properties": { - "id": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "category": { - "type": "keyword" - }, - "parent": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - } - } - }, - "visits": { - "type": "nested", - "properties": { - "objid": { - "type": "keyword" - }, - "objtype": { - "type": "keyword" - } - } - }, - "plugin": { - "properties": { - "id": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "category": { - "type": "keyword" - } - } - }, - "object": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "uaspec": { - "properties": { - "agent": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "system": { - "type": "keyword" - }, - "platform": { - "type": "keyword" - }, - "raw": { - "type": "keyword" - } - } - }, - "dspec": { - "properties": { - "camera": { - "type": "keyword" - }, - "cpu": { - "type": "keyword" - }, - "edisk": { - "type": "double" - }, - "id": { - "type": "keyword" - }, - "idisk": { - "type": "double" - }, - "make": { - "type": "keyword" - }, - "mem": { - "type": "double" - }, - "os": { - "type": "keyword" - }, - "scrn": { - "type": "double" - }, - "sims": { - "type": "double" - } - } - } - } - }, - "context": { - "properties": { - "channel": { - "type": "keyword" - }, - "env": { - "type": "keyword" - }, - "sid": { - "type": "keyword" - }, - "did": { - "type": "keyword" - }, - "pdata": { - "properties": { - "id": { - "type": "keyword" - }, - "pid": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - } - } - }, - "rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - }, - "cdata": { - "type": "nested", - "properties": { - "type": { - "type": "keyword" - }, - "id": { - "type": "keyword" - } - } - } - } - }, - "object": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "subtype": { - "type": "keyword" - }, - "parentid": { - "type": "keyword" - }, - "parenttype": { - "type": "keyword" - }, - "ver": { - "type": "keyword" - }, - "rollup": { - "properties": { - "l1": { - "type": "keyword" - }, - "l2": { - "type": "keyword" - }, - "l3": { - "type": "keyword" - }, - "l4": { - "type": "keyword" - } - } - } - } - }, - "ldata": { - "properties": { - "country": { - "type": "keyword" - }, - "state": { - "type": "keyword" - }, - "district": { - "type": "keyword" - }, - "city": { - "type": "keyword" - } - } - }, - "metadata": { - "properties": { - "source": { - "type": "keyword" - }, - "index_name": { - "type": "keyword" - }, - "index_type": { - "type": "keyword" - }, - "source_eid": { - "type": "keyword" - }, - "source_mid": { - "type": "keyword" - }, - "pump": { - "type": "keyword" - } - } - }, - "flags": { - "properties": { - "v2_converted": { - "type": "boolean" - }, - "dd_processed": { - "type": "boolean" - }, - "tv_processed": { - "type": "boolean" - }, - "ldata_retrieved": { - "type": "boolean" - } - } - } - } - } - } -} diff --git a/ansible/roles/es6/files/templates/vocabulary.json b/ansible/roles/es6/files/templates/vocabulary.json deleted file mode 100644 index d50cc7ec0e..0000000000 --- a/ansible/roles/es6/files/templates/vocabulary.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "index_patterns": "vocabularyterm", - "settings": { - "index": { - "analysis": { - "filter": { - "mynGram": { - "token_chars": [ - "letter", - "digit", - "whitespace", - "punctuation", - "symbol" - ], - "min_gram": "1", - "type": "edge_ngram", - "max_gram": "20" - } - }, - "analyzer": { - "keylower": { - "filter": "lowercase", - "tokenizer": "keyword" - }, - "vt_index_analyzer": { - "filter": [ - "lowercase", - "mynGram" - ], - "type": "custom", - "tokenizer": "standard" - }, - "vt_search_analyzer": { - "filter": [ - "standard", - "lowercase" - ], - "type": "custom", - "tokenizer": "standard" - } - } - } - }, - "number_of_shards": 5 - }, - "mappings": { - "vt": { - "dynamic_templates": [ - { - "longs": { - "mapping": { - "type": "long", - "fields": { - "raw": { - "type": "long" - } - } - }, - "match_mapping_type": "long" - } - }, - { - "booleans": { - "mapping": { - "type": "boolean", - "fields": { - "raw": { - "type": "boolean" - } - } - }, - "match_mapping_type": "boolean" - } - }, - { - "doubles": { - "mapping": { - "type": "double", - "fields": { - "raw": { - "type": "double" - } - } - }, - "match_mapping_type": "double" - } - }, - { - "dates": { - "mapping": { - "type": "date", - "fields": { - "raw": { - "type": "date" - } - } - }, - "match_mapping_type": "date" - } - }, - { - "strings": { - "mapping": { - "type": "text", - "copy_to": "all_fields", - "analyzer": "vt_index_analyzer", - "search_analyzer": "vt_search_analyzer", - "fields": { - "raw": { - "type": "text", - "fielddata": true, - "analyzer": "keylower" - } - } - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "all_fields": { - "type": "text", - "fielddata": true, - "analyzer": "vt_index_analyzer", - "search_analyzer": "vt_search_analyzer" - } - } - } - } -} \ No newline at end of file diff --git a/ansible/roles/es6/filter_plugins/custom.py b/ansible/roles/es6/filter_plugins/custom.py deleted file mode 100644 index 22177cdd19..0000000000 --- a/ansible/roles/es6/filter_plugins/custom.py +++ /dev/null @@ -1,57 +0,0 @@ -__author__ = 'dale mcdiarmid' - -import re -import os.path -from six import string_types - -def modify_list(values=[], pattern='', replacement='', ignorecase=False): - ''' Perform a `re.sub` on every item in the list''' - if ignorecase: - flags = re.I - else: - flags = 0 - _re = re.compile(pattern, flags=flags) - return [_re.sub(replacement, value) for value in values] - -def append_to_list(values=[], suffix=''): - if isinstance(values, string_types): - values = values.split(',') - return [str(value+suffix) for value in values] - -def array_to_str(values=[],separator=','): - return separator.join(values) - -def extract_role_users(users={},exclude_users=[]): - role_users=[] - for user,details in users.iteritems(): - if user not in exclude_users and "roles" in details: - for role in details["roles"]: - role_users.append(role+":"+user) - return role_users - -def filename(filename=''): - return os.path.splitext(os.path.basename(filename))[0] - -def remove_reserved(user_roles={}): - not_reserved = [] - for user_role,details in user_roles.items(): - if not "metadata" in details or not "_reserved" in details["metadata"] or not details["metadata"]["_reserved"]: - not_reserved.append(user_role) - return not_reserved - -def filter_reserved(users_role={}): - reserved = [] - for user_role,details in users_role.items(): - if "metadata" in details and "_reserved" in details["metadata"] and details["metadata"]["_reserved"]: - reserved.append(user_role) - return reserved - -class FilterModule(object): - def filters(self): - return {'modify_list': modify_list, - 'append_to_list':append_to_list, - 'filter_reserved':filter_reserved, - 'array_to_str':array_to_str, - 'extract_role_users':extract_role_users, - 'remove_reserved':remove_reserved, - 'filename':filename} \ No newline at end of file diff --git a/ansible/roles/es6/filter_plugins/custom.pyc b/ansible/roles/es6/filter_plugins/custom.pyc deleted file mode 100644 index 771ec10df5..0000000000 Binary files a/ansible/roles/es6/filter_plugins/custom.pyc and /dev/null differ diff --git a/ansible/roles/es6/handlers/main.yml b/ansible/roles/es6/handlers/main.yml deleted file mode 100644 index d71397a395..0000000000 --- a/ansible/roles/es6/handlers/main.yml +++ /dev/null @@ -1,14 +0,0 @@ - -- name: reload systemd configuration - become: yes - command: systemctl daemon-reload - -# Restart service and ensure it is enabled - -- name: restart elasticsearch - become: yes - service: name={{instance_init_script | basename}} state=restarted enabled=yes - when: - - es_restart_on_change - - es_start_service - register: es_restarted diff --git a/ansible/roles/es6/meta/main.yml b/ansible/roles/es6/meta/main.yml deleted file mode 100644 index aeecec7767..0000000000 --- a/ansible/roles/es6/meta/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- - -allow_duplicates: yes - -galaxy_info: - author: Robin Clarke, Jakob Reiter, Dale McDiarmid - description: Elasticsearch for Linux - company: "Elastic.co" - license: "license (Apache)" - min_ansible_version: 2.3.2 - platforms: - - name: EL - versions: - - 6 - - 7 - - name: Debian - versions: - - all - - name: Ubuntu - versions: - - all - categories: - - system - -dependencies: [] diff --git a/ansible/roles/es6/tasks/elasticsearch-Debian-version-lock.yml b/ansible/roles/es6/tasks/elasticsearch-Debian-version-lock.yml deleted file mode 100644 index d9fdd698bc..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-Debian-version-lock.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Debian - hold elasticsearch version - become: yes - command: apt-mark hold elasticsearch - register: hold_elasticsearch_result - changed_when: "hold_elasticsearch_result.stdout != 'elasticsearch was already set on hold.'" diff --git a/ansible/roles/es6/tasks/elasticsearch-Debian.yml b/ansible/roles/es6/tasks/elasticsearch-Debian.yml deleted file mode 100644 index ebaaa61744..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-Debian.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- - -- name: set fact force_install to no - set_fact: force_install=no - -- name: set fact force_install to yes - set_fact: force_install=yes - when: es_allow_downgrades - -- name: Debian - Install apt-transport-https to support https APT downloads - become: yes - apt: name=apt-transport-https state=present - when: es_use_repository - -- name: Debian - Add Elasticsearch repository key - become: yes - apt_key: url="{{ es_apt_key }}" state=present - when: es_use_repository and es_apt_key - -- name: Debian - Add elasticsearch repository - become: yes - apt_repository: repo={{ item.repo }} state={{ item.state}} - with_items: - - { repo: "{{ es_apt_url_old }}", state: "absent" } - - { repo: "{{ es_apt_url }}", state: "present" } - when: es_use_repository - -- name: Debian - Include versionlock - include: elasticsearch-Debian-version-lock.yml - when: es_version_lock - -- name: Debian - Ensure elasticsearch is installed - become: yes - apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} allow_unauthenticated={{ 'no' if es_apt_key else 'yes' }} cache_valid_time=86400 - when: es_use_repository - register: debian_elasticsearch_install_from_repo - notify: restart elasticsearch - -- name: Debian - Download elasticsearch from url - get_url: url={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.deb{% endif %} dest=/tmp/elasticsearch-{{ es_version }}.deb validate_certs=no - when: not es_use_repository - -- name: Debian - Ensure elasticsearch is installed from downloaded package - become: yes - apt: deb=/tmp/elasticsearch-{{ es_version }}.deb - when: not es_use_repository - register: elasticsearch_install_from_package - notify: restart elasticsearch diff --git a/ansible/roles/es6/tasks/elasticsearch-RedHat-version-lock.yml b/ansible/roles/es6/tasks/elasticsearch-RedHat-version-lock.yml deleted file mode 100644 index b5711a2f3b..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-RedHat-version-lock.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: RedHat - install yum-version-lock - become: yes - yum: name=yum-plugin-versionlock state=present update_cache=yes -- name: RedHat - lock elasticsearch version - become: yes - shell: yum versionlock delete 0:elasticsearch* ; yum versionlock add elasticsearch{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %} diff --git a/ansible/roles/es6/tasks/elasticsearch-RedHat.yml b/ansible/roles/es6/tasks/elasticsearch-RedHat.yml deleted file mode 100644 index 588f0cec04..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-RedHat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: Ensure libselinux-python on CentOS 6.x - become: yes - yum: name=libselinux-python state=present update_cache=yes - when: ( ansible_distribution == "CentOS" ) and ( ansible_distribution_major_version == "6" ) - -- name: RedHat - add Elasticsearch repo - become: yes - template: src=elasticsearch.repo dest=/etc/yum.repos.d/elasticsearch-{{ es_major_version }}.repo - when: es_use_repository - -- name: RedHat - include versionlock - include: elasticsearch-RedHat-version-lock.yml - when: es_version_lock - -- name: RedHat - Install Elasticsearch - become: yes - yum: name=elasticsearch{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %} state=present update_cache=yes - when: es_use_repository - register: redhat_elasticsearch_install_from_repo - notify: restart elasticsearch - until: '"failed" not in redhat_elasticsearch_install_from_repo' - retries: 5 - delay: 10 - -- name: RedHat - Install Elasticsearch from url - become: yes - yum: name={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.noarch.rpm{% endif %} state=present - when: not es_use_repository - register: elasticsearch_install_from_package - notify: restart elasticsearch diff --git a/ansible/roles/es6/tasks/elasticsearch-config.yml b/ansible/roles/es6/tasks/elasticsearch-config.yml deleted file mode 100644 index 987de718da..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-config.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- -# Configure Elasticsearch Node - -#Create required directories -- name: Create Directories - become: yes - file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} - with_items: - - "{{pid_dir}}" - - "{{log_dir}}" - - "{{conf_dir}}" - -- name: Create Data Directories - become: yes - file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} - with_items: - - "{{data_dirs}}" - - -#Copy the config template -- name: Copy Configuration File - become: yes - template: src=elasticsearch.yml.j2 dest={{conf_dir}}/elasticsearch.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes - register: system_change - notify: restart elasticsearch - -#Copy the instance specific default file -- name: Copy Default File for Instance - become: yes - template: src=elasticsearch.j2 dest={{instance_default_file}} mode=0644 force=yes - notify: restart elasticsearch - -#Copy the instance specific init file -- name: Copy Debian Init File for Instance - become: yes - template: src=init/debian/elasticsearch.j2 dest={{instance_init_script}} mode=0755 force=yes - when: ansible_os_family == 'Debian' and not use_system_d - notify: restart elasticsearch - -#Copy the instance specific init file -- name: Copy Redhat Init File for Instance - become: yes - template: src=init/redhat/elasticsearch.j2 dest={{instance_init_script}} mode=0755 force=yes - when: ansible_os_family == 'RedHat' and not use_system_d - notify: restart elasticsearch - -#Copy the systemd specific file if systemd is installed -- name: Copy Systemd File for Instance - become: yes - template: src=systemd/elasticsearch.j2 dest={{instance_sysd_script}} mode=0644 force=yes - when: use_system_d - notify: - - reload systemd configuration - - restart elasticsearch - -#Copy the logging.yml -- name: Copy log4j2.properties File for Instance - become: yes - template: src={{es_config_log4j2}} dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes - notify: restart elasticsearch - -- name: Copy jvm.options File for Instance - become: yes - template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes - notify: restart elasticsearch - -#Clean up un-wanted package scripts to avoid confusion - -- name: Delete Default Init - become: yes - file: dest=/etc/init.d/elasticsearch state=absent - -- name: Create empty default environment file - become: yes - changed_when: False - copy: - dest: /etc/default/elasticsearch - content: '' - when: ansible_os_family == 'Debian' - -- name: Create empty default environment file - become: yes - changed_when: False - copy: - dest: /etc/sysconfig/elasticsearch - content: '' - when: ansible_os_family == 'RedHat' - -- name: Delete Default Sysconfig File - become: yes - file: dest="{{ sysd_script }}" state=absent - -- name: Delete Default Configuration File - become: yes - file: dest=/etc/elasticsearch/elasticsearch.yml state=absent - -- name: Delete Default Logging File - become: yes - file: dest=/etc/elasticsearch/logging.yml state=absent - -- name: Delete Default Logging File - become: yes - file: dest=/etc/elasticsearch/log4j2.properties state=absent - -- name: Delete Default JVM Options File - become: yes - file: dest=/etc/elasticsearch/jvm.options state=absent diff --git a/ansible/roles/es6/tasks/elasticsearch-optional-user.yml b/ansible/roles/es6/tasks/elasticsearch-optional-user.yml deleted file mode 100644 index d8f787e4b7..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-optional-user.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -#Add the elasticsearch user before installing from packages. -- name: Ensure optional elasticsearch group is created with the correct id. - become: yes - #Restart if these change - notify: restart elasticsearch - group: - state: present - name: "{{ es_group }}" - system: yes - gid: "{{ es_group_id }}" - -- name: Ensure optional elasticsearch user is created with the correct id. - become: yes - #Restart if these change - notify: restart elasticsearch - user: - state: present - name: "{{ es_user }}" - comment: elasticsearch system user - system: yes - createhome: no - uid: "{{ es_user_id }}" - group: "{{ es_group }}" diff --git a/ansible/roles/es6/tasks/elasticsearch-parameters.yml b/ansible/roles/es6/tasks/elasticsearch-parameters.yml deleted file mode 100644 index 13c6472a3d..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-parameters.yml +++ /dev/null @@ -1,75 +0,0 @@ -# Check for mandatory parameters - -- name: fail when es_instance is not defined - fail: msg="es_instance_name must be specified and cannot be blank" - when: es_instance_name is not defined or es_instance_name == '' - -- name: fail when es_proxy_port is not defined or is blank - fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" - when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') - -- name: debug message - debug: msg="WARNING - It is recommended you specify the parameter 'http.port'" - when: es_config['http.port'] is not defined - -- name: debug message - debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port'" - when: es_config['transport.tcp.port'] is not defined - -- name: debug message - debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts'" - when: es_config['discovery.zen.ping.unicast.hosts'] is not defined - -#If the user attempts to lock memory they must specify a heap size -- name: fail when heap size is not specified when using memory lock - fail: msg="If locking memory with bootstrap.memory_lock a heap size must be specified" - when: es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True and es_heap_size is not defined - -#Check if working with security we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work -- name: fail when api credentials are not declared when using security - fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" - when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined - -- name: set fact file_reserved_users - set_fact: file_reserved_users={{ es_users.file.keys() | intersect (reserved_xpack_users) }} - when: es_users is defined and es_users.file is defined and (es_users.file.keys() | length > 0) and (es_users.file.keys() | intersect (reserved_xpack_users) | length > 0) - -- name: fail when changing users through file realm - fail: - msg: "ERROR: INVALID CONFIG - YOU CANNOT CHANGE RESERVED USERS THROUGH THE FILE REALM. THE FOLLOWING CANNOT BE CHANGED: {{file_reserved_users}}. USE THE NATIVE REALM." - when: file_reserved_users | default([]) | length > 0 - -- name: set fact instance_default_file - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} -- name: set fact instance_init_script - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} -- name: set fact conf_dir - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} -- name: set fact m_lock_enabled - set_fact: m_lock_enabled={{ es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True }} - -#TODO - if transport.host is not local maybe error on boostrap checks - - -#Use systemd for the following distributions: -#Ubuntu 15 and up -#Debian 8 and up -#Centos 7 and up -#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied. - -- name: set fact use_system_d - set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution in ['RedHat','CentOS'] and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }} - -- name: set fact instance_sysd_script - set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}} - when: use_system_d -#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN. - -- name: set fact instance_suffix - set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }} -- name: set fact pid_dir - set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}} -- name: set fact log_dir - set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}} -- name: set fact log_dir - set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }} diff --git a/ansible/roles/es6/tasks/elasticsearch-plugins.yml b/ansible/roles/es6/tasks/elasticsearch-plugins.yml deleted file mode 100644 index 5d4e2d8153..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-plugins.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- - -# es_plugins_reinstall will be set to true if elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed -# i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting. -- name: set fact es_plugins_reinstall to true - set_fact: es_plugins_reinstall=true - when: (((debian_elasticsearch_install_from_repo is defined and debian_elasticsearch_install_from_repo.changed) or (redhat_elasticsearch_install_from_repo is defined and redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none - -- name: set fact list_command - set_fact: list_command="" -#If we are reinstalling all plugins, e.g. to a version change, we need to remove all plugins (inc. x-pack) to install any plugins. Otherwise we don't consider x-pack so the role stays idempotent. -- name: set fact list_command check for x-pack - set_fact: list_command="| grep -vE 'x-pack'" - when: not es_plugins_reinstall - -#List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version. -- name: Check installed elasticsearch plugins - become: yes - shell: "ls {{es_home}}/plugins {{list_command}}" - register: installed_plugins - changed_when: False - ignore_errors: yes - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - -#if es_plugins_reinstall is set to true we remove ALL plugins -- name: set fact plugins_to_remove to install_plugins.stdout_lines - set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | default([]) }}" - when: es_plugins_reinstall - -#if the plugins listed are different than those requested, we remove those installed but not listed in the config -- name: set fact plugins_to_remove to delete plugins installed but not listed in es_plugins - set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('[*].plugin')) | default([]) }}" - when: not es_plugins_reinstall - -#if es_plugins_reinstall is set to true we (re)install ALL plugins -- name: set fact plugins_to_install to es_plugins - set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | default([]) }}" - when: es_plugins_reinstall - -#if the plugins listed are different than those requested, we install those not installed but listed in the config -- name: set fact to plugins_to_install to those in es_config but not installed - set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | difference(installed_plugins.stdout_lines) | default([]) }}" - when: not es_plugins_reinstall - -# This removes any currently installed plugins (to prevent errors when reinstalling) -- name: Remove elasticsearch plugins - become: yes - command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent" - ignore_errors: yes - with_items: "{{ plugins_to_remove | default([]) }}" - notify: restart elasticsearch - register: plugin_removed - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - -- name: Install elasticsearch plugins - become: yes - command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }} --batch --silent" - register: plugin_installed - failed_when: "'ERROR' in plugin_installed.stdout" - changed_when: plugin_installed.rc == 0 - with_items: "{{ es_plugins }}" - when: item.plugin in plugins_to_install - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - ES_JAVA_OPTS: "{% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -Dhttp.proxyHost={{ item.proxy_host }} -Dhttp.proxyPort={{ item.proxy_port }} -Dhttps.proxyHost={{ item.proxy_host }} -Dhttps.proxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }} {% endif %}" - until: plugin_installed.rc == 0 - retries: 5 - delay: 5 - -#Set permissions on plugins directory -- name: Set Plugin Directory Permissions - become: yes - file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes diff --git a/ansible/roles/es6/tasks/elasticsearch-scripts.yml b/ansible/roles/es6/tasks/elasticsearch-scripts.yml deleted file mode 100644 index e38c3b4c4d..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-scripts.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- - -- name: set fact es_script_dir - set_fact: es_script_dir={{ es_conf_dir }}/{{es_instance_name}} - tags: - - always - -- name: set fact es_script_dir when path.scripts - set_fact: es_script_dir={{es_config['path.scripts']}} - when: es_config['path.scripts'] is defined - tags: - - always - -- name: Create script dir - become: yes - file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes - -- name: Copy default scripts to elasticsearch - become: yes - copy: src=scripts dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} - when: es_scripts_fileglob is not defined - -- name: Copy scripts to elasticsearch - become: yes - copy: src={{ item }} dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} - with_fileglob: "{{ es_scripts_fileglob | default('') }}" diff --git a/ansible/roles/es6/tasks/elasticsearch-template.yml b/ansible/roles/es6/tasks/elasticsearch-template.yml deleted file mode 100644 index 6059c7d727..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch-template.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- name: ensure templates dir is created - file: - path: /etc/elasticsearch/templates - state: directory - owner: "{{ es_user }}" - group: "{{ es_group }}" - -- name: Copy templates to elasticsearch - copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }} - register: load_templates - with_fileglob: - - "{{ es_templates_fileglob | default('') }}" - -- name: Install templates without auth - uri: - url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}" - method: PUT - status_code: 200 - body_format: json - body: "{{ lookup('file', item) }}" - when: load_templates.changed and es_start_service and not es_enable_xpack or not es_xpack_features is defined or "security" not in es_xpack_features - with_fileglob: - - "{{ es_templates_fileglob | default('') }}" - run_once: True - -- name: Install templates with auth - uri: - url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}" - method: PUT - status_code: 200 - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - body_format: json - body: "{{ lookup('file', item) }}" - when: load_templates.changed and es_start_service and es_enable_xpack and es_xpack_features is defined and "security" in es_xpack_features - with_fileglob: - - "{{ es_templates_fileglob | default('') }}" - run_once: True diff --git a/ansible/roles/es6/tasks/elasticsearch.yml b/ansible/roles/es6/tasks/elasticsearch.yml deleted file mode 100644 index e2361d49bf..0000000000 --- a/ansible/roles/es6/tasks/elasticsearch.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- - -- name: Include optional user and group creation. - when: (es_user_id is defined) and (es_group_id is defined) - include: elasticsearch-optional-user.yml - -- name: Include specific Elasticsearch - include: elasticsearch-Debian.yml - when: ansible_os_family == 'Debian' - -- name: Include specific Elasticsearch - include: elasticsearch-RedHat.yml - when: ansible_os_family == 'RedHat' diff --git a/ansible/roles/es6/tasks/java.yml b/ansible/roles/es6/tasks/java.yml deleted file mode 100644 index a4ecac04a4..0000000000 --- a/ansible/roles/es6/tasks/java.yml +++ /dev/null @@ -1,51 +0,0 @@ ---- - -- name: set fact java_state to present - set_fact: java_state="present" - -- name: set fact java_state to latest - set_fact: java_state="latest" - when: update_java == true - -- name: RedHat - Ensure Java is installed - become: yes - yum: name={{ java }} state={{java_state}} - when: ansible_os_family == 'RedHat' - -- name: Get the installed java path - shell: "update-alternatives --display java | grep '^/' | awk '{print $1}' | grep 1.8.0" - register: java_full_path - failed_when: False - changed_when: False - when: ansible_os_family == 'RedHat' - -- name: correct java version selected - alternatives: - name: java - path: "{{ java_full_path.stdout }}" - link: /usr/bin/java - when: ansible_os_family == 'RedHat' and java_full_path is defined - -- name: Refresh java repo - become: yes - apt: update_cache=yes - changed_when: false - when: ansible_os_family == 'Debian' - -- name: Debian - Ensure Java is installed - become: yes - apt: name={{ java }} state={{java_state}} - when: ansible_os_family == 'Debian' - -- name: register open_jdk version - shell: java -version 2>&1 | grep OpenJDK - register: open_jdk - ignore_errors: yes - changed_when: false - -#https://github.com/docker-library/openjdk/issues/19 - ensures tests pass due to java 8 broken certs -- name: refresh the java ca-certificates - become: yes - command: /var/lib/dpkg/info/ca-certificates-java.postinst configure - when: ansible_distribution == 'Ubuntu' and open_jdk.rc == 0 - changed_when: false diff --git a/ansible/roles/es6/tasks/main.yml b/ansible/roles/es6/tasks/main.yml deleted file mode 100644 index bc6a1ba865..0000000000 --- a/ansible/roles/es6/tasks/main.yml +++ /dev/null @@ -1,84 +0,0 @@ ---- -- name: os-specific vars - include_vars: "{{ansible_os_family}}.yml" - tags: - - always - -- name: check-set-parameters - include: elasticsearch-parameters.yml - tags: - - always - -# - name: include java.yml -# include: java.yml -# when: es_java_install -# tags: -# - java - -- name: include elasticsearch.yml - include: elasticsearch.yml - tags: - - install - -- name: include elasticsearch-config.yml - include: elasticsearch-config.yml - tags: - - config - -- name: include elasticsearch-scripts.yml - include: elasticsearch-scripts.yml - when: es_scripts - tags: - - scripts - -- name: include elasticsearch-plugins.yml - include: elasticsearch-plugins.yml - when: es_plugins is defined or es_plugins_reinstall - tags: - - plugins - - #We always execute xpack as we may need to remove features -- name: include xpack/elasticsearch-xpack.yml - include: xpack/elasticsearch-xpack.yml - tags: - - xpack - -- name: flush handlers - meta: flush_handlers - -- name: Make sure elasticsearch is started - service: name={{instance_init_script | basename}} state=started enabled=yes - when: es_start_service - -- name: Wait for elasticsearch to startup - wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1 - when: es_restarted is defined and es_restarted.changed and es_start_service - -- name: set fact manage_native_realm to false - set_fact: manage_native_realm=false - -- name: set fact manage_native_realm to true - set_fact: manage_native_realm=true - when: es_start_service and (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) - -# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up -- name: Wait 15 seconds for the Native Relm to come up - pause: seconds=15 - when: manage_native_realm - -- name: activate-license - include: ./xpack/security/elasticsearch-xpack-activation.yml - when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != '' - -#perform security actions here now elasticsearch is started -- name: include xpack/security/elasticsearch-security-native.yml - include: ./xpack/security/elasticsearch-security-native.yml - when: manage_native_realm - -#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. -#We also do after the native realm to ensure any changes are applied here first and its denf up. -- name: include elasticsearch-template.yml - include: elasticsearch-template.yml - when: es_templates - tags: - - templates \ No newline at end of file diff --git a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack-install.yml b/ansible/roles/es6/tasks/xpack/elasticsearch-xpack-install.yml deleted file mode 100644 index 522f8161fd..0000000000 --- a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack-install.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- - -#Test if feature is installed -- name: Test if x-pack is installed - shell: "{{es_home}}/bin/elasticsearch-plugin list | grep x-pack" - become: yes - register: x_pack_installed - changed_when: False - failed_when: "'ERROR' in x_pack_installed.stdout" - check_mode: no - ignore_errors: yes - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - - -#Remove X-Pack if installed and its not been requested or the ES version has changed -- name: Remove x-pack plugin - become: yes - command: "{{es_home}}/bin/elasticsearch-plugin remove x-pack" - register: xpack_state - failed_when: "'ERROR' in xpack_state.stdout" - changed_when: xpack_state.rc == 0 - when: x_pack_installed.rc == 0 and (not es_enable_xpack or es_version_changed) - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - - -#Install plugin if not installed, or the es version has changed (so removed above), and its been requested -- name: Download x-pack from url - get_url: url={{ es_xpack_custom_url }} dest=/tmp/x-pack-{{ es_version }}.zip - when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined) - -- name: Install x-pack plugin from local - become: yes - command: > - {{es_home}}/bin/elasticsearch-plugin install --silent --batch file:///tmp/x-pack-{{ es_version }}.zip - register: xpack_state - changed_when: xpack_state.rc == 0 - when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined) - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - -- name: Delete x-pack zip file - file: dest=/tmp/x-pack-{{ es_version }}.zip state=absent - when: es_xpack_custom_url is defined - -- name: Install x-pack plugin from elastic.co - become: yes - command: > - {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack - register: xpack_state - failed_when: "'ERROR' in xpack_state.stdout" - changed_when: xpack_state.rc == 0 - when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is not defined) - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}" diff --git a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml b/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml deleted file mode 100644 index ec82a68a6f..0000000000 --- a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- - -- name: set fact es_version_changed - set_fact: es_version_changed={{ ((elasticsearch_install_from_package is defined and (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) }} - -- name: include elasticsearch-xpack-install.yml - include: elasticsearch-xpack-install.yml - -#Security configuration -- name: include security/elasticsearch-security.yml - include: security/elasticsearch-security.yml - -#Add any feature specific configuration here -- name: Set Plugin Directory Permissions - become: yes - file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes - -#Make sure elasticsearch.keystore has correct Permissions -- name: Set elasticsearch.keystore Permissions - become: yes - file: state=file path={{ conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }} - when: es_enable_xpack and "security" in es_xpack_features and (es_version | version_compare('6.0.0', '>')) diff --git a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-file.yml b/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-file.yml deleted file mode 100644 index 8d6f878995..0000000000 --- a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-file.yml +++ /dev/null @@ -1,80 +0,0 @@ ---- -- name: set fact manage_file_users - set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | length > 0 - -#List current users -- name: List Users - become: yes - shell: cat {{conf_dir}}/x-pack/users | awk -F':' '{print $1}' - register: current_file_users - when: manage_file_users - changed_when: False - -- name: set fact users_to_remove - set_fact: users_to_remove={{ current_file_users.stdout_lines | difference (es_users.file.keys()) }} - when: manage_file_users - -#Remove users -- name: Remove Users - become: yes - command: > - {{es_home}}/bin/x-pack/users userdel {{item}} - with_items: "{{users_to_remove | default([])}}" - when: manage_file_users - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_HOME: "{{es_home}}" - -- name: set fact users_to_add - set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }} - when: manage_file_users - -#Add users -- name: Add Users - become: yes - command: > - {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}} - with_items: "{{ users_to_add | default([]) }}" - when: manage_file_users - no_log: True - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_HOME: "{{es_home}}" - -#Set passwords for all users declared - Required as the useradd will not change existing user passwords -- name: Set User Passwords - become: yes - command: > - {{es_home}}/bin/x-pack/users passwd {{ item }} -p {{es_users.file[item].password}} - with_items: "{{ es_users.file.keys() | default([]) }}" - when: manage_file_users - #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip. - changed_when: False - no_log: True - environment: - CONF_DIR: "{{ conf_dir }}" - ES_PATH_CONF: "{{ conf_dir }}" - ES_HOME: "{{es_home}}" - -- name: set fact users_roles - set_fact: users_roles={{es_users.file | extract_role_users () }} - when: manage_file_users - -#Copy Roles files -- name: Copy roles.yml File for Instance - become: yes - template: src=security/roles.yml.j2 dest={{conf_dir}}/x-pack/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes - when: es_roles is defined and es_roles.file is defined - -#Overwrite users_roles file -- name: Copy User Roles - become: yes - template: src=security/users_roles.j2 dest={{conf_dir}}/x-pack/users_roles mode=0644 force=yes - when: manage_file_users and users_roles | length > 0 - -#Set permission on security directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions. -- name: Set Security Directory Permissions Recursive - become: yes - file: state=directory path={{conf_dir}}/x-pack/ owner={{ es_user }} group={{ es_group }} recurse=yes diff --git a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-native.yml b/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-native.yml deleted file mode 100644 index 63024fadef..0000000000 --- a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-native.yml +++ /dev/null @@ -1,191 +0,0 @@ ---- -- name: set fact change_api_password to false - set_fact: change_api_password=false - -- name: set fact manage_native_users to false - set_fact: manage_native_users=false - -- name: set fact manage_native_users to true - set_fact: manage_native_users=true - when: es_users is defined and es_users.native is defined and es_users.native.keys() | length > 0 - -- name: set fact manage_native_role to false - set_fact: manage_native_roles=false - -- name: set fact manange_native_roles to true - set_fact: manage_native_roles=true - when: es_roles is defined and es_roles.native is defined and es_roles.native.keys() | length > 0 - -#If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load - -#List current users -- name: List Native Users - uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user - method: GET - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - status_code: 200 - register: user_list_response - when: manage_native_users - -- name: set fact reserved_users equals user_list_response.json - set_fact: reserved_users={{ user_list_response.json | filter_reserved }} - when: manage_native_users - -#Current users not inc. those reserved -- name: set fact current_users equals user_list_response.json.keys not including reserved - set_fact: current_users={{ user_list_response.json.keys() | difference (reserved_users) }} - when: manage_native_users - -#We are changing the es_api_basic_auth_username password, so we need to do it first and update the param -- name: set fact native_users - set_fact: native_users={{ es_users.native }} - when: manage_native_users - -- name: set fact change_api_password to true - set_fact: change_api_password=true - when: manage_native_users and es_api_basic_auth_username in native_users and native_users[es_api_basic_auth_username].password is defined - -- name: Update API User Password - uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password - method: POST - body_format: json - body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }" - status_code: 200 - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - when: change_api_password - -- name: set fact es_api_basic_auth_password - set_fact: es_api_basic_auth_password={{native_users[es_api_basic_auth_username].password}} - when: change_api_password - -#Identify users that are present in ES but not declared and thus should be removed -- name: set fact users_to_remove - set_fact: users_to_remove={{ current_users | difference ( native_users.keys() ) }} - when: manage_native_users - -#Delete all non required users NOT inc. reserved -- name: Delete Native Users - uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}} - method: DELETE - status_code: 200 - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - when: manage_native_users - with_items: "{{ users_to_remove | default([]) }}" - -- name: set fact users_to_ignore - set_fact: users_to_ignore={{ native_users.keys() | intersect (reserved_users) }} - when: manage_native_users - -- name: debug message - debug: - msg: "WARNING: YOU CAN ONLY CHANGE THE PASSWORD FOR RESERVED USERS IN THE NATIVE REALM. ANY ROLE CHANGES WILL BE IGNORED: {{users_to_ignore}}" - when: manage_native_users and users_to_ignore | length > 0 - -#Update password on all reserved users -- name: Update Reserved User Passwords - uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password - method: POST - body_format: json - body: "{ \"password\":\"{{native_users[item].password}}\" }" - status_code: 200 - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - when: native_users[item].password is defined - no_log: True - with_items: "{{ users_to_ignore | default([]) }}" - -- name: set fact users_to_modify - set_fact: users_to_modify={{ native_users.keys() | difference (reserved_users) }} - when: manage_native_users - -#Overwrite all other users NOT inc. those reserved -- name: Update Non-Reserved Native User Details - uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}} - method: POST - body_format: json - body: "{{ native_users[item] | to_json }}" - status_code: 200 - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - when: manage_native_users - no_log: True - with_items: "{{ users_to_modify | default([]) }}" - -## ROLE CHANGES - -#List current roles not. inc those reserved -- name: List Native Roles - uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role - method: GET - body_format: json - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - status_code: 200 - register: role_list_response - when: manage_native_roles - -- name: set fact reserved roles - set_fact: reserved_roles={{ role_list_response.json | filter_reserved }} - when: manage_native_roles - -- name: set fact current roles - set_fact: current_roles={{ role_list_response.json.keys() | difference (reserved_roles) }} - when: manage_native_roles - -- name: set fact roles to ignore - set_fact: roles_to_ignore={{ es_roles.native.keys() | intersect (reserved_roles) | default([]) }} - when: manage_native_roles - -- name: debug message - debug: - msg: "WARNING: YOU CANNOT CHANGE RESERVED ROLES. THE FOLLOWING WILL BE IGNORED: {{roles_to_ignore}}" - when: manage_native_roles and roles_to_ignore | length > 0 - -- name: set fact roles_to_remove - set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys() ) }} - when: manage_native_roles - -#Delete all non required roles NOT inc. reserved -- name: Delete Native Roles - uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}} - method: DELETE - status_code: 200 - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - when: manage_native_roles - with_items: "{{roles_to_remove | default([]) }}" - -- name: set fact roles_to_modify - set_fact: roles_to_modify={{ es_roles.native.keys() | difference (reserved_roles) }} - when: manage_native_roles - -#Update other roles - NOT inc. reserved roles -- name: Update Native Roles - uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}} - method: POST - body_format: json - body: "{{ es_roles.native[item] | to_json}}" - status_code: 200 - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - when: manage_native_roles - with_items: "{{ roles_to_modify | default([]) }}" diff --git a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml b/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml deleted file mode 100644 index 14f4d20386..0000000000 --- a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -#Security specific configuration done here - -#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 - -#Ensure x-pack conf directory is created if necessary -- name: Ensure x-pack conf directory exists (file) - file: path={{ conf_dir }}/x-pack state=directory owner={{ es_user }} group={{ es_group }} - changed_when: False - when: - - es_enable_xpack and "security" in es_xpack_features - - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined) - -#-----------------------------Create Bootstrap User----------------------------------- -- name: Check if bootstrap password is set - command: > - {{es_home}}/bin/elasticsearch-keystore list - register: list_keystore - changed_when: False - environment: - ES_PATH_CONF: "{{ conf_dir }}" - when: - - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>')) - -- name: Create Bootstrap password for elastic user - shell: echo "{{es_api_basic_auth_password}}" | {{es_home}}/bin/elasticsearch-keystore add -x 'bootstrap.password' - when: - - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>')) and es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines - environment: - ES_PATH_CONF: "{{ conf_dir }}" - no_log: true - -#-----------------------------FILE BASED REALM---------------------------------------- - -- include: elasticsearch-security-file.yml - when: (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined)) - -#-----------------------------ROLE MAPPING ---------------------------------------- - -#Copy Roles files -- name: Copy role_mapping.yml File for Instance - become: yes - template: src=security/role_mapping.yml.j2 dest={{conf_dir}}/x-pack/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes - when: es_role_mapping is defined - -#-----------------------------AUTH FILE---------------------------------------- - -- name: Copy message auth key to elasticsearch - become: yes - copy: src={{ es_message_auth_file }} dest={{conf_dir}}/x-pack/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes - when: es_message_auth_file is defined - -#------------------------------------------------------------------------------------ - -#Ensure security conf directory is created -- name: Ensure security conf directory exists - become: yes - file: path={{ conf_dir }}/security state=directory owner={{ es_user }} group={{ es_group }} - changed_when: False - when: es_enable_xpack and "security" in es_xpack_features diff --git a/ansible/roles/es6/tasks/xpack/security/elasticsearch-xpack-activation.yml b/ansible/roles/es6/tasks/xpack/security/elasticsearch-xpack-activation.yml deleted file mode 100644 index cd72d6a7d2..0000000000 --- a/ansible/roles/es6/tasks/xpack/security/elasticsearch-xpack-activation.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- - -- name: Activate ES license (without security authentication) - uri: - method: PUT - url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true" - body_format: json - body: "{{ es_xpack_license }}" - return_content: yes - register: license_activated - no_log: True - when: not "security" in es_xpack_features - failed_when: > - license_activated.status != 200 or - license_activated.json.license_status is not defined or - license_activated.json.license_status != 'valid' - -- name: Activate ES license (with security authentication) - uri: - method: PUT - url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true" - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - body_format: json - force_basic_auth: yes - body: "{{ es_xpack_license }}" - return_content: yes - register: license_activated - no_log: True - when: "'security' in es_xpack_features" - failed_when: > - license_activated.status != 200 or - license_activated.json.license_status is not defined or - license_activated.json.license_status != 'valid' - -- debug: - msg: "License: {{ license_activated }}" diff --git a/ansible/roles/es6/templates/elasticsearch.j2 b/ansible/roles/es6/templates/elasticsearch.j2 deleted file mode 100644 index 5bf5746f93..0000000000 --- a/ansible/roles/es6/templates/elasticsearch.j2 +++ /dev/null @@ -1,83 +0,0 @@ -################################ -# Elasticsearch -################################ - -# Elasticsearch home directory -ES_HOME={{es_home}} - -# Elasticsearch Java path -#JAVA_HOME= - -# Elasticsearch configuration directory -CONF_DIR={{conf_dir}} -ES_PATH_CONF={{conf_dir}} - -# Elasticsearch data directory -DATA_DIR={{ data_dirs | array_to_str }} - -# Elasticsearch logs directory -LOG_DIR={{log_dir}} - -# Elasticsearch PID directory -PID_DIR={{pid_dir}} - -ES_JVM_OPTIONS={{conf_dir}}/jvm.options - -# Configure restart on package upgrade (true, every other setting will lead to not restarting) -#ES_RESTART_ON_UPGRADE=true - -# Path to the GC log file -#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log - -################################ -# Elasticsearch service -################################ - -# SysV init.d -# -# When executing the init script, this user will be used to run the elasticsearch service. -# The default value is 'elasticsearch' and is declared in the init.d file. -# Note that this setting is only used by the init script. If changed, make sure that -# the configured user can read and write into the data, work, plugins and log directories. -# For systemd service, the user is usually configured in file /usr/lib/systemd/system/elasticsearch.service -ES_USER={{es_user}} -ES_GROUP={{es_group}} - -# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process -ES_STARTUP_SLEEP_TIME=5 - -################################ -# System properties -################################ - -# Specifies the maximum file descriptor number that can be opened by this process -# When using Systemd, this setting is ignored and the LimitNOFILE defined in -# /usr/lib/systemd/system/elasticsearch.service takes precedence -{% if es_max_open_files is defined %} -#MAX_OPEN_FILES -MAX_OPEN_FILES={{es_max_open_files}} -{% endif %} - -# The maximum number of bytes of memory that may be locked into RAM -# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option -# in elasticsearch.yml -# When using Systemd, the LimitMEMLOCK property must be set -# in /usr/lib/systemd/system/elasticsearch.service -#MAX_LOCKED_MEMORY= -{% if m_lock_enabled %} -MAX_LOCKED_MEMORY=unlimited -{% endif %} - -# Maximum number of VMA (Virtual Memory Areas) a process can own -# When using Systemd, this setting is ignored and the 'vm.max_map_count' -# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf -#MAX_MAP_COUNT=262144 -{% if es_max_map_count is defined %} -MAX_MAP_COUNT={{es_max_map_count}} -{% endif %} - -# Specifies the maximum number of threads that can be started. -# Elasticsearch requires a minimum of 2048. -{% if es_max_threads is defined %} -MAX_THREADS={{ es_max_threads }} -{% endif %} diff --git a/ansible/roles/es6/templates/elasticsearch.repo b/ansible/roles/es6/templates/elasticsearch.repo deleted file mode 100644 index b6299042bd..0000000000 --- a/ansible/roles/es6/templates/elasticsearch.repo +++ /dev/null @@ -1,11 +0,0 @@ -[elasticsearch-{{ es_major_version }}] -name=Elasticsearch repository for {{ es_major_version }} packages -baseurl=https://artifacts.elastic.co/packages/{{ es_major_version }}/yum -gpgcheck=1 -gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch -enabled=1 -autorefresh=1 -type=rpm-md -{% if es_proxy_host is defined and es_proxy_host != '' and es_proxy_port is defined %} -proxy=http://{{ es_proxy_host }}:{{es_proxy_port}} -{% endif %} diff --git a/ansible/roles/es6/templates/elasticsearch.yml.j2 b/ansible/roles/es6/templates/elasticsearch.yml.j2 deleted file mode 100644 index dcf6f7aef1..0000000000 --- a/ansible/roles/es6/templates/elasticsearch.yml.j2 +++ /dev/null @@ -1,52 +0,0 @@ - -{% if es_config %} -{{ es_config | to_nice_yaml }} -{% endif %} - -{% if es_config['cluster.name'] is not defined %} -cluster.name: elasticsearch -{% endif %} - -{% if es_config['node.name'] is not defined %} -node.name: {{inventory_hostname}}-{{es_instance_name}} -{% endif %} - -#################################### Paths #################################### - -# Path to directory containing configuration (this file and logging.yml): - -{% if (es_version | version_compare('6.0.0', '<')) %} -path.conf: {{ conf_dir }} -{% endif %} - -path.data: {{ data_dirs | array_to_str }} - -path.logs: {{ log_dir }} - -{% if es_enable_xpack %} -{% if not "security" in es_xpack_features %} -xpack.security.enabled: false -{% endif %} - -{% if not "monitoring" in es_xpack_features %} -xpack.monitoring.enabled: false -{% endif %} - -{% if not "alerting" in es_xpack_features %} -xpack.watcher.enabled: false -{% endif %} - -{% if not "ml" in es_xpack_features %} -xpack.ml.enabled: false -{% endif %} - -{% if not "graph" in es_xpack_features %} -xpack.graph.enabled: false -{% endif %} -{% endif %} - -network.host: 0.0.0.0 - -{% if es_remote_reindex is defined %} -reindex.remote.whitelist: {{es_remote_host}}:9200 -{% endif %} diff --git a/ansible/roles/es6/templates/init/debian/elasticsearch.j2 b/ansible/roles/es6/templates/init/debian/elasticsearch.j2 deleted file mode 100755 index efe2c37f6c..0000000000 --- a/ansible/roles/es6/templates/init/debian/elasticsearch.j2 +++ /dev/null @@ -1,229 +0,0 @@ -#!/bin/bash -# -# /etc/init.d/elasticsearch -- startup script for Elasticsearch -# -### BEGIN INIT INFO -# Provides: elasticsearch -# Required-Start: $network $remote_fs $named -# Required-Stop: $network $remote_fs $named -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Starts elasticsearch -# Description: Starts elasticsearch using start-stop-daemon -### END INIT INFO - -PATH=/bin:/usr/bin:/sbin:/usr/sbin -NAME={{es_instance_name}}_{{default_file | basename}} -{% if es_config['node.name'] is defined %} -DESC="Elasticsearch Server - {{es_config['node.name']}}" -{% else %} -DESC="Elasticsearch Server - {{es_instance_name}}" -{% endif %} - -DEFAULT=/etc/default/$NAME - -if [ `id -u` -ne 0 ]; then - echo "You need root privileges to run this script" - exit 1 -fi - -. /lib/lsb/init-functions -if [ -r /etc/default/rcS ]; then - . /etc/default/rcS -fi - -# The following variables can be overwritten in $DEFAULT - -# Run Elasticsearch as this user ID and group ID -ES_USER={{es_user}} -ES_GROUP={{es_group}} - -# Directory where the Elasticsearch binary distribution resides -ES_HOME={{es_home}} - -# Maximum number of open files -{% if es_max_open_files is defined %} -MAX_OPEN_FILES={{es_max_open_files}} -{% endif %} - -# Maximum amount of locked memory -#MAX_LOCKED_MEMORY= -{% if m_lock_enabled %} -MAX_LOCKED_MEMORY=unlimited -{% endif %} - -# Elasticsearch log directory -LOG_DIR={{log_dir}} - -# Elasticsearch data directory -DATA_DIR={{ data_dirs | array_to_str }} - -# Elasticsearch configuration directory -CONF_DIR={{conf_dir}} -ES_PATH_CONF={{ conf_dir }} - -# Maximum number of VMA (Virtual Memory Areas) a process can own -{% if es_max_map_count is defined %} -MAX_MAP_COUNT={{es_max_map_count}} -{% endif %} - -# Elasticsearch PID file directory -PID_DIR={{pid_dir}} - -ES_JVM_OPTIONS="{{conf_dir}}/jvm.options" - -# End of variables that can be overwritten in $DEFAULT - -# overwrite settings from default file -if [ -f "$DEFAULT" ]; then - . "$DEFAULT" -fi - -# CONF_FILE setting was removed -if [ ! -z "$CONF_FILE" ]; then - echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed." - exit 1 -fi - -if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then - echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]" -fi - -# Define other required variables -PID_FILE="$PID_DIR/$NAME.pid" -DAEMON=$ES_HOME/bin/elasticsearch -{% if (es_version | version_compare('6.0.0', '<')) %} -DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR" -{% else %} -DAEMON_OPTS="-d -p $PID_FILE" -{% endif %} - -export ES_JAVA_OPTS -export JAVA_HOME -export ES_INCLUDE -export ES_JVM_OPTIONS -export ES_PATH_CONF - -# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported -if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi -if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi -if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi -if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi -if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi -if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi -if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi -if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi - -# Check DAEMON exists -if [ ! -x "$DAEMON" ]; then - echo "The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON" - exit 1 -fi - -checkJava() { - if [ -x "$JAVA_HOME/bin/java" ]; then - JAVA="$JAVA_HOME/bin/java" - else - JAVA=`which java` - fi - - if [ ! -x "$JAVA" ]; then - echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME" - exit 1 - fi -} - -case "$1" in - start) - checkJava - - log_daemon_msg "Starting $DESC" - - pid=`pidofproc -p $PID_FILE elasticsearch` - if [ -n "$pid" ] ; then - log_begin_msg "Already running." - log_end_msg 0 - exit 0 - fi - - # Ensure that the PID_DIR exists (it is cleaned at OS startup time) - if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then - mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR" - fi - if [ -n "$PID_FILE" ] && [ ! -e "$PID_FILE" ]; then - touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE" - fi - - if [ -n "$MAX_OPEN_FILES" ]; then - ulimit -n $MAX_OPEN_FILES - fi - - if [ -n "$MAX_LOCKED_MEMORY" ]; then - ulimit -l $MAX_LOCKED_MEMORY - fi - - if [ -n "$MAX_THREADS" ]; then - ulimit -u $MAX_THREADS - fi - - if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then - sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT - fi - - # Start Daemon - start-stop-daemon -d $ES_HOME --start --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS - return=$? - if [ $return -eq 0 ]; then - i=0 - timeout={{es_debian_startup_timeout}} - # Wait for the process to be properly started before exiting - until { kill -0 `cat "$PID_FILE"`; } >/dev/null 2>&1 - do - sleep 1 - i=$(($i + 1)) - if [ $i -gt $timeout ]; then - log_end_msg 1 - exit 1 - fi - done - fi - log_end_msg $return - exit $return - ;; - stop) - log_daemon_msg "Stopping $DESC" - - if [ -f "$PID_FILE" ]; then - start-stop-daemon --stop --pidfile "$PID_FILE" \ - --user "$ES_USER" \ - --quiet \ - --retry forever/TERM/20 > /dev/null - if [ $? -eq 1 ]; then - log_progress_msg "$DESC is not running but pid file exists, cleaning up" - elif [ $? -eq 3 ]; then - PID="`cat $PID_FILE`" - log_failure_msg "Failed to stop $DESC (pid $PID)" - exit 1 - fi - rm -f "$PID_FILE" - else - log_progress_msg "(not running)" - fi - log_end_msg 0 - ;; - status) - status_of_proc -p $PID_FILE elasticsearch elasticsearch && exit 0 || exit $? - ;; - restart|force-reload) - if [ -f "$PID_FILE" ]; then - $0 stop - fi - $0 start - ;; - *) - log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}" - exit 1 - ;; -esac - -exit 0 diff --git a/ansible/roles/es6/templates/init/redhat/elasticsearch.j2 b/ansible/roles/es6/templates/init/redhat/elasticsearch.j2 deleted file mode 100755 index c993e14365..0000000000 --- a/ansible/roles/es6/templates/init/redhat/elasticsearch.j2 +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/bash -# -# elasticsearch -# -# chkconfig: 2345 80 20 -# description: Starts and stops a single elasticsearch instance on this system -# - -### BEGIN INIT INFO -# Provides: Elasticsearch -# Required-Start: $network $named -# Required-Stop: $network $named -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: This service manages the elasticsearch daemon -# Description: Elasticsearch is a very scalable, schema-free and high-performance search solution supporting multi-tenancy and near realtime search. -### END INIT INFO - -# -# init.d / servicectl compatibility (openSUSE) -# -if [ -f /etc/rc.status ]; then - . /etc/rc.status - rc_reset -fi - -# -# Source function library. -# -if [ -f /etc/rc.d/init.d/functions ]; then - . /etc/rc.d/init.d/functions -fi - -# Sets the default values for elasticsearch variables used in this script -ES_USER="{{es_user}}" -ES_GROUP="{{es_group}}" -ES_HOME="{{es_home}}" -{% if es_max_open_files is defined %} -MAX_OPEN_FILES={{es_max_open_files}} -{% endif %} -# Maximum number of VMA (Virtual Memory Areas) a process can own -{% if es_max_map_count is defined %} -MAX_MAP_COUNT={{es_max_map_count}} -{% endif %} - -LOG_DIR="{{log_dir}}" -DATA_DIR={{ data_dirs | array_to_str }} -CONF_DIR="{{conf_dir}}" -ES_PATH_CONF="{{ conf_dir }}" - -PID_DIR="{{pid_dir}}" - -# Source the default env file -ES_ENV_FILE="{{instance_default_file}}" -if [ -f "$ES_ENV_FILE" ]; then - . "$ES_ENV_FILE" -fi - -if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then - echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]" -fi - -# CONF_FILE setting was removed -if [ ! -z "$CONF_FILE" ]; then - echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed." - exit 1 -fi - -exec="$ES_HOME/bin/elasticsearch" -prog="{{es_instance_name}}_{{default_file | basename}}" -pidfile="$PID_DIR/${prog}.pid" - -export ES_JAVA_OPTS -export JAVA_HOME -export ES_INCLUDE -export ES_JVM_OPTIONS -export ES_STARTUP_SLEEP_TIME -export ES_PATH_CONF - -# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported -if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi -if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi -if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi -if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi -if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi -if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi -if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi -if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi - -lockfile=/var/lock/subsys/$prog - -# backwards compatibility for old config sysconfig files, pre 0.90.1 -if [ -n $USER ] && [ -z $ES_USER ] ; then - ES_USER=$USER -fi - -if [ ! -x "$exec" ]; then - echo "The elasticsearch startup script does not exists or it is not executable, tried: $exec" - exit 1 -fi - -checkJava() { - if [ -x "$JAVA_HOME/bin/java" ]; then - JAVA="$JAVA_HOME/bin/java" - else - JAVA=`which java` - fi - - if [ ! -x "$JAVA" ]; then - echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME" - exit 1 - fi -} - -start() { - checkJava - [ -x $exec ] || exit 5 - - if [ -n "$MAX_OPEN_FILES" ]; then - ulimit -n $MAX_OPEN_FILES - fi - if [ -n "$MAX_LOCKED_MEMORY" ]; then - ulimit -l $MAX_LOCKED_MEMORY - fi - if [ -n "$MAX_THREADS" ]; then - ulimit -u $MAX_THREADS - fi - if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then - sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT - fi - - # Ensure that the PID_DIR exists (it is cleaned at OS startup time) - if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then - mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR" - fi - if [ -n "$pidfile" ] && [ ! -e "$pidfile" ]; then - touch "$pidfile" && chown "$ES_USER":"$ES_GROUP" "$pidfile" - fi - - cd $ES_HOME - echo -n $"Starting $prog: " - # if not running, start it up here, usually something like "daemon $exec" -{% if (es_version | version_compare('6.0.0', '<')) %} - daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR -{% else %} - daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -{% endif %} - retval=$? - echo - [ $retval -eq 0 ] && touch $lockfile - return $retval -} - -stop() { - echo -n $"Stopping $prog: " - # stop it here, often "killproc $prog" - killproc -p $pidfile -d 86400 $prog - retval=$? - echo - [ $retval -eq 0 ] && rm -f $lockfile - return $retval -} - -restart() { - stop - start -} - -reload() { - restart -} - -force_reload() { - restart -} - -rh_status() { - # run checks to determine if the service is running or use generic status - status -p $pidfile $prog -} - -rh_status_q() { - rh_status >/dev/null 2>&1 -} - - -case "$1" in - start) - rh_status_q && exit 0 - $1 - ;; - stop) - rh_status_q || exit 0 - $1 - ;; - restart) - $1 - ;; - reload) - rh_status_q || exit 7 - $1 - ;; - force-reload) - force_reload - ;; - status) - rh_status - ;; - condrestart|try-restart) - rh_status_q || exit 0 - restart - ;; - *) - echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" - exit 2 -esac -exit $? diff --git a/ansible/roles/es6/templates/jvm.options.j2 b/ansible/roles/es6/templates/jvm.options.j2 deleted file mode 100644 index ad30851a2b..0000000000 --- a/ansible/roles/es6/templates/jvm.options.j2 +++ /dev/null @@ -1,118 +0,0 @@ -## JVM configuration - -################################################################ -## IMPORTANT: JVM heap size -################################################################ -## -## You should always set the min and max JVM heap -## size to the same value. For example, to set -## the heap to 4 GB, set: -## -## -Xms4g -## -Xmx4g -## -## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html -## for more information -## -################################################################ - -# Xms represents the initial size of total heap space -# Xmx represents the maximum size of total heap space -{% if es_heap_size is defined %} --Xms{{ es_heap_size }} --Xmx{{ es_heap_size }} -{% else %} --Xms2g --Xmx2g -{% endif %} - -################################################################ -## Expert settings -################################################################ -## -## All settings below this section are considered -## expert settings. Don't tamper with them unless -## you understand what you are doing -## -################################################################ - -## GC configuration --XX:+UseConcMarkSweepGC --XX:CMSInitiatingOccupancyFraction=75 --XX:+UseCMSInitiatingOccupancyOnly - -## optimizations - -# pre-touch memory pages used by the JVM during initialization --XX:+AlwaysPreTouch - -## basic - -# force the server VM --server - -# set to headless, just in case --Djava.awt.headless=true - -# ensure UTF-8 encoding by default (e.g. filenames) --Dfile.encoding=UTF-8 - -# use our provided JNA always versus the system one --Djna.nosys=true - -# use old-style file permissions on JDK9 --Djdk.io.permissionsUseCanonicalPath=true - -# flags to configure Netty --Dio.netty.noUnsafe=true --Dio.netty.noKeySetOptimization=true --Dio.netty.recycler.maxCapacityPerThread=0 - -# log4j 2 --Dlog4j.shutdownHookEnabled=false --Dlog4j2.disable.jmx=true --Dlog4j.skipJansi=true - -## heap dumps - -# generate a heap dump when an allocation from the Java heap fails -# heap dumps are created in the working directory of the JVM --XX:+HeapDumpOnOutOfMemoryError - -# specify an alternative path for heap dumps -# ensure the directory exists and has sufficient space -#-XX:HeapDumpPath=${heap.dump.path} - -## GC logging - -#-XX:+PrintGCDetails -#-XX:+PrintGCTimeStamps -#-XX:+PrintGCDateStamps -#-XX:+PrintClassHistogram -#-XX:+PrintTenuringDistribution -#-XX:+PrintGCApplicationStoppedTime - -# log GC status to a file with time stamps -# ensure the directory exists -#-Xloggc:${loggc} - - -# By default, the GC log file will not rotate. -# By uncommenting the lines below, the GC log file -# will be rotated every 128MB at most 32 times. -#-XX:+UseGCLogFileRotation -#-XX:NumberOfGCLogFiles=32 -#-XX:GCLogFileSize=128M - -# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. -# If documents were already indexed with unquoted fields in a previous version -# of Elasticsearch, some operations may throw errors. -# -# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided -# only for migration purposes. -#-Delasticsearch.json.allow_unquoted_field_names=true -{% if es_jvm_custom_parameters !='' %} -{% for item in es_jvm_custom_parameters %} -{{ item }} -{% endfor %} -{% endif %} diff --git a/ansible/roles/es6/templates/log4j2.properties.j2 b/ansible/roles/es6/templates/log4j2.properties.j2 deleted file mode 100644 index 269be5290b..0000000000 --- a/ansible/roles/es6/templates/log4j2.properties.j2 +++ /dev/null @@ -1,117 +0,0 @@ -status = error - -# log action execution errors for easier debugging -logger.action.name = org.elasticsearch.action -logger.action.level = debug - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - -appender.rolling.type = RollingFile -appender.rolling.name = rolling -{% if (es_version | version_compare('6.0.0', '<')) %} -appender.rolling.fileName = ${sys:es.logs}.log -{% else %} -appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log -{% endif %} -appender.rolling.layout.type = PatternLayout -appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n -{% if (es_version | version_compare('6.0.0', '<')) %} -appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log -{% else %} -appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz -{% endif %} -appender.rolling.policies.type = Policies -appender.rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.rolling.policies.time.interval = 1 -appender.rolling.policies.time.modulate = true -{% if (es_version | version_compare('6.0.0', '>')) %} -appender.rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.rolling.policies.size.size = 128MB -appender.rolling.strategy.type = DefaultRolloverStrategy -appender.rolling.strategy.fileIndex = nomax -appender.rolling.strategy.action.type = Delete -appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} -appender.rolling.strategy.action.condition.type = IfFileName -appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* -appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize -appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB -{% endif %} -rootLogger.level = info -rootLogger.appenderRef.console.ref = console -rootLogger.appenderRef.rolling.ref = rolling - -appender.deprecation_rolling.type = RollingFile -appender.deprecation_rolling.name = deprecation_rolling -{% if (es_version | version_compare('6.0.0', '<')) %} -appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log -{% else %} -appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log -{% endif %} -appender.deprecation_rolling.layout.type = PatternLayout -appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n -{% if (es_version | version_compare('6.0.0', '<')) %} -appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz -{% else %} -appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz -{% endif %} -appender.deprecation_rolling.policies.type = Policies -appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.deprecation_rolling.policies.size.size = 1GB -appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy -appender.deprecation_rolling.strategy.max = 4 - -logger.deprecation.name = org.elasticsearch.deprecation -logger.deprecation.level = warn -logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.additivity = false - -appender.index_search_slowlog_rolling.type = RollingFile -appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -{% if (es_version | version_compare('6.0.0', '<')) %} -appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log -{% else %} -appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log -{% endif %} -appender.index_search_slowlog_rolling.layout.type = PatternLayout -appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n -{% if (es_version | version_compare('6.0.0', '<')) %} -appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log -{% else %} -appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log -{% endif %} -appender.index_search_slowlog_rolling.policies.type = Policies -appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.index_search_slowlog_rolling.policies.time.interval = 1 -appender.index_search_slowlog_rolling.policies.time.modulate = true - -logger.index_search_slowlog_rolling.name = index.search.slowlog -logger.index_search_slowlog_rolling.level = trace -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling -logger.index_search_slowlog_rolling.additivity = false - -appender.index_indexing_slowlog_rolling.type = RollingFile -appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -{% if (es_version | version_compare('6.0.0', '<')) %} -appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log -{% else %} -appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log -{% endif %} -appender.index_indexing_slowlog_rolling.layout.type = PatternLayout -appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n -{% if (es_version | version_compare('6.0.0', '<')) %} -appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log -{% else %} -appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log -{% endif %} -appender.index_indexing_slowlog_rolling.policies.type = Policies -appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.index_indexing_slowlog_rolling.policies.time.interval = 1 -appender.index_indexing_slowlog_rolling.policies.time.modulate = true - -logger.index_indexing_slowlog.name = index.indexing.slowlog.index -logger.index_indexing_slowlog.level = trace -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling -logger.index_indexing_slowlog.additivity = false diff --git a/ansible/roles/es6/templates/security/role_mapping.yml.j2 b/ansible/roles/es6/templates/security/role_mapping.yml.j2 deleted file mode 100644 index 2584375226..0000000000 --- a/ansible/roles/es6/templates/security/role_mapping.yml.j2 +++ /dev/null @@ -1 +0,0 @@ -{{ es_role_mapping | to_nice_yaml }} \ No newline at end of file diff --git a/ansible/roles/es6/templates/security/roles.yml.j2 b/ansible/roles/es6/templates/security/roles.yml.j2 deleted file mode 100644 index 9f211f2b07..0000000000 --- a/ansible/roles/es6/templates/security/roles.yml.j2 +++ /dev/null @@ -1 +0,0 @@ -{{ es_roles.file | to_nice_yaml }} \ No newline at end of file diff --git a/ansible/roles/es6/templates/security/users_roles.j2 b/ansible/roles/es6/templates/security/users_roles.j2 deleted file mode 100644 index 1c0acfa1d7..0000000000 --- a/ansible/roles/es6/templates/security/users_roles.j2 +++ /dev/null @@ -1 +0,0 @@ -{{users_roles | join("\n") }} \ No newline at end of file diff --git a/ansible/roles/es6/templates/systemd/elasticsearch.j2 b/ansible/roles/es6/templates/systemd/elasticsearch.j2 deleted file mode 100644 index 8bd5545c02..0000000000 --- a/ansible/roles/es6/templates/systemd/elasticsearch.j2 +++ /dev/null @@ -1,76 +0,0 @@ -[Unit] -Description=Elasticsearch-{{es_instance_name}} -Documentation=http://www.elastic.co -Wants=network-online.target -After=network-online.target - -[Service] -Environment=ES_HOME={{es_home}} -Environment=CONF_DIR={{conf_dir}} -Environment=ES_PATH_CONF={{conf_dir}} -Environment=DATA_DIR={{ data_dirs | array_to_str }} -Environment=LOG_DIR={{log_dir}} -Environment=PID_DIR={{pid_dir}} -EnvironmentFile=-{{instance_default_file}} - -WorkingDirectory={{es_home}} - -User={{es_user}} -Group={{es_group}} - -{% if (es_version | version_compare('6.0.0', '<')) %} -ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec -{% endif %} - -ExecStart={{es_home}}/bin/elasticsearch \ - -p ${PID_DIR}/elasticsearch.pid \ -{% if (es_version | version_compare('6.0.0', '<')) %} - -Edefault.path.logs=${LOG_DIR} \ - -Edefault.path.data=${DATA_DIR} \ - -Edefault.path.conf=${CONF_DIR} \ -{% endif %} - --quiet - - -# StandardOutput is configured to redirect to journalctl since -# some error messages may be logged in standard output before -# elasticsearch logging system is initialized. Elasticsearch -# stores its logs in /var/log/elasticsearch and does not use -# journalctl by default. If you also want to enable journalctl -# logging, you can simply remove the "quiet" option from ExecStart. -StandardOutput=journal -StandardError=inherit - -# Specifies the maximum file descriptor number that can be opened by this process -{% if es_max_open_files is defined %} -LimitNOFILE={{es_max_open_files}} -{% endif %} - -# Specifies the maximum number of bytes of memory that may be locked into RAM -# Set to "infinity" if you use the 'bootstrap.memory_lock: true' option -# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}} -{% if m_lock_enabled %} -LimitMEMLOCK=infinity -{% endif %} - -# Specifies the maximum number of threads that can be started. Elasticsearch requires a -# minimum of 2048. -LimitNPROC={{ es_max_threads }} - -# Disable timeout logic and wait until process is stopped -TimeoutStopSec=0 - -# SIGTERM signal is used to stop the Java process -KillSignal=SIGTERM - -# Send the signal only to the JVM rather than its control group -KillMode=process - -# Java process is never killed -SendSIGKILL=no - -# When a JVM receives a SIGTERM signal it exits with code 143 -SuccessExitStatus=143 - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/es6/vars/Debian.yml b/ansible/roles/es6/vars/Debian.yml deleted file mode 100644 index 071736ef70..0000000000 --- a/ansible/roles/es6/vars/Debian.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -java: "{% if es_java is defined %}{{es_java}}{% else %}openjdk-8-jre-headless{% endif %}" -default_file: "/etc/default/elasticsearch" -es_home: "/usr/share/elasticsearch" diff --git a/ansible/roles/es6/vars/RedHat.yml b/ansible/roles/es6/vars/RedHat.yml deleted file mode 100644 index b0aa42b2bb..0000000000 --- a/ansible/roles/es6/vars/RedHat.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}" -default_file: "/etc/sysconfig/elasticsearch" -es_home: "/usr/share/elasticsearch" \ No newline at end of file diff --git a/ansible/roles/es6/vars/main.yml b/ansible/roles/es6/vars/main.yml deleted file mode 100644 index b3bd126711..0000000000 --- a/ansible/roles/es6/vars/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -es_package_url: "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch" -es_conf_dir: "/etc/elasticsearch" -sysd_script: "/usr/lib/systemd/system/elasticsearch.service" -init_script: "/etc/init.d/elasticsearch" -#add supported features here -supported_xpack_features: ["alerting","monitoring","graph","security"] -reserved_xpack_users: ["elastic","kibana","logstash_system"] \ No newline at end of file diff --git a/ansible/roles/graphite-exporter/README.md b/ansible/roles/graphite-exporter/README.md new file mode 100644 index 0000000000..7dc55d1b67 --- /dev/null +++ b/ansible/roles/graphite-exporter/README.md @@ -0,0 +1,16 @@ +## Graphite exporter as systemd + +This role will install graphite exporter as systemd process in VM for applications which emit metrics in graphite format to injest the metrics in to prometheus. + +### Applications + +1. Druid: + 1. Enable druid [metrics](https://github.com/apache/druid/blob/master/docs/operations/metrics.md) in `/data/druid/whitelist` + ref: + 2. Add below config in `/data/druid/conf/druid/_common/common.runtime.properties` + ``` + druid.emitter.graphite.port=9109 + druid.emitter.graphite.hostname=localhost + druid.emitter.graphite.protocol=plaintext + ``` + 3. Add scrape config in prometheus to scrape `this_machine_ip:9108` port diff --git a/ansible/roles/graphite-exporter/defaults/main.yml b/ansible/roles/graphite-exporter/defaults/main.yml new file mode 100644 index 0000000000..6de223f0de --- /dev/null +++ b/ansible/roles/graphite-exporter/defaults/main.yml @@ -0,0 +1,12 @@ +--- + +graphite_exporter_version: 0.7.1 + +graphite_exporter_binary_url: 'https://github.com/prometheus/graphite_exporter/releases/download/v{{graphite_exporter_version}}/graphite_exporter-{{graphite_exporter_version}}.linux-amd64.tar.gz' + +graphite_exporter_user: graphite-exp +graphite_exporter_group: graphite-exp + +graphite_exporter_root_dir: /opt/graphite_exporter +graphite_exporter_dist_dir: "{{ graphite_exporter_root_dir }}/dist" +graphite_exporter_config_dir: "{{ graphite_exporter_root_dir }}/config" diff --git a/ansible/roles/graphite-exporter/handlers/main.yml b/ansible/roles/graphite-exporter/handlers/main.yml new file mode 100644 index 0000000000..cee21aab37 --- /dev/null +++ b/ansible/roles/graphite-exporter/handlers/main.yml @@ -0,0 +1,15 @@ +--- + +- name: reenable graphite exporter service + systemd: + daemon_reload: yes + enabled: yes + name: prometheus-graphite-exporter + +- name: reinit graphite exporter + command: initctl reload-configuration + +- name: restart graphite exporter + service: + name: prometheus-graphite-exporter + state: restarted diff --git a/ansible/roles/graphite-exporter/tasks/main.yml b/ansible/roles/graphite-exporter/tasks/main.yml new file mode 100644 index 0000000000..7edfe23dde --- /dev/null +++ b/ansible/roles/graphite-exporter/tasks/main.yml @@ -0,0 +1,71 @@ +--- + +- name: 'create druid exporter group' + group: + name: '{{ graphite_exporter_group }}' + system: yes + state: present + +- name: 'create graphite exporter user' + user: + name: '{{ graphite_exporter_user }}' + system: yes + shell: '/sbin/nologin' + group: '{{ graphite_exporter_group }}' + createhome: no + +- name: 'create graphite exporter directories' + file: + path: '{{ item }}' + state: directory + owner: '{{ graphite_exporter_user }}' + group: '{{ graphite_exporter_group }}' + mode: 0755 + with_items: + - '{{ graphite_exporter_root_dir }}' + - '{{ graphite_exporter_dist_dir }}' + - '{{ graphite_exporter_config_dir }}' + +- name: download and unarchive graphite exporter binary + unarchive: + src: '{{ graphite_exporter_binary_url }}' + dest: '{{ graphite_exporter_dist_dir }}/' + mode: 0755 + remote_src: yes + +- name: 'Copy graphite exporter config' + template: + src: config.j2 + dest: '{{ graphite_exporter_config_dir }}/graphite_mapping.conf' + notify: + - restart graphite exporter + +- name: 'update group and owner for files' + file: + path: '{{ item }}' + state: file + owner: '{{ graphite_exporter_user }}' + group: '{{ graphite_exporter_group }}' + with_items: + - '{{ graphite_exporter_dist_dir }}/graphite_exporter-{{graphite_exporter_version}}.linux-amd64/graphite_exporter' + - '{{ graphite_exporter_config_dir }}/graphite_mapping.conf' + +- name: 'create systemd service unit' + template: + src: prometheus-graphite-exporter.service.j2 + dest: /etc/systemd/system/prometheus-graphite-exporter.service + owner: root + group: root + mode: 0644 + when: service_mgr | default(ansible_service_mgr) == 'systemd' + notify: + - reenable graphite exporter service + - restart graphite exporter + +- meta: flush_handlers + +- name: 'ensure prometheus graphite exporter service is enabled and started' + service: + name: prometheus-graphite-exporter + state: started + enabled: yes diff --git a/ansible/roles/graphite-exporter/templates/config.j2 b/ansible/roles/graphite-exporter/templates/config.j2 new file mode 100644 index 0000000000..36069ed49b --- /dev/null +++ b/ansible/roles/graphite-exporter/templates/config.j2 @@ -0,0 +1,21 @@ +# This config is based on observed metrics from druid +# For this to work in druid /data/druid/conf/druid/_common/common.runtime.properties + +mappings: +- match: druid.metric.*.*.* + name: druid_$3 + labels: + role: $1 + ip: $2 +- match: druid.metric.*.*.*.* + name: druid_$4 + labels: + role: $1 + ip: $2 + job: $3 +- match: druid.metric.*.*.*.*.* + name: druid_${4}_${5} + labels: + role: $1 + ip: $2 + job: $3 diff --git a/ansible/roles/graphite-exporter/templates/prometheus-graphite-exporter.service.j2 b/ansible/roles/graphite-exporter/templates/prometheus-graphite-exporter.service.j2 new file mode 100644 index 0000000000..28435d0289 --- /dev/null +++ b/ansible/roles/graphite-exporter/templates/prometheus-graphite-exporter.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Prometheus graphite Exporter +After=network.target + +[Service] +Type=simple +User={{ graphite_exporter_user }} +Group={{ graphite_exporter_group }} +ExecStart={{ graphite_exporter_dist_dir }}/graphite_exporter-{{graphite_exporter_version}}.linux-amd64/graphite_exporter --graphite.mapping-config="{{ graphite_exporter_config_dir }}/graphite_mapping.conf" --graphite.listen-address=":9109" --web.listen-address=":9108" --graphite.sample-expiry=1h --graphite.cache-size=10000 + +SyslogIdentifier=prometheus_graphite_exporter +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/influxdb_backup/tasks/main.yml b/ansible/roles/influxdb_backup/tasks/main.yml index b51a34291d..eb423ff129 100644 --- a/ansible/roles/influxdb_backup/tasks/main.yml +++ b/ansible/roles/influxdb_backup/tasks/main.yml @@ -6,8 +6,8 @@ command: "influxd backup -portable -db {{ item }} {{ influxdb_backup_dir }}" with_items: "{{ influxdb_backup_databases }}" -- name: Ensure azure blob storage container exists - command: az storage container create --name "{{azure_influxdb_backup_container}}" --public-access blob +#- name: Ensure azure blob storage container exists +# command: az storage container create --name "{{azure_influxdb_backup_container}}" --public-access blob - name: zip the backup dir archive: diff --git a/ansible/roles/java/tasks/main.yml b/ansible/roles/java/tasks/main.yml deleted file mode 100644 index e142c3e8c7..0000000000 --- a/ansible/roles/java/tasks/main.yml +++ /dev/null @@ -1,27 +0,0 @@ -- name: Add Java repository to sources - action: apt_repository repo='ppa:webupd8team/java' - become: yes - -- name: Autoaccept license for Java - action: shell echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections - become: yes - - -- name: Update APT package cache - action: apt update_cache=yes - become: yes - - -- name: Install Java 8 - action: apt pkg=oracle-java8-installer state=latest install_recommends=yes - become: yes - - -- name: Set Java 8 Env - action: apt pkg=oracle-java8-set-default state=latest install_recommends=yes - become: yes - - -- name: setup envkeep - become: yes - lineinfile: dest=/etc/sudoers line='Defaults env_keep += "JAVA_HOME"' state=present insertafter=EOF create=yes \ No newline at end of file diff --git a/ansible/roles/jdk-1.8.0_121/README.md b/ansible/roles/jdk-1.8.0_121/README.md deleted file mode 100644 index 225dd44b9f..0000000000 --- a/ansible/roles/jdk-1.8.0_121/README.md +++ /dev/null @@ -1,38 +0,0 @@ -Role Name -========= - -A brief description of the role goes here. - -Requirements ------------- - -Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. - -Role Variables --------------- - -A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. - -Dependencies ------------- - -A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. - -Example Playbook ----------------- - -Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: - - - hosts: servers - roles: - - { role: username.rolename, x: 42 } - -License -------- - -BSD - -Author Information ------------------- - -An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/ansible/roles/kibana/.travis.yml b/ansible/roles/kibana/.travis.yml deleted file mode 100644 index 0767a46dba..0000000000 --- a/ansible/roles/kibana/.travis.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -sudo: required - -env: - global: - # https://github.com/travis-ci/travis-ci/issues/6461#issuecomment-239577306 - DOCKER_VERSION: "1.9.1-0~trusty" - matrix: - - distro: centos7 - init: /usr/lib/systemd/systemd - run_opts: "--privileged --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro" - - distro: centos6 - init: /sbin/init - run_opts: "" - - distro: ubuntu1604 - init: /lib/systemd/systemd - run_opts: "--privileged --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro" - - distro: ubuntu1404 - init: /sbin/init - run_opts: "" - - distro: ubuntu1204 - init: /sbin/init - run_opts: "" - -services: - - docker - -before_install: - # Downgrade to specific version of Docker engine. - - sudo apt-get update - - sudo apt-get remove docker-engine -yq - - sudo apt-get install docker-engine=$DOCKER_VERSION -yq --no-install-suggests --no-install-recommends --force-yes -o Dpkg::Options::="--force-confnew" - - # Pull container. - - 'sudo docker pull geerlingguy/docker-${distro}-ansible:latest' - -script: - - container_id=$(mktemp) - # Run container in detached state. - - 'sudo docker run --detach --volume="${PWD}":/etc/ansible/roles/role_under_test:ro ${run_opts} geerlingguy/docker-${distro}-ansible:latest "${init}" > "${container_id}"' - - # Ansible syntax check. - - 'sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm ansible-playbook /etc/ansible/roles/role_under_test/tests/test.yml --syntax-check' - - # Test role. - - 'sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm ansible-playbook /etc/ansible/roles/role_under_test/tests/test.yml' - - # Test role idempotence. - - idempotence=$(mktemp) - - sudo docker exec "$(cat ${container_id})" ansible-playbook /etc/ansible/roles/role_under_test/tests/test.yml | tee -a ${idempotence} - - > - tail ${idempotence} - | grep -q 'changed=0.*failed=0' - && (echo 'Idempotence test: pass' && exit 0) - || (echo 'Idempotence test: fail' && exit 1) - - # Test whether Kibana is running correctly. - - 'sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm curl http://localhost:5601/' - -after_success: - # Clean up. - - 'sudo docker stop "$(cat ${container_id})"' - -notifications: - webhooks: https://galaxy.ansible.com/api/v1/notifications/ diff --git a/ansible/roles/kibana/README.md b/ansible/roles/kibana/README.md deleted file mode 100644 index ac0877ba49..0000000000 --- a/ansible/roles/kibana/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Ansible Role: Kibana - -[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-kibana.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-kibana) - -An Ansible Role that installs Kibana on RedHat/CentOS or Debian/Ubuntu. - -## Requirements - -None. - -## Role Variables - -Available variables are listed below, along with default values (see `defaults/main.yml`): - - kibana_version: "4.6" - -The version of kibana to install (major and minor only). - - kibana_server_port: 5601 - kibana_server_host: "0.0.0.0" - -The FQDN or IP address and port Kibana should use. - - kibana_elasticsearch_url: "http://localhost:9200" - -The URL (including port) over which Kibana will connect to Elasticsearch. - -## Dependencies - -None. - -## Example Playbook - - - hosts: kibana - roles: - - geerlingguy.kibana - -## License - -MIT / BSD - -## Author Information - -This role was created in 2014 by [Jeff Geerling](http://www.jeffgeerling.com/), author of [Ansible for DevOps](https://www.ansiblefordevops.com/). diff --git a/ansible/roles/kibana/defaults/main.yml b/ansible/roles/kibana/defaults/main.yml deleted file mode 100644 index fb22975eb3..0000000000 --- a/ansible/roles/kibana/defaults/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -kibana_version: "6.2.3" - -kibana_base_path: "/pipeline-dashboard" -kibana_server_port: 5601 -kibana_server_host: "0.0.0.0" -kibana_elasticsearch_url: "http://localhost:9200" diff --git a/ansible/roles/kibana/handlers/main.yml b/ansible/roles/kibana/handlers/main.yml deleted file mode 100644 index 55ea3d3c1a..0000000000 --- a/ansible/roles/kibana/handlers/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- name: restart kibana - service: name=kibana state=restarted diff --git a/ansible/roles/kibana/meta/.galaxy_install_info b/ansible/roles/kibana/meta/.galaxy_install_info deleted file mode 100644 index 0983f9522c..0000000000 --- a/ansible/roles/kibana/meta/.galaxy_install_info +++ /dev/null @@ -1 +0,0 @@ -{install_date: 'Wed Apr 25 07:28:43 2018', version: 2.0.1} diff --git a/ansible/roles/kibana/meta/main.yml b/ansible/roles/kibana/meta/main.yml deleted file mode 100644 index dbbf992b98..0000000000 --- a/ansible/roles/kibana/meta/main.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -dependencies: [] - -galaxy_info: - author: geerlingguy - description: Kibana for Linux. - company: "Midwestern Mac, LLC" - license: "license (BSD, MIT)" - min_ansible_version: 2.0 - platforms: - - name: EL - versions: - - 6 - - 7 - - name: Debian - versions: - - jessie - - name: Ubuntu - versions: - - precise - - trusty - - xenial - galaxy_tags: - - web - - system - - monitoring diff --git a/ansible/roles/kibana/tasks/main.yml b/ansible/roles/kibana/tasks/main.yml deleted file mode 100644 index 51245f98c5..0000000000 --- a/ansible/roles/kibana/tasks/main.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- include: setup-RedHat.yml - when: ansible_os_family == 'RedHat' - -- include: setup-Debian.yml - when: ansible_os_family == 'Debian' - -- name: Ensure Kibana is started and enabled at boot. - service: - name: kibana - state: started - enabled: yes - -- name: Copy Kibana configuration. - template: - src: kibana.yml.j2 - dest: "/etc/kibana/kibana.yml" - owner: root - group: root - mode: 0644 - notify: restart kibana diff --git a/ansible/roles/kibana/tasks/setup-Debian.yml b/ansible/roles/kibana/tasks/setup-Debian.yml deleted file mode 100644 index e0cc720da6..0000000000 --- a/ansible/roles/kibana/tasks/setup-Debian.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Ensure depdency is installed (Ubuntu). - apt: name=apt-transport-https state=present - -- name: Download Kibana - get_url: - url: "https://artifacts.elastic.co/downloads/kibana/kibana-{{ kibana_version }}-amd64.deb" - dest: /tmp/kibana.deb - -- name: Install Kibana - apt: deb="/tmp/kibana.deb" - diff --git a/ansible/roles/kibana/tasks/setup-RedHat.yml b/ansible/roles/kibana/tasks/setup-RedHat.yml deleted file mode 100644 index bdb463ea48..0000000000 --- a/ansible/roles/kibana/tasks/setup-RedHat.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Add Elasticsearch GPG key. - rpm_key: - key: https://packages.elastic.co/GPG-KEY-elasticsearch - state: present - -- name: Add Kibana repository. - template: - src: kibana.repo.j2 - dest: /etc/yum.repos.d/kibana.repo - mode: 0644 diff --git a/ansible/roles/kibana/templates/kibana.repo.j2 b/ansible/roles/kibana/templates/kibana.repo.j2 deleted file mode 100644 index 51bfec336a..0000000000 --- a/ansible/roles/kibana/templates/kibana.repo.j2 +++ /dev/null @@ -1,6 +0,0 @@ -[kibana-{{ kibana_version }}] -name=Kibana repository for {{ kibana_version }}.x packages -baseurl=https://packages.elastic.co/kibana/{{ kibana_version }}/centos -gpgcheck=1 -gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch -enabled=1 diff --git a/ansible/roles/kibana/templates/kibana.yml.j2 b/ansible/roles/kibana/templates/kibana.yml.j2 deleted file mode 100644 index 639ca4e86f..0000000000 --- a/ansible/roles/kibana/templates/kibana.yml.j2 +++ /dev/null @@ -1,83 +0,0 @@ -# Kibana is served by a back end server. This controls which port to use. -server.port: {{ kibana_server_port }} - -# The host to bind the server to. -server.host: {{ kibana_server_host }} - -# If you are running kibana behind a proxy, and want to mount it at a path, -# specify that path here. The basePath can't end in a slash. -server.basePath: {{ kibana_base_path }} - -# The maximum payload size in bytes on incoming server requests. -# server.maxPayloadBytes: 1048576 - -# The Elasticsearch instance to use for all your queries. -elasticsearch.url: {{ kibana_elasticsearch_url }} - -# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false, -# then the host you use to connect to *this* Kibana instance will be sent. -# elasticsearch.preserveHost: true - -# Kibana uses an index in Elasticsearch to store saved searches, visualizations -# and dashboards. It will create a new index if it doesn't already exist. -# kibana.index: ".kibana" - -# The default application to load. -# kibana.defaultAppId: "discover" - -# If your Elasticsearch is protected with basic auth, these are the user credentials -# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana -# users will still need to authenticate with Elasticsearch (which is proxied through -# the Kibana server) -# elasticsearch.username: "user" -# elasticsearch.password: "pass" - -# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted) -# server.ssl.cert: /path/to/your/server.crt -# server.ssl.key: /path/to/your/server.key - -# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted) -# elasticsearch.ssl.cert: /path/to/your/client.crt -# elasticsearch.ssl.key: /path/to/your/client.key - -# If you need to provide a CA certificate for your Elasticsearch instance, put -# the path of the pem file here. -# elasticsearch.ssl.ca: /path/to/your/CA.pem - -# Set to false to have a complete disregard for the validity of the SSL -# certificate. -# elasticsearch.ssl.verify: true - -# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to -# request_timeout setting -# elasticsearch.pingTimeout: 1500 - -# Time in milliseconds to wait for responses from the back end or elasticsearch. -# This must be > 0 -# elasticsearch.requestTimeout: 30000 - -# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten -# by client-side headers. -# elasticsearch.customHeaders: {} - -# Time in milliseconds for Elasticsearch to wait for responses from shards. -# Set to 0 to disable. -# elasticsearch.shardTimeout: 0 - -# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying -# elasticsearch.startupTimeout: 5000 - -# Set the path to where you would like the process id file to be created. -# pid.file: /var/run/kibana.pid - -# If you would like to send the log output to a file you can set the path below. -# logging.dest: stdout - -# Set this to true to suppress all logging output. -# logging.silent: false - -# Set this to true to suppress all logging output except for error messages. -# logging.quiet: false - -# Set this to true to log all events, including system usage information and all requests. -# logging.verbose: false diff --git a/ansible/roles/kibana/tests/test.yml b/ansible/roles/kibana/tests/test.yml deleted file mode 100644 index 991eb5bef8..0000000000 --- a/ansible/roles/kibana/tests/test.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- hosts: all - - pre_tasks: - - name: Update apt cache. - apt: update_cache=yes - when: ansible_os_family == 'Debian' - - - name: Install test dependencies. - package: name=curl state=present - - roles: - - role_under_test diff --git a/ansible/roles/ml-druid-ingestion/defaults/main.yml b/ansible/roles/ml-druid-ingestion/defaults/main.yml new file mode 100644 index 0000000000..feceb11844 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/defaults/main.yml @@ -0,0 +1,48 @@ +remote_user: druid +remote_group: users +broker_port: 8082 +druid_home: /home/druid +desktop_pdata_id: "{{env}}.sunbird.desktop" + +druid_mm_heap_dump_file: /var/log/druid/crash_logs/middlemanager.hprof + +druid_overlord_port: 8090 + +ingestion_spec_configs: + raw_telemetry_events: + minTaskMemory: 512m + maxTaskMemory: 1024m + maxDirectTaskMemory: 1024m + +raw_offline_desktop_taskcount: 1 +raw_summary_events_taskcount: 1 +raw_telemetry_events_taskcount: 1 +raw_cb_audit_events_taskcount: 1 +raw_dashboards_user_course_program_progress_taskcount: 1 +raw_dashboards_user_assessment_taskcount: 1 +raw_dashboards_assessment_taskcount: 1 +raw_dashboards_user_org_taskcount: 1 +raw_dashboards_org_taskcount: 1 +raw_dashboards_course_taskcount: 1 +raw_dashboards_role_count_taskcount: 1 +raw_dashboards_org_role_count_taskcount: 1 +raw_dashboards_course_competency_taskcount: 1 +raw_dashboards_declared_user_competency_taskcount: 1 +raw_dashboards_expected_user_competency_taskcount: 1 +raw_dashboards_frac_competency_taskcount: 1 +raw_dashboards_user_competency_gap_taskcount: 1 +raw_dashboards_acbp_enrolment_taskcount: 1 +raw_telemetry_feedback_events_taskcount: 1 +rollup_error_hourly_syncts_taskcount: 1 +rollup_summary_distinct_counts_taskcount: 1 +rollup_summary_syncts_taskcount: 1 +rollup_telemetry_audit_syncts_taskcount: 1 +rollup_telemetry_hourly_syncts_taskcount: 1 +rollup_telemetry_syncts_taskcount: 1 +rollup_tpd_hourly_taskcount: 1 +sl_survey_evidence_task_count: 1 +sl_observation_task_count: 1 +sl_observation_evidence_task_count: 1 +sl_survey_task_count: 1 +rollup_course_completion_audit_syncts_taskcount: 1 +rollup_ml_project_taskcount: 1 diff --git a/ansible/roles/ml-druid-ingestion/tasks/main.yml b/ansible/roles/ml-druid-ingestion/tasks/main.yml new file mode 100644 index 0000000000..da8e8c5899 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/tasks/main.yml @@ -0,0 +1,35 @@ + +- name: Load the context json + set_fact: {"{{ item }}_context":"{{ lookup('template', 'context.json.j2')}}"} + with_items: "{{ ingestion_spec_configs }}" + tags: start-task + +- name: Load the ingestion json + set_fact: {"{{ item }}_json":"{{ lookup('template', (item))}}"} + with_items: "{{ ingestion_task_names.split(',')|list }}" + tags: start-task + +- name: Override the schema spec with context and Copy schema file + copy: + content: "{{ (lookup('vars', item + '_context' , default={}) | combine(vars[(item + '_json')])) | to_json(indent=4) }}" + dest: "{{druid_home}}/{{item}}.json" + become_user: "{{remote_user}}" + with_items: "{{ ingestion_task_names.split(',')|list }}" + tags: start-task + + +- name: Start kafka supervisor tasks for defined datasources + shell: 'curl -X POST -H "Content-Type:application/json" -d @{{druid_home}}/{{item}}.json http://{{overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/supervisor' + with_items: "{{ ingestion_task_names.split(',')|list }}" + become_user: "{{remote_user}}" + tags: start-task + +- name: Stop kafka supervisor tasks for defined datasources + shell: 'curl -X POST http://{{overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/supervisor/{{item}}/terminate' + with_items: "{{ ingestion_task_names.split(',')|list }}" + tags: stop-task + +- name: Stop currently running ingestion tasks + shell: 'curl -XPOST http://{{overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/datasources/{{item}}/shutdownAllTasks' + with_items: "{{ ingestion_task_names.split(',')|list }}" + tags: stop-task diff --git a/ansible/roles/ml-druid-ingestion/templates/context.json.j2 b/ansible/roles/ml-druid-ingestion/templates/context.json.j2 new file mode 100644 index 0000000000..11fc87a2f0 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/context.json.j2 @@ -0,0 +1 @@ +{"context":{"druid.indexer.runner.javaOpts":"-server -Xms{{ ingestion_spec_configs[item].minTaskMemory }} -Xmx{{ ingestion_spec_configs[item].maxTaskMemory }} -XX:MaxDirectMemorySize={{ ingestion_spec_configs[item].maxDirectTaskMemory }} -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"}} \ No newline at end of file diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation new file mode 100644 index 0000000000..043a0a9fd0 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation @@ -0,0 +1,279 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-observation", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "completedDate", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "completedDate" + }, + { + "type": "string", + "name": "createdAt" + }, + { + "type": "string", + "name": "createdBy" + }, + { + "type": "string", + "name": "criteriaExternalId" + }, + { + "type": "string", + "name": "criteriaId" + }, + { + "type": "string", + "name": "criteriaName" + }, + { + "type": "string", + "name": "entityType" + }, + { + "type": "string", + "name": "observationId" + }, + { + "type": "string", + "name": "observationName" + }, + { + "type": "string", + "name": "observationSubmissionId" + }, + { + "type": "string", + "name": "questionAnswer" + }, + { + "type": "string", + "name": "questionECM" + }, + { + "type": "string", + "name": "questionExternalId" + }, + { + "type": "string", + "name": "questionId" + }, + { + "type": "string", + "name": "questionName" + }, + { + "type": "string", + "name": "questionResponseLabel" + }, + { + "type": "string", + "name": "questionResponseType" + }, + { + "type": "string", + "name": "solutionExternalId" + }, + { + "type": "string", + "name": "solutionId" + }, + { + "type": "string", + "name": "solutionName" + }, + { + "type": "string", + "name": "updatedAt" + }, + { + "type": "string", + "name": "instanceParentId" + }, + { + "type": "string", + "name": "instanceId" + }, + { + "type": "string", + "name": "instanceParentResponsetype" + }, + { + "type": "string", + "name": "instanceParentQuestion" + }, + { + "type": "long", + "name": "questionSequenceByEcm" + }, + { + "type": "string", + "name": "instanceParentExternalId" + }, + { + "type": "string", + "name": "instanceParentEcmSequence" + }, + { + "type": "string", + "name": "remarks" + }, + { + "type": "long", + "name": "totalEvidences" + }, + { + "type": "string", + "name": "evidenceCount" + }, + { + "type": "string", + "name": "roleTitle" + }, + { + "type": "string", + "name": "entity" + }, + { + "type": "string", + "name": "entityExternalId" + }, + { + "type": "string", + "name": "isAPrivateProgram" + }, + { + "type": "string", + "name": "programId" + }, + { + "type": "string", + "name": "programName" + }, + { + "type": "string", + "name": "programExternalId" + }, + { + "name": "questionResponseLabelNumber", + "type": "float" + }, + { + "type": "long", + "name": "submissionNumber" + }, + { + "type": "string", + "name": "submissionTitle" + }, + { + "type": "string", + "name": "channel" + }, + { + "type": "string", + "name": "parentChannel" + }, + { + "type": "string", + "name": "appName" + }, + { + "type": "string", + "name": "evidences" + }, + { + "type": "string", + "name": "criteriaDescription" + }, + { + "type": "string", + "name": "programDescription" + }, + { + "type": "string", + "name": "solutionDescription" + }, + { + "name": "criteriaLevelReport", + "type": "string" + }, + { + "type": "string", + "name": "solution_type" + }, + { + "type": "string", + "name": "organisationName" + }, + { + "type": "string", + "name": "organisationId" + }, + { + "type": "string", + "name": "userType" + }, + { + "type": "string", + "name": "userBoardName" + }, + { + "type": "string", + "name": "userProfile" + }, + { + "type": "string", + "name": "course" + }, + { + "type": "string", + "name": "section" + }, + { + "type": "string", + "name": "isRubricDriven" + } + ] + } + } + }, + "metricsSpec": [ + { + "type": "floatSum", + "name": "questionResponseNumber", + "fieldName": "questionResponseLabelNumber" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.observation.druid", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } +} diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_evidence b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_evidence new file mode 100644 index 0000000000..f02a293520 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_evidence @@ -0,0 +1,75 @@ +{ + "type":"kafka", + "dataSchema":{ + "dataSource":"sl-observation-evidence", + "parser":{ + "type":"string", + "parseSpec":{ + "format":"json", + "timestampSpec":{ + "column":"completedDate", + "format":"auto" + }, + "dimensionsSpec":{ + "dimensions":[ + "total_evidences", + "observationSubmissionId", + "entity", + "entityExternalId", + "entityName", + "entityTypeId", + "entityType", + "createdBy", + "solutionExternalId", + "solutionId", + "observationId", + "remarks", + "questionId", + "questionExternalId", + "questionResponseType", + "questionName", + "evidence_count", + "fileName", + "fileSourcePath", + "completedDate", + "appName" + ] + } + } + }, + "metricsSpec":[ + + ], + "granularitySpec":{ + "type":"uniform", + "segmentGranularity":"day", + "queryGranularity":"day", + "rollup": true + } + }, + "tuningConfig":{ + "type":"kafka", + "reportParseExceptions":false + "maxRowsPerSegment": 5000000 + }, + "ioConfig":{ + "topic":"{{env}}.sl.observation.evidence.druid", + "consumerProperties":{ + "bootstrap.servers":"{{kafka_brokers}}" + }, + "taskCount":"{{sl_observation_evidence_task_count}}", + "replicas":1, + "taskDuration":"PT14400S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S" + } +} + + + + + + + + + diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_meta b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_meta new file mode 100644 index 0000000000..549be55dbe --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_meta @@ -0,0 +1,109 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-observation-meta", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "completedDate", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "completedDate" + }, + { + "type": "string", + "name": "createdBy" + }, + { + "type": "string", + "name": "createdAt" + }, + { + "type": "string", + "name": "entity" + }, + { + "type": "string", + "name": "entityExternalId" + }, + { + "type": "string", + "name": "entityType" + }, + { + "type": "string", + "name": "isAPrivateProgram" + }, + { + "type": "string", + "name": "observationName" + }, + { + "type": "string", + "name": "observationSubmissionId" + }, + { + "type": "string", + "name": "observationId" + }, + { + "type": "string", + "name": "organisationName" + }, + { + "type": "string", + "name": "solutionType" + }, + { + "type": "string", + "name": "solutionExternalId" + }, + { + "type": "string", + "name": "solutionId" + }, + { + "type": "string", + "name": "solutionName" + }, + { + "type": "string", + "name": "userType" + }, + { + "type": "string", + "name": "userProfile" + } + ] + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.observation.meta", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } +} diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_status_completed b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_status_completed new file mode 100644 index 0000000000..24f09991e5 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_status_completed @@ -0,0 +1,49 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-observation-status-completed", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "completedAt", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "completedAt" + }, + { + "type": "string", + "name": "observationSubmissionId" + } + ] + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.observation.status.completed", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } +} diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_status_inprogress b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_status_inprogress new file mode 100644 index 0000000000..f9f43ae5e3 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_status_inprogress @@ -0,0 +1,49 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-observation-status-inprogress", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "inprogressAt", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "inprogressAt" + }, + { + "type": "string", + "name": "observationSubmissionId" + } + ] + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.observation.status.inprogress", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } +} \ No newline at end of file diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_status_started b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_status_started new file mode 100644 index 0000000000..9b2c7b438b --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_observation_status_started @@ -0,0 +1,49 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-observation-status-started", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "startedAt", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "startedAt" + }, + { + "type": "string", + "name": "observationSubmissionId" + } + ] + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.observation.status.started", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } +} diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey new file mode 100644 index 0000000000..df0f11b2f5 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey @@ -0,0 +1,163 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-survey", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "completedDate", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "completedDate" + }, + { + "type": "string", + "name": "createdAt" + }, + { + "type": "string", + "name": "createdBy" + }, + { + "type": "string", + "name": "criteriaExternalId" + }, + { + "type": "string", + "name": "criteriaId" + }, + { + "type": "string", + "name": "criteriaName" + }, + { + "type": "string", + "name": "evidenceCount" + }, + { + "type": "string", + "name": "evidences" + }, + { + "type": "string", + "name": "isAPrivateProgram" + }, + { + "type": "string", + "name": "organisationId" + }, + { + "type": "string", + "name": "organisationName" + }, + { + "type": "string", + "name": "questionAnswer" + }, + { + "type": "string", + "name": "questionECM" + }, + { + "type": "string", + "name": "questionExternalId" + }, + { + "type": "string", + "name": "questionId" + }, + { + "type": "string", + "name": "questionName" + }, + { + "type": "string", + "name": "questionResponseLabel" + }, + { + "type": "float", + "name": "questionResponseLabelNumber" + }, + { + "type": "string", + "name": "questionResponseType" + }, + { + "type": "string", + "name": "remarks" + }, + { + "type": "string", + "name": "solutionExternalId" + }, + { + "type": "string", + "name": "solutionId" + }, + { + "type": "string", + "name": "solutionName" + }, + { + "type": "string", + "name": "surveyId" + }, + { + "type": "string", + "name": "surveyName" + }, + { + "type": "string", + "name": "surveySubmissionId" + }, + { + "type": "string", + "name": "totalEvidences" + }, + { + "type": "string", + "name": "updatedAt" + }, + { + "type": "string", + "name": "userProfile" + } + ] + } + } + }, + "metricsSpec": [ + { + "type": "floatSum", + "name": "questionResponseNumber", + "fieldName": "questionResponseLabelNumber" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.survey.druid", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } + } diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_evidence b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_evidence new file mode 100644 index 0000000000..4c144a08da --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_evidence @@ -0,0 +1,61 @@ +{ + "type":"kafka", + "dataSchema":{ + "dataSource":"sl-survey-evidence", + "parser":{ + "type":"string", + "parseSpec":{ + "format":"json", + "timestampSpec":{ + "column":"completedDate", + "format":"auto" + }, + "dimensionsSpec":{ + "dimensions":[ + "total_evidences", + "surveySubmissionId", + "createdBy", + "solutionExternalId", + "solutionId", + "surveyId", + "remarks", + "questionId", + "questionExternalId", + "questionResponseType", + "questionName", + "evidence_count", + "fileName", + "fileSourcePath", + "completedDate", + "appName" + ] + } + } + }, + "metricsSpec":[ + + ], + "granularitySpec":{ + "type":"uniform", + "segmentGranularity":"day", + "queryGranularity":"day", + "rollup": true + } + }, + "tuningConfig":{ + "type":"kafka", + "reportParseExceptions":false + "maxRowsPerSegment": 5000000 + }, + "ioConfig":{ + "topic":"{{env}}.sl.survey.evidence.druid", + "consumerProperties":{ + "bootstrap.servers":"{{kafka_brokers}}" + }, + "taskCount":"{{sl_survey_evidence_task_count}}", + "replicas":1, + "taskDuration":"PT14400S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S" + } +} diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_meta b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_meta new file mode 100644 index 0000000000..5071ead9f3 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_meta @@ -0,0 +1,89 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-survey-meta", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "completedDate", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "completedDate" + }, + { + "type": "string", + "name": "createdBy" + }, + { + "type": "string", + "name": "createdAt" + }, + { + "type": "string", + "name": "isAPrivateProgram" + }, + { + "type": "string", + "name": "organisationName" + }, + { + "type": "string", + "name": "solutionExternalId" + }, + { + "type": "string", + "name": "solutionId" + }, + { + "type": "string", + "name": "solutionName" + }, + { + "type": "string", + "name": "surveyName" + }, + { + "type": "string", + "name": "surveySubmissionId" + }, + { + "type": "string", + "name": "surveyId" + }, + { + "type": "string", + "name": "userProfile" + } + ] + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.survey.meta", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } +} diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_status_completed b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_status_completed new file mode 100644 index 0000000000..97c4cf0ed3 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_status_completed @@ -0,0 +1,49 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-survey-status-completed", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "completedAt", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "completedAt" + }, + { + "type": "string", + "name": "surveySubmissionId" + } + ] + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.survey.status.completed", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } +} diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_status_inprogress b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_status_inprogress new file mode 100644 index 0000000000..74e5cc4cf8 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_status_inprogress @@ -0,0 +1,49 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-survey-status-inprogress", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "inprogressAt", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "inprogressAt" + }, + { + "type": "string", + "name": "surveySubmissionId" + } + ] + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.survey.status.inprogress", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } +} diff --git a/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_status_started b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_status_started new file mode 100644 index 0000000000..31408a92e5 --- /dev/null +++ b/ansible/roles/ml-druid-ingestion/templates/raw_sl_survey_status_started @@ -0,0 +1,49 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sl-survey-status-started", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "startedAt", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "startedAt" + }, + { + "type": "string", + "name": "surveySubmissionId" + } + ] + } + } + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "none", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "{{env}}.ml.survey.status.started", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": "{{sl_survey_task_count}}", + "replicas": 1, + "taskDuration": "PT14400S", + "useEarliestOffset": true + } +} diff --git a/ansible/roles/monit/defaults/main.yml b/ansible/roles/monit/defaults/main.yml index 4b2174f4f7..d1744ddddb 100644 --- a/ansible/roles/monit/defaults/main.yml +++ b/ansible/roles/monit/defaults/main.yml @@ -157,6 +157,70 @@ monit_learning-failed-backup: pid_file: /home/analytics/sbin/learning-failed-backup.pid retry: 3 +monit_ingestion-cluster-telemetry-backup: + app_name: ingestion-cluster-telemetry-backup + custom_start: /home/analytics/sbin/secor ingestion-cluster-telemetry-backup start + custom_stop: /home/analytics/sbin/secor ingestion-cluster-telemetry-backup stop + user: analytics + pid_file: /home/analytics/sbin/ingestion-cluster-telemetry-backup.pid + retry: 3 + +monit_assess-raw-events-backup: + app_name: assess-raw-events-backup + custom_start: /home/analytics/sbin/secor assess-raw-events-backup start + custom_stop: /home/analytics/sbin/secor assess-raw-events-backup stop + user: analytics + pid_file: /home/analytics/sbin/assess-raw-events-backup.pid + retry: 3 + +monit_telemetry-duplicate-backup: + app_name: telemetry-duplicate-backup + custom_start: /home/analytics/sbin/secor telemetry-duplicate-backup start + custom_stop: /home/analytics/sbin/secor telemetry-duplicate-backup stop + user: analytics + pid_file: /home/analytics/sbin/telemetry-duplicate-backup.pid + retry: 3 + +monit_telemetry-ingest-backup: + app_name: telemetry-ingest-backup + custom_start: /home/analytics/sbin/secor telemetry-ingest-backup start + custom_stop: /home/analytics/sbin/secor telemetry-ingest-backup stop + user: analytics + pid_file: /home/analytics/sbin/telemetry-ingest-backup.pid + retry: 3 + +monit_extractor-duplicate-backup: + app_name: extractor-duplicate-backup + custom_start: /home/analytics/sbin/secor extractor-duplicate-backup start + custom_stop: /home/analytics/sbin/secor extractor-duplicate-backup stop + user: analytics + pid_file: /home/analytics/sbin/extractor-duplicate-backup.pid + retry: 3 + +monit_content-consumption-events-backup: + app_name: content-consumption-events-backup + custom_start: /home/analytics/sbin/secor content-consumption-events-backup start + custom_stop: /home/analytics/sbin/secor content-consumption-events-backup stop + user: analytics + pid_file: /home/analytics/sbin/content-consumption-events-backup.pid + retry: 3 + +monit_derived-denorm-events-backup: + app_name: derived-denorm-events-backup + custom_start: /home/analytics/sbin/secor derived-denorm-events-backup start + custom_stop: /home/analytics/sbin/secor derived-denorm-events-backup stop + user: analytics + pid_file: /home/analytics/sbin/derived-denorm-events-backup.pid + retry: 3 + +monit_channel-summary-backup: + app_name: channel-summary-backup + custom_start: /home/analytics/sbin/secor channel-summary-backup start + custom_stop: /home/analytics/sbin/secor channel-summary-backup stop + user: analytics + pid_file: /home/analytics/sbin/channel-summary-backup.pid + retry: 3 + monit_cassandra-dp: app_name: "cassandra" custom_start: /bin/systemctl start cassandra diff --git a/ansible/roles/openjdk/tasks/main.yml b/ansible/roles/openjdk/tasks/main.yml new file mode 100644 index 0000000000..140ead0417 --- /dev/null +++ b/ansible/roles/openjdk/tasks/main.yml @@ -0,0 +1,5 @@ +- name: installing repo for open jdk in Ubuntu + apt_repository: repo='ppa:openjdk-r/ppa' + +- name: ensure openjdk 8 is present + apt: name='openjdk-8-jdk-headless' state=present diff --git a/ansible/roles/postgres-azure-managed-service/tasks/main.yml b/ansible/roles/postgres-azure-managed-service/tasks/main.yml index ab415d7566..74178c43da 100644 --- a/ansible/roles/postgres-azure-managed-service/tasks/main.yml +++ b/ansible/roles/postgres-azure-managed-service/tasks/main.yml @@ -29,12 +29,12 @@ async: 500 poll: 10 -- name: Ensure azure blob storage container exists - command: az storage container create --name {{ postgresql_backup_azure_container_name }} - ignore_errors: true - environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}" - AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}" +#- name: Ensure azure blob storage container exists +# command: az storage container create --name {{ postgresql_backup_azure_container_name }} +# ignore_errors: true +# environment: +# AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}" +# AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}" - name: Upload to azure blob storage command: az storage blob upload --name {{ postgresql_backup_gzip_file_name }}.zip --file {{ postgresql_backup_gzip_file_path }} --container-name {{ postgresql_backup_azure_container_name }} diff --git a/ansible/roles/postgres-db-update/defaults/main.yml b/ansible/roles/postgres-db-update/defaults/main.yml new file mode 100644 index 0000000000..4f74dde390 --- /dev/null +++ b/ansible/roles/postgres-db-update/defaults/main.yml @@ -0,0 +1,24 @@ +--- +#postgres: + #db_url: "{{ groups['postgres'][0] }}" + #db_username: analytics + #db_name: analytics + #db_password: "{{vault_postges_password}}" + #db_tabel_name: "{{ env}}_consumer_channel_mapping" + #db_admin_user: postgres + #db_port: 5432 + + +#pgdb_password: "{{vault_postges_password}}" +#db_admin_password: "{{vault_postgres_admin_password}}" + +consumer_id: id #mention the consumer ID +channel: channel_id #Channel ID to map with the Consumer ID +status: 1 #It should be either 1 or 0, 1-to activate, 0-to make it inactive +created_by: analytics-team + +postgresql_user: postgres +postgresql_auth_method: "{{ ansible_fips | ternary('scram-sha-256', 'md5') }}" +postgres_users_no_log: true +postgresql_unix_socket_directories: + - /var/run/postgresql \ No newline at end of file diff --git a/ansible/roles/postgres-db-update/tasks/main.yml b/ansible/roles/postgres-db-update/tasks/main.yml new file mode 100644 index 0000000000..d582c4841a --- /dev/null +++ b/ansible/roles/postgres-db-update/tasks/main.yml @@ -0,0 +1,149 @@ +- name: Install PostgreSQL client + apt: name={{ item }} update_cache=yes cache_valid_time=36000 state=installed + with_items: + - python3-psycopg2 + - postgresql-client + ignore_errors: yes + +- import_tasks: users.yml +- import_tasks: users_props.yml + +- name: Creating databases + become: yes + become_user: "{{ postgresql_user }}" + block: + - name: Ensure database is created + postgresql_db: name="{{ postgres.db_name }}" \ + login_host="{{ postgres.db_url }}" \ + port="{{ postgres.db_port }}" \ + login_user="{{ postgres.db_admin_user }}" \ + login_password="{{ postgres.db_admin_password }}" \ + encoding='UTF-8' \ + state=present + tags: createdb + + - name: Ensure database for superset is created + postgresql_db: name="superset" \ + login_host="{{ postgres.db_url }}" \ + port="{{ postgres.db_port }}" \ + login_user="{{ postgres.db_admin_user }}" \ + login_password="{{ postgres.db_admin_password }}" \ + encoding='UTF-8' \ + state=present + tags: createdb + + - name: Ensure user has access to the database + postgresql_user: name="{{ postgres.db_username }}" \ + password="{{ postgres.db_password }}" \ + no_password_changes=true \ + priv=ALL \ + state=present \ + login_host="{{ postgres.db_url }}" \ + port="{{ postgres.db_port }}" \ + login_user="{{ postgres.db_admin_user }}" \ + login_password="{{ postgres.db_admin_password }}" \ + db="{{ postgres.db_name }}" + tags: createuser + + - name: Ensure user has access to the superset database + postgresql_user: name="{{ postgres.db_username }}" \ + password="{{ postgres.db_password }}" \ + no_password_changes=true \ + priv=ALL \ + state=present \ + login_host="{{ postgres.db_url }}" \ + port="{{ postgres.db_port }}" \ + login_user="{{ postgres.db_admin_user }}" \ + login_password="{{ postgres.db_admin_password }}" \ + db="superset" + tags: createuser + +- name: create {{ env }}_consumer-channel-mapping + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_consumer_channel_mapping(consumer_id text, channel text, status smallint, created_by text, created_on timestamptz, updated_on timestamptz, PRIMARY KEY(consumer_id, channel))" + run_once: true + tags: create + +- name: update {{ env }}_consumer_channel_mapping + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "insert into {{ env }}_consumer_channel_mapping (consumer_id, channel, status, created_by, created_on, updated_on) values ('{{ consumer_id }}', '{{ channel }}', {{ status }}, '{{created_by }}', current_timestamp, current_timestamp) on conflict on constraint {{ env }}_consumer_channel_mapping_pkey DO UPDATE set status=EXCLUDED.status, updated_on= current_timestamp" + run_once: true + tags: insert + +- name: create {{ env }}_device_profile + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_device_profile(device_id text, api_last_updated_on timestamptz, avg_ts float, city text, country text, country_code text, device_spec json, district_custom text, fcm_token text, first_access timestamptz, last_access timestamptz, producer_id text, state text, state_code text, state_code_custom text, state_custom text, total_launches bigint, total_ts float, uaspec json, updated_date timestamptz, user_declared_district text, user_declared_state text, user_declared_on timestamptz, PRIMARY KEY(device_id))" + run_once: true + tags: create + +- name: create {{ env }}_report_config + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_report_config(report_id text, updated_on timestamptz,report_description text,requested_by text,report_schedule text,config json,created_on timestamptz,submitted_on timestamptz,status text,status_msg text,PRIMARY KEY(report_id))" + run_once: true + tags: create + +- name: update {{ env }}_report_config + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_report_config ADD COLUMN IF NOT EXISTS batch_number INTEGER" + run_once: true + tags: update + +- name: create report + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report(reportid varchar(40) NOT NULL PRIMARY KEY, title text NOT NULL, description text NOT NULL, authorizedroles jsonb NOT NULL, status varchar(8) NOT NULL CHECK (status IN ('live', 'draft', 'retired')) DEFAULT 'draft', type varchar(8) NOT NULL CHECK (type in ('public', 'private')) DEFAULT 'private', reportaccessurl text NOT NULL UNIQUE, createdon timestamptz NOT NULL DEFAULT now(), updatedon timestamptz NOT NULL DEFAULT now(), createdby varchar(50) NOT NULL, reportconfig jsonb NOT NULL, templateurl text, slug varchar(10) NOT NULL, reportgenerateddate timestamptz NOT NULL DEFAULT now(), reportduration jsonb NOT NULL DEFAULT jsonb_build_object('startDate', now()::timestamptz, 'endDate', now()::timestamptz), tags jsonb NOT NULL, updatefrequency text NOT NULL)" + run_once: true + tags: create + +- name: create report_summary + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report_summary(id varchar(40) NOT NULL PRIMARY KEY, reportid varchar(40) NOT NULL, chartid text, createdon timestamptz NOT NULL DEFAULT now(), createdby varchar(50) NOT NULL, summary text NOT NULL)" + run_once: true + tags: create + +- name: create {{ env }}_consumer-channel-mapping + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "select * from {{ env }}_consumer_channel_mapping" + run_once: true + tags: select + +- name: create report_status + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report_status(reportid varchar(40) NOT NULL REFERENCES report(reportid) ON DELETE CASCADE, hashed_val text NOT NULL, status varchar(8) NOT NULL CHECK (status IN ('live', 'draft', 'retired')) DEFAULT 'draft', PRIMARY KEY (reportid, hashed_val))" + run_once: true + tags: create + +- name: create {{ env }}_job_request + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_job_request(tag VARCHAR(100), request_id VARCHAR(50), job_id VARCHAR(50), status VARCHAR(50), request_data json, requested_by VARCHAR(50), requested_channel VARCHAR(50), dt_job_submitted TIMESTAMP, download_urls text[], dt_file_created TIMESTAMP, dt_job_completed TIMESTAMP, execution_time INTEGER, err_message VARCHAR(300), iteration INTEGER, encryption_key VARCHAR(50), PRIMARY KEY (tag, request_id))" + run_once: true + tags: create + +- name: update {{ env }}_job_request + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_job_request ADD COLUMN IF NOT EXISTS batch_number INTEGER" + run_once: true + tags: update + +- name: update {{ env }}_job_request + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_job_request ADD COLUMN IF NOT EXISTS processed_batches text" + run_once: true + tags: update + +- name: create {{ env }}_experiment_definition + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_experiment_definition (exp_id VARCHAR(50), created_by VARCHAR(50), created_on TIMESTAMP, criteria VARCHAR(100), exp_data VARCHAR(300), exp_description VARCHAR(200), exp_name VARCHAR(50), stats VARCHAR(300), status VARCHAR(50), status_message VARCHAR(50), updated_by VARCHAR(50), updated_on TIMESTAMP, PRIMARY KEY(exp_id))" + run_once: true + tags: create + +- name: create {{ env }}_dataset_metadata + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_dataset_metadata(dataset_id VARCHAR(50), dataset_sub_id VARCHAR(150), dataset_config json, visibility VARCHAR(50), dataset_type VARCHAR(50), version VARCHAR(10), authorized_roles text[], available_from TIMESTAMP, sample_request text, sample_response text, validation_json json, druid_query json, limits json, supported_formats text[], exhaust_type VARCHAR(50), PRIMARY KEY (dataset_id, dataset_sub_id))" + run_once: true + tags: update + +- name: update {{ env }}_dataset_metadata + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_dataset_metadata ALTER COLUMN dataset_sub_id TYPE VARCHAR(150)" + run_once: true + tags: update + +- name: update report + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ADD COLUMN IF NOT EXISTS parameters jsonb" + run_once: true + tags: update + +- name: update report_summary + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report_summary ADD COLUMN IF NOT EXISTS param_hash text" + run_once: true + tags: update + +- name: update report + shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ADD COLUMN IF NOT EXISTS report_type varchar(8) NOT NULL DEFAULT 'report'" + run_once: true + tags: update diff --git a/ansible/roles/postgres-db-update/tasks/users.yml b/ansible/roles/postgres-db-update/tasks/users.yml new file mode 100644 index 0000000000..4d346a9c81 --- /dev/null +++ b/ansible/roles/postgres-db-update/tasks/users.yml @@ -0,0 +1,14 @@ +--- +- name: Ensure PostgreSQL users are present. + postgresql_user: + name: "{{ item.name }}" + password: "{{ item.password | default(omit) }}" + with_items: "{{ postgresql_users }}" + no_log: "{{ postgres_users_no_log }}" + become: true + become_user: "{{ postgresql_user }}" + # See: https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 + vars: + ansible_ssh_pipelining: true + environment: + PGOPTIONS: "{{ (postgresql_auth_method == 'scram-sha-256') | ternary('-c password_encryption=scram-sha-256', '') }}" diff --git a/ansible/roles/postgres-db-update/tasks/users_props.yml b/ansible/roles/postgres-db-update/tasks/users_props.yml new file mode 100644 index 0000000000..992ccf5c9e --- /dev/null +++ b/ansible/roles/postgres-db-update/tasks/users_props.yml @@ -0,0 +1,24 @@ +--- +- name: Ensure PostgreSQL users are configured correctly. + postgresql_user: + name: "{{ item.name }}" + password: "{{ item.password | default(omit) }}" + encrypted: "{{ item.encrypted | default(omit) }}" + priv: "{{ item.priv | default(omit) }}" + role_attr_flags: "{{ item.role_attr_flags | default(omit) }}" + db: "{{ item.db | default(omit) }}" + login_host: "{{ item.login_host | default('localhost') }}" + login_password: "{{ item.login_password | default(omit) }}" + login_user: "{{ item.login_user | default(postgresql_user) }}" + login_unix_socket: "{{ item.login_unix_socket | default(postgresql_unix_socket_directories[0]) }}" + port: "{{ item.port | default(omit) }}" + state: "{{ item.state | default('present') }}" + with_items: "{{ postgresql_users }}" + no_log: "{{ postgres_users_no_log }}" + become: true + become_user: "{{ postgresql_user }}" + # See: https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 + vars: + ansible_ssh_pipelining: true + environment: + PGOPTIONS: "{{ (postgresql_auth_method == 'scram-sha-256') | ternary('-c password_encryption=scram-sha-256', '') }}" diff --git a/ansible/roles/postgres-provision/.gitignore b/ansible/roles/postgres-provision/.gitignore deleted file mode 100644 index f56f5b578f..0000000000 --- a/ansible/roles/postgres-provision/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.retry -*/__pycache__ -*.pyc diff --git a/ansible/roles/postgres-provision/.travis.yml b/ansible/roles/postgres-provision/.travis.yml deleted file mode 100644 index d6e345276b..0000000000 --- a/ansible/roles/postgres-provision/.travis.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -language: python -services: docker - -env: - global: - - ROLE_NAME: postgresql - matrix: - - MOLECULE_DISTRO: centos7 - MOLECULE_DOCKER_COMMAND: /usr/lib/systemd/systemd - - MOLECULE_DISTRO: ubuntu1804 - - MOLECULE_DISTRO: ubuntu1604 - - MOLECULE_DISTRO: ubuntu1404 - - MOLECULE_DISTRO: debian9 - - MOLECULE_DISTRO: debian8 - -install: - # Install test dependencies. - - pip install molecule docker - -before_script: - # Use actual Ansible Galaxy role name for the project directory. - - cd ../ - - mv ansible-role-$ROLE_NAME geerlingguy.$ROLE_NAME - - cd geerlingguy.$ROLE_NAME - -script: - # Run tests. - - molecule test - -notifications: - webhooks: https://galaxy.ansible.com/api/v1/notifications/ diff --git a/ansible/roles/postgres-provision/LICENSE b/ansible/roles/postgres-provision/LICENSE deleted file mode 100644 index 4275cf3c10..0000000000 --- a/ansible/roles/postgres-provision/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 Jeff Geerling - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/ansible/roles/postgres-provision/README.md b/ansible/roles/postgres-provision/README.md index 6d81e4e9f9..a30ec22270 100644 --- a/ansible/roles/postgres-provision/README.md +++ b/ansible/roles/postgres-provision/README.md @@ -1,6 +1,6 @@ # Ansible Role: PostgreSQL -[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-postgresql.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-postgresql) +[![CI](https://github.com/geerlingguy/ansible-role-postgresql/workflows/CI/badge.svg?event=push)](https://github.com/geerlingguy/ansible-role-postgresql/actions?query=workflow%3ACI) Installs and configures PostgreSQL server on RHEL/CentOS or Debian/Ubuntu servers. @@ -106,6 +106,10 @@ A list of databases to ensure exist on the server. Only the `name` is required; A list of users to ensure exist on the server. Only the `name` is required; all other properties are optional. + postgres_users_no_log: true + +Whether to output user data (which may contain sensitive information, like passwords) when managing users. + postgresql_version: [OS-specific] postgresql_data_dir: [OS-specific] postgresql_bin_path: [OS-specific] diff --git a/ansible/roles/postgres-provision/defaults/main.yml b/ansible/roles/postgres-provision/defaults/main.yml index 73e40f0125..160e7828ce 100644 --- a/ansible/roles/postgres-provision/defaults/main.yml +++ b/ansible/roles/postgres-provision/defaults/main.yml @@ -10,6 +10,9 @@ postgresql_python_library: python-psycopg2 postgresql_user: postgres postgresql_group: postgres +# `md5` or `scram-sha-256` (https://www.postgresql.org/docs/10/auth-methods.html) +postgresql_auth_method: "{{ ansible_fips | ternary('scram-sha-256', 'md5') }}" + postgresql_unix_socket_directories: - /var/run/postgresql @@ -24,10 +27,10 @@ postgresql_global_config_options: # Host based authentication (hba) entries to be added to the pg_hba.conf. This # variable's defaults reflect the defaults that come with a fresh installation. postgresql_hba_entries: - - { type: local, database: all, user: postgres, auth_method: peer } - - { type: local, database: all, user: all, auth_method: peer } - - { type: host, database: all, user: all, address: '{{ postgres_address_space }}', auth_method: md5 } - - { type: host, database: all, user: all, address: '::1/128', auth_method: md5 } + - {type: local, database: all, user: postgres, auth_method: peer} + - {type: local, database: all, user: all, auth_method: peer} + - {type: host, database: all, user: all, address: '127.0.0.1/32', auth_method: "{{ postgresql_auth_method }}"} + - {type: host, database: all, user: all, address: '::1/128', auth_method: "{{ postgresql_auth_method }}"} # Debian only. Used to generate the locales used by PostgreSQL databases. postgresql_locales: @@ -62,3 +65,10 @@ postgresql_users: [] # login_unix_socket: # defaults to 1st of postgresql_unix_socket_directories # port: # defaults to not set # state: # defaults to 'present' + +# Whether to output user data when managing users. +postgres_users_no_log: true + +# apt postgres repo details +postgres_repo_key: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' +postgres_repo: 'deb http://apt.postgresql.org/pub/repos/apt/ jammy-pgdg main' diff --git a/ansible/roles/postgres-provision/meta/.galaxy_install_info b/ansible/roles/postgres-provision/meta/.galaxy_install_info deleted file mode 100644 index fa7f3604a7..0000000000 --- a/ansible/roles/postgres-provision/meta/.galaxy_install_info +++ /dev/null @@ -1 +0,0 @@ -{install_date: 'Thu Oct 11 13:38:42 2018', version: 1.4.4} diff --git a/ansible/roles/postgres-provision/meta/main.yml b/ansible/roles/postgres-provision/meta/main.yml deleted file mode 100644 index ccdd8577d1..0000000000 --- a/ansible/roles/postgres-provision/meta/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -dependencies: [] - -galaxy_info: - author: geerlingguy - description: PostgreSQL server for Linux. - company: "Midwestern Mac, LLC" - license: "license (BSD, MIT)" - min_ansible_version: 2.4 - platforms: - - name: EL - versions: - - 6 - - 7 - - name: Ubuntu - versions: - - all - - name: Debian - versions: - - all - galaxy_tags: - - database - - postgresql - - postgres - - rdbms diff --git a/ansible/roles/postgres-provision/molecule/default/molecule.yml b/ansible/roles/postgres-provision/molecule/default/molecule.yml deleted file mode 100644 index 033970241f..0000000000 --- a/ansible/roles/postgres-provision/molecule/default/molecule.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker -lint: - name: yamllint - options: - config-file: molecule/default/yaml-lint.yml -platforms: - - name: instance - image: geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible - command: ${MOLECULE_DOCKER_COMMAND:-"sleep infinity"} - privileged: true - pre_build_image: true -provisioner: - name: ansible - lint: - name: ansible-lint - playbooks: - converge: ${MOLECULE_PLAYBOOK:-playbook.yml} -scenario: - name: default -verifier: - name: testinfra - lint: - name: flake8 diff --git a/ansible/roles/postgres-provision/molecule/default/playbook.yml b/ansible/roles/postgres-provision/molecule/default/playbook.yml deleted file mode 100644 index 5f152a6cab..0000000000 --- a/ansible/roles/postgres-provision/molecule/default/playbook.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -- name: Converge - hosts: all - become: true - - vars: - postgresql_databases: - - name: example - postgresql_users: - - name: jdoe - - pre_tasks: - - name: Update apt cache. - apt: update_cache=true cache_valid_time=600 - when: ansible_os_family == 'Debian' - - - name: Set custom variables for old CentOS 6 PostgreSQL install. - set_fact: - postgresql_hba_entries: [] - postgresql_global_config_options: - - option: unix_socket_directory - value: '{{ postgresql_unix_socket_directories[0] }}' - when: - - ansible_os_family == 'RedHat' - - ansible_distribution_version.split('.')[0] == '6' - - roles: - - role: geerlingguy.postgresql - - post_tasks: - - name: Verify postgres is running. - command: "{{ postgresql_bin_path }}/pg_ctl -D {{ postgresql_data_dir }} status" - changed_when: false - become: true - become_user: postgres diff --git a/ansible/roles/postgres-provision/molecule/default/tests/test_default.py b/ansible/roles/postgres-provision/molecule/default/tests/test_default.py deleted file mode 100644 index eedd64a1d3..0000000000 --- a/ansible/roles/postgres-provision/molecule/default/tests/test_default.py +++ /dev/null @@ -1,14 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_hosts_file(host): - f = host.file('/etc/hosts') - - assert f.exists - assert f.user == 'root' - assert f.group == 'root' diff --git a/ansible/roles/postgres-provision/molecule/default/yaml-lint.yml b/ansible/roles/postgres-provision/molecule/default/yaml-lint.yml deleted file mode 100644 index a3dbc38eef..0000000000 --- a/ansible/roles/postgres-provision/molecule/default/yaml-lint.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -extends: default -rules: - line-length: - max: 120 - level: warning diff --git a/ansible/roles/postgres-provision/tasks/configure.yml b/ansible/roles/postgres-provision/tasks/configure.yml index e542b12952..df4346450f 100644 --- a/ansible/roles/postgres-provision/tasks/configure.yml +++ b/ansible/roles/postgres-provision/tasks/configure.yml @@ -5,24 +5,19 @@ regexp: "^#?{{ item.option }}.+$" line: "{{ item.option }} = '{{ item.value }}'" state: "{{ item.state | default('present') }}" + mode: 0644 with_items: "{{ postgresql_global_config_options }}" notify: restart postgresql -- name: Configure global settings. - lineinfile: - dest: "{{ postgresql_config_path }}/postgresql.conf" - line: "listen_addresses = '*'" - notify: restart postgresql - - name: Configure host based authentication (if entries are configured). template: - src: "templates/pg_hba.conf.j2" + src: "pg_hba.conf.j2" dest: "{{ postgresql_config_path }}/pg_hba.conf" owner: "{{ postgresql_user }}" group: "{{ postgresql_group }}" mode: 0600 notify: restart postgresql - when: postgresql_hba_entries + when: postgresql_hba_entries | length > 0 - name: Ensure PostgreSQL unix socket dirs exist. file: @@ -30,5 +25,5 @@ state: directory owner: "{{ postgresql_user }}" group: "{{ postgresql_group }}" - mode: 02775 + mode: "{{ postgresql_unix_socket_directories_mode }}" with_items: "{{ postgresql_unix_socket_directories }}" diff --git a/ansible/roles/postgres-provision/tasks/databases.yml b/ansible/roles/postgres-provision/tasks/databases.yml index e01d804d47..d13313e58c 100644 --- a/ansible/roles/postgres-provision/tasks/databases.yml +++ b/ansible/roles/postgres-provision/tasks/databases.yml @@ -2,8 +2,6 @@ - name: Ensure PostgreSQL databases are present. postgresql_db: name: "{{ item.name }}" - lc_collate: "{{ item.lc_collate | default('en_US.UTF-8') }}" - lc_ctype: "{{ item.lc_ctype | default('en_US.UTF-8') }}" encoding: "{{ item.encoding | default('UTF-8') }}" template: "{{ item.template | default('template0') }}" login_host: "{{ item.login_host | default('localhost') }}" diff --git a/ansible/roles/postgres-provision/tasks/main.yml b/ansible/roles/postgres-provision/tasks/main.yml index d2536265c6..fe99b4fc10 100644 --- a/ansible/roles/postgres-provision/tasks/main.yml +++ b/ansible/roles/postgres-provision/tasks/main.yml @@ -1,16 +1,16 @@ --- # Variable configuration. -- include: variables.yml +- include_tasks: variables.yml # Setup/install tasks. -- include: setup-RedHat.yml +- include_tasks: setup-RedHat.yml when: ansible_os_family == 'RedHat' -- include: setup-Debian.yml +- include_tasks: setup-Debian.yml when: ansible_os_family == 'Debian' -- include: initialize.yml -- include: configure.yml +- include_tasks: initialize.yml +- include_tasks: configure.yml - name: Ensure PostgreSQL is started and enabled on boot. service: @@ -19,11 +19,6 @@ enabled: "{{ postgresql_service_enabled }}" # Configure PostgreSQL. -- include: users.yml -- include: databases.yml -- name: Ensure PostgreSQL is started and enabled on boot. - service: - name: "{{ postgresql_daemon }}" - state: started - enabled: yes - +#- import_tasks: users.yml +#- import_tasks: databases.yml +#- import_tasks: users_props.yml diff --git a/ansible/roles/postgres-provision/tasks/setup-Debian.yml b/ansible/roles/postgres-provision/tasks/setup-Debian.yml index fc47d0f02a..b8e68b82b7 100644 --- a/ansible/roles/postgres-provision/tasks/setup-Debian.yml +++ b/ansible/roles/postgres-provision/tasks/setup-Debian.yml @@ -4,11 +4,20 @@ name: "{{ postgresql_python_library }}" state: present +- name: debian | Adding Postgres Debian Repo Key + apt_key: + url: "{{ postgres_repo_key }}" + state: "present" + +- name: debian | Adding postgres Debian Repo + apt_repository: + repo: "{{ postgres_repo }}" + state: "present" + - name: Ensure PostgreSQL packages are installed. apt: - name: "{{ item }}" + name: "{{ postgresql_packages }}" state: present - with_items: "{{ postgresql_packages }}" - name: Ensure all configured locales are present. locale_gen: "name={{ item }} state=present" diff --git a/ansible/roles/postgres-provision/tasks/setup-RedHat.yml b/ansible/roles/postgres-provision/tasks/setup-RedHat.yml index 9cd84ed01a..d536bcb012 100644 --- a/ansible/roles/postgres-provision/tasks/setup-RedHat.yml +++ b/ansible/roles/postgres-provision/tasks/setup-RedHat.yml @@ -1,12 +1,16 @@ --- - name: Ensure PostgreSQL packages are installed. - package: + yum: name: "{{ postgresql_packages }}" state: present enablerepo: "{{ postgresql_enablerepo | default(omit, true) }}" + # Don't let postgresql-contrib cause the /usr/bin/python symlink + # to be installed, which breaks later Ansible runs on Fedora 30, + # and affects system behavior in multiple ways. + exclude: python-unversioned-command - name: Ensure PostgreSQL Python libraries are installed. - package: + yum: name: "{{ postgresql_python_library }}" state: present enablerepo: "{{ postgresql_enablerepo | default(omit, true) }}" diff --git a/ansible/roles/postgres-provision/tasks/users.yml b/ansible/roles/postgres-provision/tasks/users.yml index 34746ebe9d..4d346a9c81 100644 --- a/ansible/roles/postgres-provision/tasks/users.yml +++ b/ansible/roles/postgres-provision/tasks/users.yml @@ -3,20 +3,12 @@ postgresql_user: name: "{{ item.name }}" password: "{{ item.password | default(omit) }}" - encrypted: "{{ item.encrypted | default(omit) }}" - priv: "{{ item.priv | default(omit) }}" - role_attr_flags: "{{ item.role_attr_flags | default(omit) }}" - db: "{{ item.db | default(omit) }}" - login_host: "{{ item.login_host | default('localhost') }}" - login_password: "{{ item.login_password | default(omit) }}" - login_user: "{{ item.login_user | default(postgresql_user) }}" - login_unix_socket: "{{ item.login_unix_socket | default(postgresql_unix_socket_directories[0]) }}" - port: "{{ item.port | default(omit) }}" - state: "{{ item.state | default('present') }}" with_items: "{{ postgresql_users }}" - no_log: true + no_log: "{{ postgres_users_no_log }}" become: true become_user: "{{ postgresql_user }}" # See: https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 vars: ansible_ssh_pipelining: true + environment: + PGOPTIONS: "{{ (postgresql_auth_method == 'scram-sha-256') | ternary('-c password_encryption=scram-sha-256', '') }}" diff --git a/ansible/roles/postgres-provision/tasks/users_props.yml b/ansible/roles/postgres-provision/tasks/users_props.yml new file mode 100644 index 0000000000..992ccf5c9e --- /dev/null +++ b/ansible/roles/postgres-provision/tasks/users_props.yml @@ -0,0 +1,24 @@ +--- +- name: Ensure PostgreSQL users are configured correctly. + postgresql_user: + name: "{{ item.name }}" + password: "{{ item.password | default(omit) }}" + encrypted: "{{ item.encrypted | default(omit) }}" + priv: "{{ item.priv | default(omit) }}" + role_attr_flags: "{{ item.role_attr_flags | default(omit) }}" + db: "{{ item.db | default(omit) }}" + login_host: "{{ item.login_host | default('localhost') }}" + login_password: "{{ item.login_password | default(omit) }}" + login_user: "{{ item.login_user | default(postgresql_user) }}" + login_unix_socket: "{{ item.login_unix_socket | default(postgresql_unix_socket_directories[0]) }}" + port: "{{ item.port | default(omit) }}" + state: "{{ item.state | default('present') }}" + with_items: "{{ postgresql_users }}" + no_log: "{{ postgres_users_no_log }}" + become: true + become_user: "{{ postgresql_user }}" + # See: https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 + vars: + ansible_ssh_pipelining: true + environment: + PGOPTIONS: "{{ (postgresql_auth_method == 'scram-sha-256') | ternary('-c password_encryption=scram-sha-256', '') }}" diff --git a/ansible/roles/postgres-provision/tasks/variables.yml b/ansible/roles/postgres-provision/tasks/variables.yml index 530990d7ea..575897207f 100644 --- a/ansible/roles/postgres-provision/tasks/variables.yml +++ b/ansible/roles/postgres-provision/tasks/variables.yml @@ -6,23 +6,29 @@ - name: Include OS-specific variables (RedHat). include_vars: "{{ ansible_os_family }}-{{ ansible_distribution_version.split('.')[0] }}.yml" - when: ansible_os_family == 'RedHat' + when: + - ansible_os_family == 'RedHat' + - ansible_distribution != 'Fedora' + +- name: Include OS-specific variables (Fedora). + include_vars: "{{ ansible_distribution }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + when: ansible_distribution == 'Fedora' - name: Define postgresql_packages. set_fact: postgresql_packages: "{{ __postgresql_packages | list }}" when: postgresql_packages is not defined -- name: Define postgresql_daemon. - set_fact: - postgresql_daemon: "{{ __postgresql_daemon }}" - when: postgresql_daemon is not defined - - name: Define postgresql_version. set_fact: postgresql_version: "{{ __postgresql_version }}" when: postgresql_version is not defined +- name: Define postgresql_daemon. + set_fact: + postgresql_daemon: "{{ __postgresql_daemon }}" + when: postgresql_daemon is not defined + - name: Define postgresql_data_dir. set_fact: postgresql_data_dir: "{{ __postgresql_data_dir }}" @@ -37,3 +43,9 @@ set_fact: postgresql_config_path: "{{ __postgresql_config_path }}" when: postgresql_config_path is not defined + +- name: Define postgresql_unix_socket_directories_mode. + set_fact: + postgresql_unix_socket_directories_mode: >- + {{ __postgresql_unix_socket_directories_mode | default('02775') }} + when: postgresql_unix_socket_directories_mode is not defined diff --git a/ansible/roles/postgres-provision/vars/Debian-10.yml b/ansible/roles/postgres-provision/vars/Debian-10.yml new file mode 100644 index 0000000000..d8b5103b6f --- /dev/null +++ b/ansible/roles/postgres-provision/vars/Debian-10.yml @@ -0,0 +1,12 @@ +--- +__postgresql_version: "11" +__postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main" +__postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin" +__postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main" +__postgresql_daemon: "postgresql@{{ postgresql_version }}-main" +__postgresql_packages: + - postgresql + - postgresql-contrib + - libpq-dev +# Debian 10 uses Python 3 by default. +postgresql_python_library: python3-psycopg2 diff --git a/ansible/roles/postgres-provision/vars/Debian-8.yml b/ansible/roles/postgres-provision/vars/Debian-8.yml index f6ad232b51..ec86f938fb 100644 --- a/ansible/roles/postgres-provision/vars/Debian-8.yml +++ b/ansible/roles/postgres-provision/vars/Debian-8.yml @@ -3,7 +3,7 @@ __postgresql_version: "9.4" __postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main" __postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin" __postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main" -__postgresql_daemon: postgresql +__postgresql_daemon: "postgresql@{{ postgresql_version }}-main" __postgresql_packages: - postgresql - postgresql-contrib diff --git a/ansible/roles/postgres-provision/vars/Debian-9.yml b/ansible/roles/postgres-provision/vars/Debian-9.yml index 7b57d53241..2afb9f42f0 100644 --- a/ansible/roles/postgres-provision/vars/Debian-9.yml +++ b/ansible/roles/postgres-provision/vars/Debian-9.yml @@ -3,7 +3,7 @@ __postgresql_version: "9.6" __postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main" __postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin" __postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main" -__postgresql_daemon: postgresql +__postgresql_daemon: "postgresql@{{ postgresql_version }}-main" __postgresql_packages: - postgresql - postgresql-contrib diff --git a/ansible/roles/postgres-provision/vars/Fedora-29.yml b/ansible/roles/postgres-provision/vars/Fedora-29.yml new file mode 100644 index 0000000000..4e099324bf --- /dev/null +++ b/ansible/roles/postgres-provision/vars/Fedora-29.yml @@ -0,0 +1,12 @@ +--- +__postgresql_version: "10.5" +__postgresql_data_dir: "/var/lib/pgsql/data" +__postgresql_bin_path: "/usr/bin" +__postgresql_config_path: "/var/lib/pgsql/data" +__postgresql_daemon: postgresql +__postgresql_packages: + - postgresql + - postgresql-server + - postgresql-contrib + - postgresql-libs +postgresql_python_library: python2-psycopg2 diff --git a/ansible/roles/postgres-provision/vars/Fedora-30.yml b/ansible/roles/postgres-provision/vars/Fedora-30.yml new file mode 100644 index 0000000000..d07f14b7bd --- /dev/null +++ b/ansible/roles/postgres-provision/vars/Fedora-30.yml @@ -0,0 +1,13 @@ +--- +__postgresql_version: "11.2" +__postgresql_data_dir: "/var/lib/pgsql/data" +__postgresql_bin_path: "/usr/bin" +__postgresql_config_path: "/var/lib/pgsql/data" +__postgresql_daemon: postgresql +__postgresql_packages: + - postgresql + - postgresql-server + - postgresql-contrib + - postgresql-libs +# Fedora 30 containers only have python3 by default +postgresql_python_library: python3-psycopg2 diff --git a/ansible/roles/postgres-provision/vars/Fedora-31.yml b/ansible/roles/postgres-provision/vars/Fedora-31.yml new file mode 100644 index 0000000000..27a023eccd --- /dev/null +++ b/ansible/roles/postgres-provision/vars/Fedora-31.yml @@ -0,0 +1,14 @@ +--- +__postgresql_version: "11.5" +__postgresql_data_dir: "/var/lib/pgsql/data" +__postgresql_bin_path: "/usr/bin" +__postgresql_config_path: "/var/lib/pgsql/data" +__postgresql_daemon: postgresql +__postgresql_packages: + - postgresql + - postgresql-server + - postgresql-contrib + - postgresql-libs +__postgresql_unix_socket_directories_mode: '0755' +# Fedora 31 containers only have python3 by default +postgresql_python_library: python3-psycopg2 diff --git a/ansible/roles/postgres-provision/vars/Fedora-32.yml b/ansible/roles/postgres-provision/vars/Fedora-32.yml new file mode 100644 index 0000000000..6ce9d9bfe4 --- /dev/null +++ b/ansible/roles/postgres-provision/vars/Fedora-32.yml @@ -0,0 +1,14 @@ +--- +__postgresql_version: "12.2" +__postgresql_data_dir: "/var/lib/pgsql/data" +__postgresql_bin_path: "/usr/bin" +__postgresql_config_path: "/var/lib/pgsql/data" +__postgresql_daemon: postgresql +__postgresql_packages: + - postgresql + - postgresql-server + - postgresql-contrib + - postgresql-libs +__postgresql_unix_socket_directories_mode: '0755' +# Fedora 32 containers only have python3 by default +postgresql_python_library: python3-psycopg2 diff --git a/ansible/roles/postgres-provision/vars/RedHat-6.yml b/ansible/roles/postgres-provision/vars/RedHat-6.yml deleted file mode 100644 index 8923c50dd9..0000000000 --- a/ansible/roles/postgres-provision/vars/RedHat-6.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -__postgresql_version: "8.4" -__postgresql_data_dir: "/var/lib/pgsql/data" -__postgresql_bin_path: "/usr/bin" -__postgresql_config_path: "/var/lib/pgsql/data" -__postgresql_daemon: postgresql -__postgresql_packages: - - postgresql - - postgresql-server - - postgresql-contrib - - postgresql-libs diff --git a/ansible/roles/postgres-provision/vars/RedHat-8.yml b/ansible/roles/postgres-provision/vars/RedHat-8.yml new file mode 100644 index 0000000000..e519ea9a08 --- /dev/null +++ b/ansible/roles/postgres-provision/vars/RedHat-8.yml @@ -0,0 +1,12 @@ +--- +__postgresql_version: "10" +__postgresql_data_dir: "/var/lib/pgsql/data" +__postgresql_bin_path: "/usr/bin" +__postgresql_config_path: "/var/lib/pgsql/data" +__postgresql_daemon: postgresql +__postgresql_packages: + - postgresql + - postgresql-server + - postgresql-contrib +__postgresql_unix_socket_directories_mode: '0755' +postgresql_python_library: python3-psycopg2 diff --git a/ansible/roles/postgres-provision/vars/Ubuntu-14.yml b/ansible/roles/postgres-provision/vars/Ubuntu-14.yml deleted file mode 100644 index bd6c174757..0000000000 --- a/ansible/roles/postgres-provision/vars/Ubuntu-14.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -__postgresql_version: "9.3" -__postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main" -__postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin" -__postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main" -__postgresql_daemon: postgresql -__postgresql_packages: - - postgresql - - postgresql-contrib - - libpq-dev diff --git a/ansible/roles/postgres-provision/vars/Ubuntu-18.yml b/ansible/roles/postgres-provision/vars/Ubuntu-18.yml index 201acc365e..8136224f2b 100644 --- a/ansible/roles/postgres-provision/vars/Ubuntu-18.yml +++ b/ansible/roles/postgres-provision/vars/Ubuntu-18.yml @@ -8,3 +8,4 @@ __postgresql_packages: - postgresql - postgresql-contrib - libpq-dev +postgresql_python_library: python3-psycopg2 diff --git a/ansible/roles/postgres-provision/vars/Ubuntu-20.yml b/ansible/roles/postgres-provision/vars/Ubuntu-20.yml new file mode 100644 index 0000000000..9a9a065a28 --- /dev/null +++ b/ansible/roles/postgres-provision/vars/Ubuntu-20.yml @@ -0,0 +1,11 @@ +--- +__postgresql_version: "12" +__postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main" +__postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin" +__postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main" +__postgresql_daemon: postgresql +__postgresql_packages: + - postgresql + - postgresql-contrib + - libpq-dev +postgresql_python_library: python3-psycopg2 diff --git a/ansible/roles/postgres-provision/vars/Ubuntu-22.yml b/ansible/roles/postgres-provision/vars/Ubuntu-22.yml new file mode 100644 index 0000000000..650ca25e33 --- /dev/null +++ b/ansible/roles/postgres-provision/vars/Ubuntu-22.yml @@ -0,0 +1,11 @@ +--- +__postgresql_version: "10" +__postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main" +__postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin" +__postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main" +__postgresql_daemon: postgresql +__postgresql_packages: + - postgresql-10 + - postgresql-contrib + - libpq-dev +postgresql_python_library: python3-psycopg2 diff --git a/ansible/roles/postgres/defaults/main.yml b/ansible/roles/postgres/defaults/main.yml deleted file mode 100644 index a908a2df20..0000000000 --- a/ansible/roles/postgres/defaults/main.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -#postgres: - #db_url: "{{ groups['postgres'][0] }}" - #db_username: analytics - #db_name: analytics - #db_password: "{{vault_postges_password}}" - #db_tabel_name: "{{ env}}_consumer_channel_mapping" - #db_admin_user: postgres - #db_port: 5432 - - -#pgdb_password: "{{vault_postges_password}}" -#db_admin_password: "{{vault_postgres_admin_password}}" - -consumer_id: id #mention the consumer ID -channel: channel_id #Channel ID to map with the Consumer ID -status: 1 #It should be either 1 or 0, 1-to activate, 0-to make it inactive -created_by: analytics-team diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml deleted file mode 100644 index 9da7f6a158..0000000000 --- a/ansible/roles/postgres/tasks/main.yml +++ /dev/null @@ -1,64 +0,0 @@ -- name: Install PostgreSQL client - apt: name={{ item }} update_cache=yes cache_valid_time=36000 state=installed - with_items: - - python-psycopg2 - - postgresql-client - ignore_errors: yes - -- name: Ensure database is created - postgresql_db: name="{{ postgres.db_name }}" \ - login_host="{{ postgres.db_url }}" \ - port="{{ postgres.db_port }}" \ - login_user="{{ postgres.db_admin_user }}" \ - login_password="{{ postgres.db_admin_password }}" \ - encoding='UTF-8' \ - state=present - tags: createdb - -- name: Ensure user has access to the database - postgresql_user: name="{{ postgres.db_username }}" \ - password="{{ postgres.db_password }}" \ - no_password_changes=true \ - priv=ALL \ - state=present \ - login_host="{{ postgres.db_url }}" \ - port="{{ postgres.db_port }}" \ - login_user="{{ postgres.db_admin_user }}" \ - login_password="{{ postgres.db_admin_password }}" \ - db="{{ postgres.db_name }}" - tags: createuser - -- name: create {{ env }}_consumer-channel-mapping - shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_consumer_channel_mapping(consumer_id text, channel text, status smallint, created_by text, created_on timestamptz, updated_on timestamptz, PRIMARY KEY(consumer_id, channel))" - run_once: true - tags: create - -- name: update {{ env }}_consumer_channel_mapping - shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "insert into {{ env }}_consumer_channel_mapping (consumer_id, channel, status, created_by, created_on, updated_on) values ('{{ consumer_id }}', '{{ channel }}', {{ status }}, '{{created_by }}', current_timestamp, current_timestamp) on conflict on constraint {{ env }}_consumer_channel_mapping_pkey DO UPDATE set status=EXCLUDED.status, updated_on= current_timestamp" - run_once: true - tags: insert - -- name: create {{ env }}_device_profile - shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_device_profile(device_id text, api_last_updated_on timestamptz, avg_ts float, city text, country text, country_code text, device_spec json, district_custom text, fcm_token text, first_access timestamptz, last_access timestamptz, producer_id text, state text, state_code text, state_code_custom text, state_custom text, total_launches bigint, total_ts float, uaspec json, updated_date timestamptz, user_declared_district text, user_declared_state text, user_declared_on timestamptz, PRIMARY KEY(device_id))" - run_once: true - tags: create - -- name: create {{ env }}_report_config - shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_report_config(report_id text, updated_on timestamptz,report_description text,requested_by text,report_schedule text,config json,created_on timestamptz,submitted_on timestamptz,status text,status_msg text,PRIMARY KEY(report_id))" - run_once: true - tags: create - -- name: create report - shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report(reportid varchar(40) NOT NULL PRIMARY KEY, title text NOT NULL, description text NOT NULL, authorizedroles jsonb NOT NULL, status varchar(8) NOT NULL CHECK (status IN ('live', 'draft', 'retired')) DEFAULT 'draft', type varchar(8) NOT NULL CHECK (type in ('public', 'private')) DEFAULT 'private', reportaccessurl text NOT NULL UNIQUE, createdon timestamptz NOT NULL DEFAULT now(), updatedon timestamptz NOT NULL DEFAULT now(), createdby varchar(50) NOT NULL, reportconfig jsonb NOT NULL, templateurl text, slug varchar(10) NOT NULL, reportgenerateddate timestamptz NOT NULL DEFAULT now(), reportduration jsonb NOT NULL DEFAULT jsonb_build_object('startDate', now()::timestamptz, 'endDate', now()::timestamptz), tags jsonb NOT NULL, updatefrequency text NOT NULL)" - run_once: true - tags: create - -- name: create report_summary - shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report_summary(id varchar(40) NOT NULL PRIMARY KEY, reportid varchar(40) NOT NULL, chartid text, createdon timestamptz NOT NULL DEFAULT now(), createdby varchar(50) NOT NULL, summary text NOT NULL)" - run_once: true - tags: create - -- name: create {{ env }}_consumer-channel-mapping - shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "select * from {{ env }}_consumer_channel_mapping" - run_once: true - tags: select diff --git a/ansible/roles/postgresql-backup/tasks/main.yml b/ansible/roles/postgresql-backup/tasks/main.yml index 9e3ee22008..357a30680b 100755 --- a/ansible/roles/postgresql-backup/tasks/main.yml +++ b/ansible/roles/postgresql-backup/tasks/main.yml @@ -13,12 +13,12 @@ async: 3600 poll: 10 -- name: Ensure azure blob storage container exists - command: az storage container create --name {{ postgresql_backup_azure_container_name }} - ignore_errors: true - environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}" - AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}" +#- name: Ensure azure blob storage container exists +# command: az storage container create --name {{ postgresql_backup_azure_container_name }} +# ignore_errors: true +# environment: +# AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}" +# AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}" - name: Upload to azure blob storage command: az storage blob upload --name {{ postgresql_backup_gzip_file_name }} --file {{ postgresql_backup_gzip_file_path }} --container-name {{ postgresql_backup_azure_container_name }} diff --git a/ansible/roles/provision-azure-spark-cluster/defaults/main.yml b/ansible/roles/provision-azure-spark-cluster/defaults/main.yml new file mode 100644 index 0000000000..f487bf69cb --- /dev/null +++ b/ansible/roles/provision-azure-spark-cluster/defaults/main.yml @@ -0,0 +1,19 @@ +spark_folder: /usr/hdp/current/spark2-client +guava_version: 19.0 +log4j_version: 2.16.0 +spark_redis_version: 2.5.0 +guava_default_jre_version: 28.0-jre +guava_jre_version: 24.1.1-jre +jedis_version: 3.2.0 +zip4j_version: 2.6.2 +guice_default_version: 4.0 +guice_version: 3.0 +guava_url: https://repo1.maven.org/maven2/com/google/guava/guava/{{guava_version}}/guava-{{guava_version}}.jar +guava_jre_url: https://repo1.maven.org/maven2/com/google/guava/guava/{{guava_jre_version}}/guava-{{guava_jre_version}}.jar +log4j_core_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/{{log4j_version}}/log4j-core-{{log4j_version}}.jar +log4j_api_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/{{log4j_version}}/log4j-api-{{log4j_version}}.jar +spark_redis_url: https://repo1.maven.org/maven2/com/redislabs/spark-redis_2.11/{{spark_redis_version}}/spark-redis_2.11-{{spark_redis_version}}.jar +jedis_url: https://repo1.maven.org/maven2/redis/clients/jedis/{{jedis_version}}/jedis-{{jedis_version}}.jar +zip4j_url: https://repo1.maven.org/maven2/net/lingala/zip4j/zip4j/{{zip4j_version}}/zip4j-{{zip4j_version}}.jar +guice_url: https://repo1.maven.org/maven2/com/google/inject/guice/{{guice_version}}/guice-{{guice_version}}.jar +guice_servlet_url: https://repo1.maven.org/maven2/com/google/inject/extensions/guice-servlet/{{guice_version}}/guice-servlet-{{guice_version}}.jar \ No newline at end of file diff --git a/ansible/roles/provision-azure-spark-cluster/tasks/main.yml b/ansible/roles/provision-azure-spark-cluster/tasks/main.yml new file mode 100644 index 0000000000..f4cbc36e58 --- /dev/null +++ b/ansible/roles/provision-azure-spark-cluster/tasks/main.yml @@ -0,0 +1,64 @@ +- name: Adding azure blob variable to spark env file + lineinfile: + path: "{{spark_folder}}/conf/spark-env.sh" + line: '{{item.var}}={{item.value}}' + regexp: "{{ item.var }}.*" + with_items: + - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'} + - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'} + no_log: true + +- name: Remove guava-jre, guice default jars + become: yes + file: + path: "{{ spark_folder }}/jars/{{item.var}}-{{item.value}}.jar" + state: absent + with_items: + - {var: 'guava', value: '{{ guava_default_jre_version }}'} + - {var: 'guice', value: '{{ guice_default_version }}'} + - {var: 'guice-servlet', value: '{{ guice_default_version }}'} + +- name: Download guava_jre_url and copy to Spark jars folder + become: yes + get_url: url={{ guava_jre_url }} dest={{ spark_folder }}/jars/guava-{{guava_jre_version}}.jar timeout=1000 force=no + +- name: Download log4j api and copy to Spark jars folder + become: yes + get_url: url={{ log4j_api_url }} dest={{ spark_folder }}/jars/log4j-api-{{log4j_version}}.jar timeout=1000 force=no + +- name: Download log4j core and copy to Spark jars folder + become: yes + get_url: url={{ log4j_core_url }} dest={{ spark_folder }}/jars/log4j-core-{{log4j_version}}.jar timeout=1000 force=no + +- name: Download spark-redis and copy to Spark jars folder + become: yes + get_url: url={{ spark_redis_url }} dest={{ spark_folder }}/jars/spark-redis_2.11-{{spark_redis_version}}.jar timeout=1000 force=no + +- name: Download jedis and copy to Spark jars folder + become: yes + get_url: url={{ jedis_url }} dest={{ spark_folder }}/jars/jedis-{{jedis_version}}.jar timeout=1000 force=no + +- name: Download zip4j and copy to Spark jars folder + become: yes + get_url: url={{ zip4j_url }} dest={{ spark_folder }}/jars/zip4j-{{zip4j_version}}.jar timeout=1000 force=no + +- name: Download guice and copy to Spark jars folder + become: yes + get_url: url={{ guice_url }} dest={{ spark_folder }}/jars/guice-{{guice_version}}.jar timeout=1000 force=no + +- name: Download guice-servlet and copy to Spark jars folder + become: yes + get_url: url={{ guice_servlet_url }} dest={{ spark_folder }}/jars/guice-servlet-{{guice_version}}.jar timeout=1000 force=no + +- name: Update log4j.properties + become: yes + blockinfile: + path: "{{ spark_folder }}/conf/log4j.properties" + block: | + log4j.logger.org.ekstep.analytics=INFO + log4j.appender.org.ekstep.analytics=org.apache.log4j.RollingFileAppender + log4j.appender.org.ekstep.analytics.File=./joblog.log + log4j.appender.org.ekstep.analytics.MaxFileSize=${log4jspark.log.maxfilesize} + log4j.appender.org.ekstep.analytics.MaxBackupIndex=${log4jspark.log.maxbackupindex} + log4j.appender.org.ekstep.analytics.layout=org.apache.log4j.PatternLayout + log4j.appender.org.ekstep.analytics.layout.ConversionPattern=%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n diff --git a/ansible/roles/provision-geo-location-db/defaults/main.yml b/ansible/roles/provision-geo-location-db/defaults/main.yml index 2b126ea7c4..666fae2ff1 100644 --- a/ansible/roles/provision-geo-location-db/defaults/main.yml +++ b/ansible/roles/provision-geo-location-db/defaults/main.yml @@ -1,6 +1,5 @@ -download_user: deployer -download_group: deployer -maxmind_geocity_db_url: "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City-CSV.zip" +download_user: "{{ ansible_ssh_user }}" +download_group: "{{ ansible_ssh_user }}" maxmind_geoip2_csv_converter_url: "https://github.com/maxmind/geoip2-csv-converter/releases/download/v1.0.0/geoip2-csv-converter-v1.0.0-linux-amd64.tar.gz" maxmind_db_download_dir: /tmp maxmind_db_dir_name: "geoip2-city-csv" @@ -9,4 +8,4 @@ maxmind_db_unarchived_dir_prefix: "GeoLite2-City-CSV_*" maxmind_db_geo_city_blocks_filename: "GeoLite2-City-Blocks-IPv4.csv" maxmind_db_geo_city_locations_filename: "GeoLite2-City-Locations-en.csv" maxmind_db_geo_city_ip_range_filename: "GeoLite2-City-Range-IPv4.csv" -maxmind_custom_data_mapping_url: "https://sunbirddevtelemetry.blob.core.windows.net/public/maxmind_custom_data_mapping.csv" \ No newline at end of file +maxmind_custom_data_mapping_url: "https://sunbirddevtelemetry.blob.core.windows.net/public/maxmind_custom_data_mapping.csv" diff --git a/ansible/roles/provision-geo-location-db/files/.gitkeep b/ansible/roles/provision-geo-location-db/files/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ansible/roles/provision-geo-location-db/tasks/main.yml b/ansible/roles/provision-geo-location-db/tasks/main.yml index 74ef8a2497..e3f9d3fe9b 100644 --- a/ansible/roles/provision-geo-location-db/tasks/main.yml +++ b/ansible/roles/provision-geo-location-db/tasks/main.yml @@ -16,8 +16,11 @@ become: yes file: path={{ maxmind_db_download_dir }}/{{ maxmind_db_dir_name }}.zip state=absent -- name: Download Maxmind GeoCity database - get_url: url={{ maxmind_geocity_db_url }} dest={{ maxmind_db_download_dir }}/{{ maxmind_db_dir_name }}.zip force=no +- name: Copy the Maxmind GeoCity database to remote + copy: + src: "files/{{ geoip_zip_csv_file_name }}" + dest: "{{ maxmind_db_download_dir }}/{{ maxmind_db_dir_name }}.zip" + force: no - name: Unarchive Maxmind GeoCity database unarchive: src={{ maxmind_db_download_dir }}/{{ maxmind_db_dir_name }}.zip dest={{ maxmind_db_download_dir }} copy=no diff --git a/ansible/roles/redis-backup/defaults/main.yml b/ansible/roles/redis-backup/defaults/main.yml index 155d6ea162..fc2a2bd83a 100644 --- a/ansible/roles/redis-backup/defaults/main.yml +++ b/ansible/roles/redis-backup/defaults/main.yml @@ -1,2 +1,6 @@ redis_backup_dir: /tmp/redis-backup -redis_backup_azure_container_name: dp-redis-backup +redis_user: analytics +redis_version: 6.2.5 +redis_data_dir: "/home/{{ redis_user }}/redis-{{ redis_version }}/data" +backup_bucket: backup-bucket +redis_backup_folder: redis-backup-{{ ansible_host }} \ No newline at end of file diff --git a/ansible/roles/redis-backup/tasks/main.yml b/ansible/roles/redis-backup/tasks/main.yml index 6218852add..e6140d91b6 100644 --- a/ansible/roles/redis-backup/tasks/main.yml +++ b/ansible/roles/redis-backup/tasks/main.yml @@ -1,33 +1,26 @@ - name: Create the directory file: path={{ redis_backup_dir }} state=directory recurse=yes - - set_fact: - redis_backup_file_name: "redis-backup-{{ lookup('pipe', 'date +%Y-%m-%d-%T') }}-{{ ansible_hostname }}.zip" - + redis_backup_file_name: "redis-backup-{{ lookup('pipe', 'date +%Y-%m-%d-%T') }}.rdb" + - set_fact: redis_backup_file_path: "{{ redis_backup_dir }}/{{ redis_backup_file_name }}" - - name: copy dump.rdb file copy: - src: /home/analytics/redis-stable/dump.rdb - dest: "{{ redis_backup_dir }}/dump.rdb" + src: "{{ redis_data_dir }}/dump.rdb" + dest: "{{ redis_backup_dir }}/{{ redis_backup_file_name }}" remote_src: yes -- name: Create archive of backup directory - archive: path="{{ redis_backup_dir }}/*" dest="{{ redis_backup_dir }}/{{ redis_backup_file_name }}" format=zip - async: 3600 - poll: 10 - - -- name: upload to azure +- name: upload to gcs include_role: - name: artifacts-upload-azure + name: artifacts-upload-gcp vars: - artifact: "{{ redis_backup_file_name }}" + artifacts_container: "{{ backup_bucket }}/{{ redis_backup_folder }}" artifact_path: "{{ redis_backup_file_path }}" - artifacts_container: "{{ redis_backup_azure_container_name }}" - + artifact: "{{ redis_backup_file_name }}" + service_account: "{{ gcp_backups_service_account }}" + - name: clean up backup dir after upload file: path={{ redis_backup_dir }} state=absent diff --git a/ansible/roles/redis-multiprocess-backup/defaults/main.yml b/ansible/roles/redis-multiprocess-backup/defaults/main.yml new file mode 100644 index 0000000000..b089819b99 --- /dev/null +++ b/ansible/roles/redis-multiprocess-backup/defaults/main.yml @@ -0,0 +1,13 @@ +redis_backup_dir: /tmp/redis-backup +redis_backup_azure_container_name: dp-redis-backup +redis_data_dir: /data +redis: + config: + device: + name: 'device' + user: + name: 'user' + content: + name: 'content' + dialcode: + name: 'dialcode' diff --git a/ansible/roles/redis-backup/meta/main.yml b/ansible/roles/redis-multiprocess-backup/meta/main.yml similarity index 100% rename from ansible/roles/redis-backup/meta/main.yml rename to ansible/roles/redis-multiprocess-backup/meta/main.yml diff --git a/ansible/roles/redis-multiprocess-backup/tasks/main.yml b/ansible/roles/redis-multiprocess-backup/tasks/main.yml new file mode 100644 index 0000000000..f59e63ce75 --- /dev/null +++ b/ansible/roles/redis-multiprocess-backup/tasks/main.yml @@ -0,0 +1,31 @@ +- name: Create the directory + file: path={{ redis_backup_dir }}/{{ item.value.name }} state=directory recurse=yes + with_dict: "{{ redis.config }}" + +- set_fact: + redis_backup_file_name: "redis-backup-{{ lookup('pipe', 'date +%Y-%m-%d-%T') }}-{{ ansible_hostname }}.zip" + +- name: copy dump.rdb files + copy: + src: "{{ redis_data_dir }}/{{ item.value.name }}-dump.rdb" + dest: "{{ redis_backup_dir }}/{{ item.value.name }}/{{ item.value.name }}-dump.rdb" + remote_src: yes + with_dict: "{{ redis.config }}" + +- name: Create archive of backup directory + archive: path="{{ redis_backup_dir }}/{{ item.value.name }}/*" dest="{{ redis_backup_dir }}/{{ item.value.name }}/{{ redis_backup_file_name }}" format=zip + async: 3600 + poll: 10 + with_dict: "{{ redis.config }}" + +- name: upload to azure + include_role: + name: artifacts-upload-azure + vars: + artifact: "{{ item.value.name }}/{{ redis_backup_file_name }}" + artifact_path: "{{ redis_backup_dir }}/{{ item.value.name }}/{{ redis_backup_file_name }}" + artifacts_container: "{{ redis_backup_azure_container_name }}" + with_dict: "{{ redis.config }}" + +- name: clean up backup dir after upload + file: path={{ redis_backup_dir }} state=absent diff --git a/ansible/roles/redis-multiprocess-restore/defaults/main.yml b/ansible/roles/redis-multiprocess-restore/defaults/main.yml new file mode 100644 index 0000000000..d39efbb3ae --- /dev/null +++ b/ansible/roles/redis-multiprocess-restore/defaults/main.yml @@ -0,0 +1,9 @@ +redis_user: analytics +redis_restore_user: analytics +redis_backup_azure_container_name: dp-redis-backup +analytics_user_home: /home/analytics + +device_port: "{{ device_port }}" +user_port: "{{ user_port }}" +content_port: "{{ content_port }}" +dialcode_port: "{{ dialcode_port }}" \ No newline at end of file diff --git a/ansible/roles/redis-multiprocess-restore/meta/main.yml b/ansible/roles/redis-multiprocess-restore/meta/main.yml new file mode 100644 index 0000000000..a124d4f7cb --- /dev/null +++ b/ansible/roles/redis-multiprocess-restore/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - azure-cli diff --git a/ansible/roles/redis-multiprocess-restore/tasks/main.yml b/ansible/roles/redis-multiprocess-restore/tasks/main.yml new file mode 100644 index 0000000000..8acb3c1a16 --- /dev/null +++ b/ansible/roles/redis-multiprocess-restore/tasks/main.yml @@ -0,0 +1,55 @@ +--- + +- name: Create the directory + become: yes + become_user: "{{ redis_restore_user }}" + file: path=/tmp/{{ item }} state=directory recurse=yes + with_items: "{{ redis_restore_process }}" + +- name: Download backup file + become: yes + become_user: "{{ redis_restore_user }}" + shell: "az storage blob download --container-name {{ redis_backup_azure_container_name }} --file {{ item }}/{{ redis_restore_file_name }} --name {{ item }}/{{ redis_restore_file_name }} --account-name {{sunbird_management_storage_account_name}} --account-key {{sunbird_management_storage_account_key}}" + args: + chdir: /tmp/ + with_items: "{{ redis_restore_process }}" + +- name: get redis process ports to stop + shell: "echo {{ lookup ('vars', item + '_port') }}" + register: ports + with_items: "{{ redis_restore_process }}" + +- name: stop redis to restore backup + become: yes + become_user: "{{ redis_restore_user }}" + shell: "echo shutdown | {{ analytics_user_home }}/redis-stable/src/redis-cli -p {{ item.stdout }}" + with_items: "{{ ports.results }}" + +- name: Extract backup zip into tmp + become: yes + become_user: "{{ redis_restore_user }}" + unarchive: + src: "/tmp/{{ item }}/{{ redis_restore_file_name }}" + dest: "/tmp/{{ item }}" + remote_src: True + with_items: "{{ redis_restore_process }}" + +- name: copy dump.rdb files to data dir + become: yes + become_user: "{{ redis_restore_user }}" + shell: cp /tmp/{{ item }}/{{ item }}-dump.rdb {{ analytics_user_home }}/redis-stable/{{ item }}-dump.rdb + with_items: "{{ redis_restore_process }}" + +- name: start redis + become: yes + become_user: "{{ redis_restore_user }}" + command: '{{ analytics_user_home }}/redis-stable/src/redis-server {{ analytics_user_home }}/redis-stable/redis-{{ item }}.conf' + with_items: "{{ redis_restore_process }}" + +- name: Keyspace info + shell: "echo info keyspace | {{ analytics_user_home }}/redis-stable/src/redis-cli -p {{ item.stdout }}" + register: restoreinfo + with_items: "{{ ports.results }}" + +- debug: var=item.stdout_lines + with_items: "{{ restoreinfo.results }}" diff --git a/ansible/roles/redis-multiprocess/defaults/main.yml b/ansible/roles/redis-multiprocess/defaults/main.yml new file mode 100644 index 0000000000..404aaa56d7 --- /dev/null +++ b/ansible/roles/redis-multiprocess/defaults/main.yml @@ -0,0 +1,28 @@ +redis_user: analytics +analytics_redis_user_home: /home/{{ redis_user }} +redis_home: "{{ analytics_redis_user_home }}/redis/redis-{{ redis_version }}" +redis_tar_ball: "http://download.redis.io/releases/redis-{{ redis_version }}.tar.gz" +redis_version: 6.2.5 +redis_dir: "{{ analytics_redis_user_home }}/redis" +redis_dump_rdb_time_in_seconds: 900 +redis_dump_rdb_keys_to_save_state: 100 +redis_data_dir: /data + +redis: + config: + device: + port: "{{ device_port }}" + name: 'device' + max_memory: 100mb + user: + port: "{{ user_port }}" + name: 'user' + max_memory: 100mb + content: + port: "{{ content_port }}" + name: 'content' + max_memory: 100mb + dialcode: + port: "{{ dialcode_port }}" + name: 'dialcode' + max_memory: 100mb diff --git a/ansible/roles/redis-multiprocess/tasks/main.yml b/ansible/roles/redis-multiprocess/tasks/main.yml new file mode 100755 index 0000000000..7d1d13c1a3 --- /dev/null +++ b/ansible/roles/redis-multiprocess/tasks/main.yml @@ -0,0 +1,60 @@ +--- +# tasks file for redis +- name: change the data dir ownership + file: + path: /data + state: directory + owner: "{{ redis_user }}" + group: "{{ redis_user }}" + mode: 0755 + become: yes + +- name: create the redis dir + become: yes + become_user: "{{ redis_user }}" + file: + path: "{{ analytics_redis_user_home }}/redis" + state: directory + +- name: download redis zipfile + become: yes + become_user: "{{ redis_user }}" + get_url: url={{ redis_tar_ball }} dest={{ redis_dir }}/redis-{{ redis_version }}.tar.gz timeout=50 force=no owner={{ redis_user }} + +- name: unzip + become: yes + become_user: "{{ redis_user }}" + unarchive: src={{ redis_dir }}/redis-{{ redis_version }}.tar.gz dest={{ redis_dir }} copy=no group={{ redis_user }} owner={{ redis_user }} creates={{ redis_home }} + +- name: Change ownership of redis installation + become: yes + file: path={{ redis_home }} owner={{ redis_user }} group={{ redis_user }} state=directory recurse=yes + +- name: Copy redis conf files + become: yes + become_user: "{{ redis_user }}" + template: src=redis.conf.j2 dest={{ redis_dir }}/redis-{{ redis_version }}/redis-{{ item.value.name }}.conf + vars: + _port_: "{{ item.value.port }}" + _name_: "{{ item.value.name }}" + max_memory: "{{ item.value.max_memory }}" + with_dict: "{{ redis.config }}" + +- name: Install required utils + apt: name={{ item }} state=present + become: yes + with_items: + - make + - gcc + - build-essential + +- name: Go to the folder and run make + become: yes + become_user: "{{ redis_user }}" + command: chdir={{ redis_home }} make + +- name: start redis + become: yes + become_user: "{{ redis_user }}" + command: '{{ redis_dir }}/redis-{{ redis_version }}/src/redis-server {{ redis_dir }}/redis-{{ redis_version }}/redis-{{ item.value.name }}.conf' + with_dict: "{{ redis.config }}" diff --git a/ansible/roles/redis-multiprocess/templates/redis.conf.j2 b/ansible/roles/redis-multiprocess/templates/redis.conf.j2 new file mode 100644 index 0000000000..456cc0d42f --- /dev/null +++ b/ansible/roles/redis-multiprocess/templates/redis.conf.j2 @@ -0,0 +1,1026 @@ +# Redis configuration file example. +# +# Note that in order to read the configuration file, Redis must be +# started with the file path as first argument: +# +# ./redis-server /path/to/redis.conf + +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis servers but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################## NETWORK ##################################### + +# By default, if no "bind" configuration directive is specified, Redis listens +# for connections from all the network interfaces available on the server. +# It is possible to listen to just one or multiple selected interfaces using +# the "bind" configuration directive, followed by one or more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 +# bind 127.0.0.1 ::1 +# +# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the +# internet, binding to all the interfaces is dangerous and will expose the +# instance to everybody on the internet. So by default we uncomment the +# following bind directive, that will force Redis to listen only into +# the IPv4 lookback interface address (this means Redis will be able to +# accept connections only from clients running into the same computer it +# is running). +# +# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES +# JUST COMMENT THE FOLLOWING LINE. +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# bind 127.0.0.1 + +# Protected mode is a layer of security protection, in order to avoid that +# Redis instances left open on the internet are accessed and exploited. +# +# When protected mode is on and if: +# +# 1) The server is not binding explicitly to a set of addresses using the +# "bind" directive. +# 2) No password is configured. +# +# The server only accepts connections from clients connecting from the +# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain +# sockets. +# +# By default protected mode is enabled. You should disable it only if +# you are sure you want clients from other hosts to connect to Redis +# even if no authentication is configured, nor a specific set of interfaces +# are explicitly listed using the "bind" directive. +protected-mode no + +# Accept connections on the specified port, default is {{ _port_ }} (IANA #815344). +# If port 0 is specified Redis will not listen on a TCP socket. +port {{ _port_ }} + +# TCP listen() backlog. +# +# In high requests-per-second environments you need an high backlog in order +# to avoid slow clients connections issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to get the desired effect. +tcp-backlog 511 + +# Unix socket. +# +# Specify the path for the Unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +# unixsocket /tmp/redis.sock +# unixsocketperm 700 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Take the connection alive from the point of view of network +# equipment in the middle. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 300 seconds, which is the new +# Redis default starting with Redis 3.2.1. +tcp-keepalive 300 + +################################# GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize yes + +# If you run Redis from upstart or systemd, Redis can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous liveness pings back to your supervisor. +supervised systemd + +# If a pid file is specified, Redis writes it where specified at startup +# and removes it at exit. +# +# When the server runs non daemonized, no pid file is created if none is +# specified in the configuration. When the server is daemonized, the pid file +# is used even if not specified, defaulting to "/var/run/redis.pid". +# +# Creating a pid file is best effort: if Redis is not able to create it +# nothing bad happens, the server will start and run normally. +pidfile /var/run/redis_{{ _name_ }}.pid + +# Specify the server verbosity level. +# This can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile "{{ redis_data_dir }}/redis-{{ _name_ }}.log" + + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +################################ SNAPSHOTTING ################################ +# +# Save the DB on disk: +# +# save +# +# Will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# In the example below the behaviour will be to save: +# after 900 sec (15 min) if at least 1 key changed +# after 300 sec (5 min) if at least 10 keys changed +# after 60 sec if at least 10000 keys changed +# +# Note: you can disable saving completely by commenting out all "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# +# save "" + +#save 900 1 +#save 300 10 +#save 60 10000 +save {{ redis_dump_rdb_time_in_seconds }} {{ redis_dump_rdb_keys_to_save_state }} + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in a hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# disaster will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usual even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + +# Compress string objects using LZF when dump .rdb databases? +# For default that's set to 'yes' as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# The filename where to dump the DB +dbfilename {{ _name_ }}-dump.rdb + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# The Append Only File will also be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir "{{redis_data_dir}}" + + +################################# REPLICATION ################################# + +# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# another Redis server. A few things to understand ASAP about Redis replication. +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of slaves. +# 2) Redis slaves are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition slaves automatically try to reconnect to masters +# and resynchronize with them. +# +# slaveof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth + +# When a slave loses its connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Replication SYNC strategy: disk or socket. +# +# ------------------------------------------------------- +# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY +# ------------------------------------------------------- +# +# New slaves and reconnecting slaves that are not able to continue the replication +# process just receiving differences, need to do what is called a "full +# synchronization". An RDB file is transmitted from the master to the slaves. +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the slaves incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to slave sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more slaves +# can be queued and served with the RDB file as soon as the current child producing +# the RDB file finishes its work. With diskless replication instead once +# the transfer starts, new slaves arriving will be queued and a new transfer +# will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple slaves +# will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that transfers the RDB via socket +# to the slaves. +# +# This is important since once the transfer starts, it is not possible to serve +# new slaves arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more slaves arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of slave. +# 2) Master timeout from the point of view of slaves (data, pings). +# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +# repl-timeout 60 + +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# slave data when slaves are disconnected for some time, so that when a slave +# wants to reconnect again, often a full resync is not needed, but a partial +# resync is enough, just passing the portion of data the slave missed while +# disconnected. +# +# The bigger the replication backlog, the longer the time the slave can be +# disconnected and later be able to perform a partial resynchronization. +# +# The backlog is only allocated once there is at least a slave connected. +# +# repl-backlog-size 1mb + +# After a master has no longer connected slaves for some time, the backlog +# will be freed. The following option configures the amount of seconds that +# need to elapse, starting from the time the last slave disconnected, for +# the backlog buffer to be freed. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The slave priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the slave as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# It is possible for a master to stop accepting writes if there are less than +# N slaves connected, having a lag less or equal than M seconds. +# +# The N slaves need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the slave, that is usually sent every second. +# +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough slaves +# are available, to the specified number of seconds. +# +# For example to require at least 3 slaves with a lag <= 10 seconds use: +# +# min-slaves-to-write 3 +# min-slaves-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-slaves-to-write is set to 0 (feature disabled) and +# min-slaves-max-lag is set to 10. + +################################## SECURITY ################################### + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running redis-server. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to slaves may cause problems. + +################################### LIMITS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# maxclients 10000 + +# Don't use more memory than the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# according to the eviction policy selected (see maxmemory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# a hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). +# +maxmemory {{ max_memory }} + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select among five behaviors: +# +# volatile-lru -> remove the key with an expire set using an LRU algorithm +# allkeys-lru -> remove any key according to the LRU algorithm +# volatile-random -> remove a random key with an expire set +# allkeys-random -> remove a random key, any key +# volatile-ttl -> remove the key with the nearest expire time (minor TTL) +# noeviction -> don't expire at all, just return an error on write operations +# +# Note: with any of the above policies, Redis will return an error on write +# operations, when there are no suitable keys for eviction. +# +# At the date of writing these commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +# maxmemory-policy noeviction + +# LRU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can tune it for speed or +# accuracy. For default Redis will check five keys and pick the one that was +# used less recently, you can change the sample size using the following +# configuration directive. +# +# The default of 5 produces good enough results. 10 Approximates very closely +# true LRU but costs a bit more CPU. 3 is very fast but not very accurate. +# +# maxmemory-samples 5 + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. + +appendonly no + +# The name of the append only file (default: "appendonly.aof") + +appendfilename "appendonly.aof" + +# The fsync() call tells the Operating System to actually write data on disk +# instead of waiting for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log. Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec", as that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. + +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (if no rewrite has happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceeds the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################ REDIS CLUSTER ############################### +# +# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# WARNING EXPERIMENTAL: Redis Cluster is considered to be stable code, however +# in order to mark it as "mature" we need to wait for a non trivial percentage +# of users to deploy it in production. +# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# +# Normal Redis instances can't be part of a Redis Cluster; only nodes that are +# started as cluster nodes can. In order to start a Redis instance as a +# cluster node enable the cluster support uncommenting the following: +# +# cluster-enabled yes + +# Every cluster node has a cluster configuration file. This file is not +# intended to be edited by hand. It is created and updated by Redis nodes. +# Every Redis Cluster node requires a different cluster configuration file. +# Make sure that instances running in the same system do not have +# overlapping cluster configuration file names. +# +# cluster-config-file nodes-{{ _name_ }}.conf + +# Cluster node timeout is the amount of milliseconds a node must be unreachable +# for it to be considered in failure state. +# Most other internal time limits are multiple of the node timeout. +# +# cluster-node-timeout 15000 + +# A slave of a failing master will avoid to start a failover if its data +# looks too old. +# +# There is no simple way for a slave to actually have a exact measure of +# its "data age", so the following two checks are performed: +# +# 1) If there are multiple slaves able to failover, they exchange messages +# in order to try to give an advantage to the slave with the best +# replication offset (more data from the master processed). +# Slaves will try to get their rank by offset, and apply to the start +# of the failover a delay proportional to their rank. +# +# 2) Every single slave computes the time of the last interaction with +# its master. This can be the last ping or command received (if the master +# is still in the "connected" state), or the time that elapsed since the +# disconnection with the master (if the replication link is currently down). +# If the last interaction is too old, the slave will not try to failover +# at all. +# +# The point "2" can be tuned by user. Specifically a slave will not perform +# the failover if, since the last interaction with the master, the time +# elapsed is greater than: +# +# (node-timeout * slave-validity-factor) + repl-ping-slave-period +# +# So for example if node-timeout is 30 seconds, and the slave-validity-factor +# is 10, and assuming a default repl-ping-slave-period of 10 seconds, the +# slave will not try to failover if it was not able to talk with the master +# for longer than 310 seconds. +# +# A large slave-validity-factor may allow slaves with too old data to failover +# a master, while a too small value may prevent the cluster from being able to +# elect a slave at all. +# +# For maximum availability, it is possible to set the slave-validity-factor +# to a value of 0, which means, that slaves will always try to failover the +# master regardless of the last time they interacted with the master. +# (However they'll always try to apply a delay proportional to their +# offset rank). +# +# Zero is the only value able to guarantee that when all the partitions heal +# the cluster will always be able to continue. +# +# cluster-slave-validity-factor 10 + +# Cluster slaves are able to migrate to orphaned masters, that are masters +# that are left without working slaves. This improves the cluster ability +# to resist to failures as otherwise an orphaned master can't be failed over +# in case of failure if it has no working slaves. +# +# Slaves migrate to orphaned masters only if there are still at least a +# given number of other working slaves for their old master. This number +# is the "migration barrier". A migration barrier of 1 means that a slave +# will migrate only if there is at least 1 other working slave for its master +# and so forth. It usually reflects the number of slaves you want for every +# master in your cluster. +# +# Default is 1 (slaves migrate only if their masters remain with at least +# one slave). To disable migration just set it to a very large value. +# A value of 0 can be set but is useful only for debugging and dangerous +# in production. +# +# cluster-migration-barrier 1 + +# By default Redis Cluster nodes stop accepting queries if they detect there +# is at least an hash slot uncovered (no available node is serving it). +# This way if the cluster is partially down (for example a range of hash slots +# are no longer covered) all the cluster becomes, eventually, unavailable. +# It automatically returns available as soon as all the slots are covered again. +# +# However sometimes you want the subset of the cluster which is working, +# to continue to accept queries for the part of the key space that is still +# covered. In order to do so, just set the cluster-require-full-coverage +# option to no. +# +# cluster-require-full-coverage yes + +# In order to setup your cluster make sure to read the documentation +# available at http://redis.io web site. + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enabled at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# EVENT NOTIFICATION ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at http://redis.io/topics/notifications +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# +# The "notify-keyspace-events" takes as argument a string that is composed +# of zero or multiple characters. The empty string means that notifications +# are disabled. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 + +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happen to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into a hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# actively rehash the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply from time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients including MONITOR clients +# slave -> slave clients +# pubsub -> clients subscribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled by setting them to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeout, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are performed with the same frequency, but Redis checks for +# tasks to perform according to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes diff --git a/ansible/roles/redis-multiprocess/templates/systemd.j2 b/ansible/roles/redis-multiprocess/templates/systemd.j2 new file mode 100644 index 0000000000..b9633f9183 --- /dev/null +++ b/ansible/roles/redis-multiprocess/templates/systemd.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Redis In-Memory Data Store +After=network.target + +[Service] +User={{ redis_user }} +Group={{ redis_user }} +ExecStart={{ redis_dir }}/redis-stable/src/redis-server {{ redis_dir }}/redis-stable/redis-_port_.conf +ExecStop={{ redis_dir }}/redis-stable/src/redis-cli -p _port_ shutdown +Restart=always +Type=notify + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/redis/defaults/main.yml b/ansible/roles/redis/defaults/main.yml index 9d07c5517c..9eeb548f27 100644 --- a/ansible/roles/redis/defaults/main.yml +++ b/ansible/roles/redis/defaults/main.yml @@ -1,11 +1,10 @@ redis_user: analytics -analytics_user_home: /home/{{ redis_user }} -redis_home: "{{ analytics_user_home }}/redis-stable" -redis_tar_ball: http://download.redis.io/releases/redis-stable.tar.gz +analytics_redis_user_home: /home/{{ redis_user }} +redis_home: "{{ analytics_redis_user_home }}/redis-{{ redis_version }}" +redis_tar_ball: "http://download.redis.io/releases/redis-{{ redis_version }}.tar.gz" dp_redis_max_memory: 2gb -redis_dir: "{{ analytics_user_home }}" +redis_dir: "{{ analytics_redis_user_home }}" redis_dump_rdb_time_in_seconds: 900 redis_dump_rdb_keys_to_save_state: 100 - - - +redis_version: 6.2.5 +redis_data_dir: "{{ redis_dir }}/redis-{{ redis_version }}/data" diff --git a/ansible/roles/redis/tasks/main.yml b/ansible/roles/redis/tasks/main.yml index ed2821dc1a..39ef3327ef 100755 --- a/ansible/roles/redis/tasks/main.yml +++ b/ansible/roles/redis/tasks/main.yml @@ -1,52 +1,59 @@ --- -# tasks file for redis - name: download redis zipfile - become: yes + become: true become_user: "{{ redis_user }}" - get_url: url={{ redis_tar_ball }} dest={{ redis_dir }}/redis-stable.tar.gz timeout=50 force=no owner={{ redis_user }} + get_url: url={{ redis_tar_ball }} dest={{ redis_dir }}/redis-{{ redis_version }}.tar.gz timeout=50 force=no owner={{ redis_user }} - name: unzip - become: yes + become: true become_user: "{{ redis_user }}" - unarchive: src={{ redis_dir }}/redis-stable.tar.gz dest={{ redis_dir }} copy=no group={{ redis_user }} owner={{ redis_user }} creates={{ redis_home }} + unarchive: src={{ redis_dir }}/redis-{{ redis_version }}.tar.gz dest={{ redis_dir }} copy=no group={{ redis_user }} owner={{ redis_user }} creates={{ redis_home }} + +- name: create redis data dir + file: + path: "{{ redis_dir }}/redis-{{ redis_version }}/data" + state: directory + owner: "{{ redis_user }}" + group: "{{ redis_user }}" + mode: 0755 + become: true - name: Change ownership of redis installation - become: yes + become: true file: path={{ redis_home }} owner={{ redis_user }} group={{ redis_user }} state=directory recurse=yes - name: redis conf - become: yes - template: src=redis.conf.j2 dest={{ redis_dir }}/redis-stable/redis.conf + become: true + template: src=redis.conf.j2 dest={{ redis_dir }}/redis-{{ redis_version }}/redis.conf - name: Install required utils apt: name={{ item }} state=present - become: yes + become: true with_items: - make - gcc - build-essential - - tcl8.5 - name: Go to the folder and run make - become: yes + become: true become_user: "{{ redis_user }}" command: chdir={{ redis_home }} make - name: copy systemd script template - become: yes + become: true template: src: systemd.j2 dest: /etc/systemd/system/redis.service mode: 0755 - name: enable redis systemd service - become: yes + become: true systemd: name: redis enabled: yes - name: start redis - become: yes + become: true systemd: name: redis state: restarted diff --git a/ansible/roles/redis/templates/redis.conf.j2 b/ansible/roles/redis/templates/redis.conf.j2 index f1d0bbdd7f..e7c725c58d 100644 --- a/ansible/roles/redis/templates/redis.conf.j2 +++ b/ansible/roles/redis/templates/redis.conf.j2 @@ -160,7 +160,7 @@ loglevel notice # Specify the log file name. Also the empty string can be used to force # Redis to log on the standard output. Note that if you use standard # output for logging but daemonize, logs will be sent to /dev/null -logfile "{{ redis_dir }}/redis-stable/redis.log" +logfile "{{ redis_data_dir }}/redis.log" # To enable logging to the system logger, just set 'syslog-enabled' to yes, @@ -246,7 +246,7 @@ dbfilename dump.rdb # The Append Only File will also be created inside this directory. # # Note that you must specify a directory here, not a file name. -dir "{{ redis_dir }}/redis-stable/" +dir "{{redis_data_dir}}" ################################# REPLICATION ################################# diff --git a/ansible/roles/redis/templates/systemd.j2 b/ansible/roles/redis/templates/systemd.j2 index 87f7a2dab4..9f9cf59fde 100644 --- a/ansible/roles/redis/templates/systemd.j2 +++ b/ansible/roles/redis/templates/systemd.j2 @@ -5,10 +5,10 @@ After=network.target [Service] User={{ redis_user }} Group={{ redis_user }} -ExecStart={{ redis_dir }}/redis-stable/src/redis-server {{ redis_dir }}/redis-stable/redis.conf -ExecStop={{ redis_dir }}/redis-stable/src/redis-cli shutdown +ExecStart={{ redis_dir }}/redis-{{ redis_version }}/src/redis-server {{ redis_dir }}/redis-{{ redis_version }}/redis.conf +ExecStop={{ redis_dir }}/redis-{{ redis_version }}/src/redis-cli shutdown Restart=always -Type=notify +Type=forking [Install] WantedBy=multi-user.target diff --git a/ansible/roles/samza-jobs/defaults/main.yml b/ansible/roles/samza-jobs/defaults/main.yml deleted file mode 100644 index 8f2ac30540..0000000000 --- a/ansible/roles/samza-jobs/defaults/main.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- -samza_jobs_dir: /home/hduser/samza-jobs/{{ env }} -job_status_file: /home/hduser/samza-jobs/{{ env }}/extract/job_status -yarn_path: /usr/local/hadoop/bin -lpdeploy: no -dpdeploy: no -hadoop_version: 2.7.2 -# Defaults -telemetry_extractor_yarn_container_count: 1 -telemetry_validator_yarn_container_count: 1 -telemetry_de_duplication_yarn_container_count: 1 -telemetry_router_yarn_container_count: 1 -telemetry_reverse_search_yarn_container_count: 1 -telemetry_object_de_normalization_yarn_container_count: 1 -telemetry_es_indexer_yarn_container_count: 1 -telemetry_location_updater_yarn_container_count: 1 -events_router_yarn_container_count: 1 -denormalization_yarn_container_count: 1 -druid_events_validator_yarn_container_count: 1 -redis_updater_yarn_container_count: 1 -druidprocessor_yarn_container_count: 1 -device_profile_updater_yarn_container_count: 1 -assessment_aggregator_yarn_container_count: 1 -telemetry_derived_de_duplication_yarn_container_count: 1 -events_flattener_yarn_container_count: 1 -telemetry_redacter_yarn_container_count: 1 - - -# samza jobs checkpoint replication factor -samza_checkpoint_replication_factor: 1 - -max_iteration_count_for_samza_job: 2 -location_db_redis_key_expiry_seconds: 86400 -cache_unresolved_location_key_expiry_seconds: 3600 -redis_host: "{{ groups['redis'][0] }}" -redis_port: 6379 -metadata_redis_host: "{{ groups['redis'][0] }}" -metadata_redis_port: 6379 -__yarn_port__: 8000 -delayInMilliSeconds: 60000 -retryTimeInMilliSeconds: 10000 -retry_backoff_base_in_seconds: 10 -bypass_reverse_search: true -retry_limit: 4 -retry_limit_enable: true -environment_id: 10000000 -tr_secondary_route_events: "LOG,ERROR" -google_vision_tagging: false -google_api_key: "123" -cassandra_port: 9042 -postgres_password: "{{ postgres.db_password }}" -postgres_db: "{{ postgres.db_name }}" -postgres_user: "{{ postgres.db_username }}" -postgres_host: "{{ postgres.db_url }}" -postgres_port: "{{ postgres.db_port }}" -es_port: 9200 - -middleware_cassandra_keyspace: sunbird -middleware_cassandra_user_table: user -middleware_cassandra_location_table: location -middleware_cassandra_courses_keyspace: sunbird_courses -middleware_cassandra_assessment_aggregator_table: assessment_aggregator -middleware_cassandra_assessment_question_type : question - -telemetry_extractor_consumer_fetch_max_bytes: 5242880 -telemetry_extractor_container_memory_mb: 1024 -telemetry_extractor_messages_fetch_threshold: 3000 -consumer_fetch_max_bytes: 1572864 -producer_max_request_size_bytes: 1572864 -assessment_aggregator_messages_fetch_threshold: 3000 -assessment_aggregator_consumer_fetch_max_bytes: 5242880 - -dedup_include_env_producer_ids: "" - -content_metadata_fields: "name, objectType, contentType, mediaType, language, medium, subject, gradeLevel, mimeType, createdBy, createdFor, framework, board, status, pkgVersion, lastSubmittedOn, lastUpdatedOn, lastPublishedOn, channel" -user_metadata_fields: "usertype, grade, subject, language, state, district, usersignintype,userlogintype" -dialcode_metadata_fields: "identifier, channel, batchcode, publisher, generated_on, published_on, status" - -user_signin_type_default : Anonymous -user_login_type_default : NA - -#Purpose: Content cache update job uses the dialcode api -dialcode_api_host: "{{ dialcode_host }}" -dialcode_api_endpoint: "{{ dialcode_endpoint }}" -dialcode_api_auth_key: "{{ dialcode_api_key }}" -dedup_exclude_eids: "LOG,ERROR" diff --git a/ansible/roles/samza-jobs/tasks/main.yml b/ansible/roles/samza-jobs/tasks/main.yml deleted file mode 100644 index 3e28039576..0000000000 --- a/ansible/roles/samza-jobs/tasks/main.yml +++ /dev/null @@ -1,73 +0,0 @@ - - name: Create Directory for Jobs - file: path={{ item }} owner=hduser group=hadoop recurse=yes state=directory - with_items: - - "{{ samza_jobs_dir }}" - - "{{ samza_jobs_dir }}/extract" - - - name: Get the application id to kill the app - shell: "{{ yarn_path }}/yarn application --list | grep -i {{ item }} | awk '{print $1}'" - with_items: "{{ job_names_to_kill.split(',')|list }}" - register: appid - - - name: Kill the mentioned applications - shell: "{{ yarn_path }}/yarn application -kill {{ item.stdout }}" - with_items: - - "{{ appid['results'] }}" - when: item.stdout | length > 0 - - - name: find the existing file names to remove - find: - paths: "{{ samza_jobs_dir }}" - patterns: "{{ job_names['%s'|format(item)].job_file_name }}*" - recurse: yes - with_items: "{{ job_names_to_kill.split(',') }}" - register: existing_files - - - name: remove the files under "{{ samza_jobs_dir }}" directory - command: rm -rf "{{ item.path | basename }}" - with_items: "{{ existing_files | json_query('results[].files[]') }}" - args: - chdir: "{{ samza_jobs_dir }}" - - - name: remove the files under "{{ samza_jobs_dir }}/extract" directory - command: rm -rf "{{ item.path | basename }}" - with_items: "{{ existing_files | json_query('results[].files[]') }}" - args: - chdir: "{{ samza_jobs_dir }}/extract" - - - name: copy new jobs tar ball - copy: src={{ item }} dest={{ samza_jobs_dir }}/ force=no owner=hduser group=hadoop - with_fileglob: - - ../defaults/jobs/* - register: new_jobs - - - name: Create directory to extract new jobs - file: path="{{ samza_jobs_dir }}/extract/{{ item }}" owner=hduser group=hadoop recurse=yes state=directory - with_items: - - "{{ new_jobs | json_query('results[].invocation.module_args.original_basename') }}" - - - name: extract new jobs - unarchive: - src: "{{ samza_jobs_dir }}/{{ item }}" - dest: "{{ samza_jobs_dir }}/extract/{{ item }}" - remote_src: yes - with_items: - - "{{ new_jobs | json_query('results[].invocation.module_args.original_basename') }}" - - - name: Get all new jobs config - shell: "ls -d -1 {{ samza_jobs_dir }}/extract/{{ item }}/config/*.properties" - register: config_files - with_items: - - "{{ new_jobs | json_query('results[].invocation.module_args.original_basename') }}" - - - name: update environment specific details in new job configs - replace: dest="{{ item[1].stdout }}" regexp="{{ item[0].key }}" replace="{{ item[0].value }}" - with_nested: - - [{key: "__yarn_host__", value: "{{ __yarn_host__ }}"}, {key: "__yarn_port__", value: "{{ __yarn_port__ }}"}, {key: "__env__", value: "{{ env }}" }, {key: "__env_name__", value: "{{ env_name }}" }, {key: "__zookeepers__", value: "{{ zookeepers }}"}, {key: "__kafka_brokers__", value: "{{ kafka_brokers }}"}, {key: "__delayInMilliSeconds__", value: "{{ delayInMilliSeconds }}" }, {key: "__retryTimeInMilliSeconds__", value: "{{ retryTimeInMilliSeconds }}" }, {key: "__bypass_reverse_search__", value: "{{ bypass_reverse_search }}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{ retry_backoff_base_in_seconds }}" }, {key: "__retryLimit__", value: "{{ retry_limit }}" }, {key: "__retryLimitEnable__", value: "{{ retry_limit_enable }}" }, {key: "__google_api_key__", value: "{{ google_api_key }}" }, {key: "__searchServiceEndpoint__", value: "{{ search_service_endpoint }}" }, {key: "__environment_id__", value: "{{ environment_id }}"}, {key: "__google_vision_tagging__", value: "{{ google_vision_tagging }}"}, {key: "__cassandra_host__", value: "{{ cassandra_host }}"},{key: "__cassandra_port__", value: "{{ cassandra_port }}"}, {key: "__content_to_vec_url__", value: "{{ content_to_vec_url }}"},{key: "__max_iteration_count_for_samza_job__", value: "{{ max_iteration_count_for_samza_job }}"},{key: "__device_profile_updater_yarn_container_count__", value: "{{ device_profile_updater_yarn_container_count }}"},{key: "__telemetry_extractor_yarn_container_count__", value: "{{ telemetry_extractor_yarn_container_count }}"},{key: "__telemetry_validator_yarn_container_count__", value: "{{ telemetry_validator_yarn_container_count }}"},{key: "__telemetry_de_duplication_yarn_container_count__", value: "{{ telemetry_de_duplication_yarn_container_count }}"},{key: "__telemetry_router_yarn_container_count__", value: "{{ telemetry_router_yarn_container_count }}"},{key: "__telemetry_reverse_search_yarn_container_count__", value: "{{ telemetry_reverse_search_yarn_container_count }}"},{key: "__telemetry_object_de_normalization_yarn_container_count__", value: "{{ telemetry_object_de_normalization_yarn_container_count }}"}, {key: "__events_router_yarn_container_count__", value: "{{ events_router_yarn_container_count }}"}, {key: "__ingestion_zookeepers__", value: "{{ ingestion_zookeepers }}"}, {key: "__ingestion_kafka_brokers__", value: "{{ ingestion_kafka_brokers }}"}, {key: "__tr_secondary_route_events__", value: "{{ tr_secondary_route_events }}"},{key: "__telemetry_schema_path__", value: "{{ telemetry_schema_path }}"},{key: "__default_channel__", value: "{{ default_channel }}"},{key: "__telemetry_location_updater_yarn_container_count__", value: "{{ telemetry_location_updater_yarn_container_count }}"},{key: "__channelSearchServiceEndpoint__", value: "{{ channelSearchServiceEndpoint }}"},{key: "__locationSearchServiceEndpoint__", value: "{{ locationSearchServiceEndpoint }}"},{key: "__searchServiceAuthorizationToken__", value: "{{ searchServiceAuthorizationToken }}"},{key: "__redis_host__", value: "{{ redis_host }}"}, {key: "__redis_port__", value: "{{ redis_port }}"},{key: "__location_db_redis_key_expiry_seconds__", value: "{{ location_db_redis_key_expiry_seconds }}"}, {key: "__cache_unresolved_location_key_expiry_seconds__", value: "{{ cache_unresolved_location_key_expiry_seconds }}"}, {key: "__middleware_cassandra_host__", value: "{{ core_cassandra_host }}"},{key: "__middleware_cassandra_port__", value: "{{ cassandra_port }}"},{key: "__middleware_cassandra_keyspace__", value: "{{ middleware_cassandra_keyspace }}"}, {key: "__middleware_cassandra_user_table__", value: "{{ middleware_cassandra_user_table }}"}, {key: "__middleware_cassandra_location_table__", value: "{{ middleware_cassandra_location_table }}"}, {key: "__telemetry_extractor_consumer_fetch_max_bytes__", value: "{{ telemetry_extractor_consumer_fetch_max_bytes }}"}, {key: "__telemetry_extractor_container_memory_mb__", value: "{{ telemetry_extractor_container_memory_mb }}"}, {key: "__telemetry_extractor_messages_fetch_threshold__", value: "{{ telemetry_extractor_messages_fetch_threshold }}"}, {key: "__denormalization_yarn_container_count__", value: "{{ denormalization_yarn_container_count }}"}, {key: "__druidprocessor_yarn_container_count__", value: "{{ druidprocessor_yarn_container_count }}"}, {key: "__content_metadata_fields__", value: "{{ content_metadata_fields }}"}, {key: "__user_metadata_fields__", value: "{{ user_metadata_fields }}"}, {key: "__dialcode_metadata_fields__", value: "{{ dialcode_metadata_fields }}"}, {key: "__druid_events_validator_yarn_container_count__", value: "{{ druid_events_validator_yarn_container_count }}"}, {key: "__redis_updater_yarn_container_count__", value: "{{ redis_updater_yarn_container_count }}"}, {key: "__samza_checkpoint_replication_factor__", value: "{{ samza_checkpoint_replication_factor }}"}, {key: "__consumer_fetch_max_bytes__", value: "{{ consumer_fetch_max_bytes }}"}, {key: "__producer_max_request_size_bytes__", value: "{{ producer_max_request_size_bytes }}"}, {key: "__dedup_include_producer_ids__", value: "{{ dedup_include_env_producer_ids }}"}, {key: "__user_sigin_type_default__", value: "{{ user_signin_type_default }}"}, {key: "__user_login_type_default__", value: "{{ user_login_type_default }}"}, {key: "__middleware_cassandra__courses_keyspace__", value: "{{ middleware_cassandra_courses_keyspace }}"}, {key: "__middleware_cassandra_assessment_aggregator_table__", value: "{{ middleware_cassandra_assessment_aggregator_table }}"}, {key: "__middleware_cassandra_assessment_question_type__", value: "{{ middleware_cassandra_assessment_question_type }}"}, {key: "__assessment_aggregator_yarn_container_count__", value: "{{ assessment_aggregator_yarn_container_count }}"}, {key: "__assessment_aggregator_consumer_fetch_max_bytes__", value: "{{ assessment_aggregator_consumer_fetch_max_bytes }}"}, {key: "__postgres_db__", value: "{{ postgres_db }}"}, {key: "__postgres_user__", value: "{{ postgres_user }}"}, {key: "__postgres_password__", value: "{{ postgres_password }}"}, {key: "__postgres_host__", value: "{{ postgres_host }}"}, {key: "__postgres_port__", value: "{{ postgres_port }}"}, {key: "__assessment_aggregator_messages_fetch_threshold__", value: "{{ assessment_aggregator_messages_fetch_threshold }}"},{key: "__telemetry_derived_de_duplication_yarn_container_count__", value: "{{ telemetry_derived_de_duplication_yarn_container_count }}"},{key: "__events_flattener_yarn_container_count__", value: "{{ events_flattener_yarn_container_count}}"},{key: "__metadata_redis_host__", value: "{{ metadata_redis_host }}"}, {key: "__metadata_redis_port__", value: "{{ metadata_redis_port }}"},{key: "__dedup_exclude_eids__", value: "{{ dedup_exclude_eids }}"},{key: "__telemetry_redacter_yarn_container_count__", value: "{{ telemetry_redacter_yarn_container_count }}"},{key: "__dialcode_host__", value: "{{ dialcode_api_host }}"}, {key: "__dialcode_api_endpoint__", value: "{{ dialcode_api_endpoint }}"}, {key: "__dialcode_authorization_key__", value: "{{ dialcode_api_auth_key }}"}] - - "{{ config_files | json_query('results[]') }}" - - - name: Start the jobs - shell: "{{ samza_jobs_dir }}/extract/{{ item.0 }}/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path={{ item.1.stdout }}" - with_together: - - "{{ new_jobs | json_query('results[].invocation.module_args.original_basename') }}" - - "{{ config_files | json_query('results[]') }}" diff --git a/ansible/roles/secor-deploy/defaults/main.yml b/ansible/roles/secor-deploy/defaults/main.yml index 2af95c45d1..05e63e6ad1 100644 --- a/ansible/roles/secor-deploy/defaults/main.yml +++ b/ansible/roles/secor-deploy/defaults/main.yml @@ -8,6 +8,6 @@ secor_username: secor-monitor script_dir: "{{ sbin_path }}" secor: - properties: ['secor.common', 'secor', 'secor.partition'] + properties: ['secor.common', 'secor', 'secor.partition', 'log4j'] artifact_dir: /mount/secor channel: "#monitor-sunbirddev-lpa" \ No newline at end of file diff --git a/ansible/roles/secor-deploy/tasks/main.yml b/ansible/roles/secor-deploy/tasks/main.yml index 8411b28664..978359ee2e 100644 --- a/ansible/roles/secor-deploy/tasks/main.yml +++ b/ansible/roles/secor-deploy/tasks/main.yml @@ -25,4 +25,4 @@ owner: "{{ analytics_user }}" group: "{{ analytics_user }}" with_fileglob: - - "artifacts/target/secor-*-SNAPSHOT-bin.tar.gz" + - "artifacts/target/secor-{{ secor.artifact_ver }}-SNAPSHOT-bin.tar.gz" diff --git a/ansible/roles/secor-deploy/templates/secor-service.j2 b/ansible/roles/secor-deploy/templates/secor-service.j2 index 2fd50cb136..9c38980f14 100644 --- a/ansible/roles/secor-deploy/templates/secor-service.j2 +++ b/ansible/roles/secor-deploy/templates/secor-service.j2 @@ -29,7 +29,7 @@ function start() { fi cd /mount/$SECOR_NAME - nohup /usr/bin/java -Xms256M -Xmx1000M -ea -Duser.timezone=UTC -Dsecor_group=$SECOR_NAME -Dlog4j.configuration=log4j.properties -Dconfig=secor.partition.properties -cp secor-0.25-SNAPSHOT.jar:lib/* com.pinterest.secor.main.ConsumerMain > /mount/secor/logs/$SECOR_NAME-service.log 2>&1 & + nohup /usr/bin/java -Xms1024M -Xmx1024M -ea -Duser.timezone=UTC -Dsecor_group=$SECOR_NAME -Dlog4j.configuration=log4j.properties -Dconfig=secor.partition.properties -cp secor-0.25-SNAPSHOT.jar:lib/* com.pinterest.secor.main.ConsumerMain > /mount/secor/logs/$SECOR_NAME-service.log 2>&1 & pid=$! sleep 1 diff --git a/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml b/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml index 2b23e0a15e..ce23c020ac 100644 --- a/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml +++ b/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml @@ -2,13 +2,21 @@ analytics_user: analytics analytics_group: analytics analytics_user_home: /home/{{analytics_user}} sbin_path: "{{ analytics_user_home }}/sbin" -azure_container_name: "{{secor_azure_container_name}}" -azure_account_key: "{{sunbird_private_storage_account_key}}" -telemetry_ingestion_topic: "{{ env }}.telemetry.ingest" +gs_base_path: "secor-{{ env }}" +s3_base_path: "secor-{{ env }}" +s3_container_name: "{{ cephs3_dp_storage_container | default('') }}" +s3_access_key: "{{ cephs3_storage_key | default('') }}" +s3_secret_key: "{{ cephs3_storage_secret | default('') }}" +s3_endpoint: "{{ cephs3_storage_endpoint | default('') }}" +# s3_endpoint: "https://static.karmyogiprod.nic.in" +s3_region: "{{ cephs3_storage_region | default('us-east-1') }}" +s3_path_style_access: "true" + +telemetry_ingestion_topic: "{{ env }}.telemetry.ingestion" secor: - properties: ['secor.azure', 'secor.common', 'secor', 'secor.partition', 'log4j'] + properties: ['secor.common', 'secor', 'secor.partition', 'log4j'] artifact_dir: /mount/secor secor_service_name: @@ -18,7 +26,7 @@ secor_service_name: timestamp_key: "syncts" fallback_timestamp_key: "@timestamp" topic: "{{ env }}.telemetry.failed" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" max_file_size: "100000000" max_file_age: "14400" @@ -26,10 +34,11 @@ secor_service_name: partition_prefix_enabled: "false" partition_prefix_key: "" partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" service_description: "Failed Telemetry Backup" service_monitor_schedule_hr: "23" service_monitor_schedule_min: "50" + message_channel_identifier: "" message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" raw-telemetry-backup: consumer_group: "{{ env }}.telemetry.raw.backup" @@ -37,7 +46,7 @@ secor_service_name: timestamp_key: "syncts" fallback_timestamp_key: "@timestamp" topic: "{{ env }}.telemetry.raw" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" max_file_size: "100000000" max_file_age: "14400" @@ -45,10 +54,11 @@ secor_service_name: partition_prefix_enabled: "false" partition_prefix_key: "" partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" service_description: "Raw Telemetry Backup" service_monitor_schedule_hr: "23" service_monitor_schedule_min: "50" + message_channel_identifier: "" message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" unique-telemetry-backup: consumer_group: "{{ env }}.telemetry.unique.backup" @@ -56,11 +66,11 @@ secor_service_name: timestamp_key: "syncts" fallback_timestamp_key: "@timestamp" topic: "{{ env }}.telemetry.unique" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" max_file_size: "100000000" max_file_age: "14400" - ostrich_port: "9993" + ostrich_port: "9969" partition_prefix_enabled: "false" partition_prefix_key: "" partition_prefix_mapping: "{}" @@ -68,25 +78,27 @@ secor_service_name: service_description: "Unique Telemetry Backup" service_monitor_schedule_hr: "23" service_monitor_schedule_min: "50" + message_channel_identifier: "" message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" - denorm-events-backup: - consumer_group: "{{ env }}.telemetry.denorm.backup" - base_path: "telemetry-denormalized" + derived-denorm-events-backup: + consumer_group: "{{ env }}.summary.backup" + base_path: "telemetry-denormalized/summary" timestamp_key: "syncts" fallback_timestamp_key: "@timestamp" - topic: "{{ env }}.telemetry.denorm" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" + topic: "{{ env }}.druid.events.summary" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" max_file_size: "100000000" max_file_age: "14400" - ostrich_port: "9990" - partition_prefix_enabled: "true" - partition_prefix_key: "eid" - partition_prefix_mapping: "{\"ME_SESSION_SUMMARY\":\"summary\",\"ME_WORKFLOW_SUMMARY\":\"summary\",\"ME_DEIVCE_SUMMARY\":\"summary\",\"ME_ITEM_SUMMARY\":\"summary\",\"ME_WORKFLOW_USAGE_SUMMARY\":\"summary\",\"ME_ITEM_USAGE_SUMMARY\":\"summary\",\"ME_USAGE_SUMMARY\":\"summary\",\"ME_DIALCODE_USAGE_SUMMARY\":\"summary\",\"DEFAULT\":\"raw\"}" + ostrich_port: "9992" + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" - service_description: "Denormalized Events Backup" + service_description: "Denormalized Summary Events Backup" service_monitor_schedule_hr: "23" service_monitor_schedule_min: "50" + message_channel_identifier: "" message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" extractor-failed-backup: consumer_group: "{{ env }}.extractor.failed.backup" @@ -94,18 +106,19 @@ secor_service_name: timestamp_key: "syncts" fallback_timestamp_key: "@timestamp" topic: "{{ env }}.telemetry.extractor.failed" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" max_file_size: "100000000" - max_file_age: "21600" + max_file_age: "14400" ostrich_port: "9989" partition_prefix_enabled: "false" partition_prefix_key: "" partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" service_description: "Extractor failed events backup" service_monitor_schedule_hr: "23" service_monitor_schedule_min: "50" + message_channel_identifier: "" message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" assess-raw-events-backup: consumer_group: "{{ env }}.telemetry.assess.raw" @@ -113,113 +126,19 @@ secor_service_name: timestamp_key: "syncts" fallback_timestamp_key: "@timestamp" topic: "{{ env }}.telemetry.assess.raw" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" max_file_size: "100000000" - max_file_age: "21600" + max_file_age: "14400" ostrich_port: "9986" partition_prefix_enabled: "false" partition_prefix_key: "" partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" service_description: "Telemetry raw assess events backup" service_monitor_schedule_hr: "23" service_monitor_schedule_min: "50" - message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" - telemetry-ingest-backup: - consumer_group: "{{ env }}.telemetry.ingest.backup" - base_path: "ingest-telemetry" - timestamp_key: "syncts" - fallback_timestamp_key: "@timestamp" - topic: "{{ env }}.telemetry.ingest" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" - zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" - max_file_size: "100000000" - max_file_age: "14400" - ostrich_port: "9985" - partition_prefix_enabled: "false" - partition_prefix_key: "" - partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" - service_description: "Ingestion cluster Telemetry Backup" - service_monitor_schedule_hr: "23" - service_monitor_schedule_min: "54" - message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" - telemetry-duplicate-backup: - consumer_group: "{{ env }}.telemetry.duplicate.backup" - base_path: "telemetry-duplicate" - timestamp_key: "syncts" - fallback_timestamp_key: "@timestamp" - topic: "{{ env }}.telemetry.duplicate" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" - zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" - max_file_size: "100000000" - max_file_age: "14400" - ostrich_port: "9984" - partition_prefix_enabled: "false" - partition_prefix_key: "" - partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" - service_description: "Telemetry Duplicate Backup" - service_monitor_schedule_hr: "23" - service_monitor_schedule_min: "50" - message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" - extractor-duplicate-backup: - consumer_group: "{{ env }}.extractor.duplicate.backup" - base_path: "extractor-duplicate" - timestamp_key: "syncts" - fallback_timestamp_key: "@timestamp" - topic: "{{ env }}.telemetry.extractor.duplicate" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" - zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" - max_file_size: "100000000" - max_file_age: "14400" - ostrich_port: "9983" - partition_prefix_enabled: "false" - partition_prefix_key: "" - partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" - service_description: "Telemetry Duplicate Backup" - service_monitor_schedule_hr: "23" - service_monitor_schedule_min: "50" - message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" - derived-telemetry-backup: - consumer_group: "{{ env }}.telemetry.derived.unique.backup" - base_path: "unique/summary" - timestamp_key: "syncts" - fallback_timestamp_key: "@timestamp" - topic: "{{ env }}.telemetry.derived.unique" - kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" - zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" - max_file_size: "100000000" - max_file_age: "600" - ostrich_port: "9982" - partition_prefix_enabled: "true" - partition_prefix_key: "eid" - partition_prefix_mapping: "{\"ME_SESSION_SUMMARY\":\"session_summary\",\"ME_WORKFLOW_SUMMARY\":\"workflow_summary\",\"ME_DEIVCE_SUMMARY\":\"device_summary\",\"ME_ITEM_SUMMARY\":\"item_summary\",\"ME_WORKFLOW_USAGE_SUMMARY\":\"workflow_usage_summary\",\"ME_ITEM_USAGE_SUMMARY\":\"item_usage_summary\",\"ME_USAGE_SUMMARY\":\"usage_summary\",\"ME_DIALCODE_USAGE_SUMMARY\":\"dialcode_usage_summary\",\"DEFAULT\":\"me\"}" - output_file_pattern: "{partition}-{currentTimestamp}.json" - service_description: "Derived Telemetry Backup" - service_monitor_schedule_hr: "23" - service_monitor_schedule_min: "52" - message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" - device-profile-backup: - consumer_group: "{{ env }}.events.device.profile.backup" - base_path: "device-profile-events" - timestamp_key: "updated_date" - fallback_timestamp_key: "updated_date" - topic: "{{ env }}.events.deviceprofile" - kafka_broker_host: "{{ ingestion_kafka_brokers }}" - zookeeper_quorum: "{{ ingestion_zookeepers }}" - max_file_size: "100000000" - max_file_age: "14400" - ostrich_port: "9981" - partition_prefix_enabled: "false" - partition_prefix_key: "" - partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" - service_description: "Device Profile Backup" - service_monitor_schedule_hr: "23" - service_monitor_schedule_min: "50" + message_channel_identifier: "" message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" learning-events-backup: consumer_group: "{{ env }}.learning.graph.events.backup" @@ -227,76 +146,57 @@ secor_service_name: timestamp_key: "createdOn" fallback_timestamp_key: "@timestamp" topic: "{{ env }}.learning.graph.events" - kafka_broker_host: "{{ ingestion_kafka_brokers }}" - zookeeper_quorum: "{{ ingestion_zookeepers }}" + kafka_broker_host: "{{groups['ingestion-cluster-kafka']|join(':9092,')}}:9092" + zookeeper_quorum: "{{groups['ingestion-cluster-zookeepers']|join(':2181,')}}:2181" max_file_size: "100000000" max_file_age: "14400" ostrich_port: "9980" partition_prefix_enabled: "false" partition_prefix_key: "" partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" service_description: "Learning Events Backup" service_monitor_schedule_hr: "23" service_monitor_schedule_min: "56" + message_channel_identifier: "" message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" - learning-failed-backup: - consumer_group: "{{ env }}.failed.learning.events.backup" - base_path: "learning-failed-events" + issue-certificate-events-backup: + consumer_group: "{{ env }}.issue.certificate.request.backup" + base_path: "issue-certificate-events" timestamp_key: "ets" fallback_timestamp_key: "@timestamp" - azure_account_name: "{{ secor.azure.account_name }}" - container_name: "{{ secor.azure.container_name }}" - azure_account_key: "{{ secor.azure.account_key }}" - topic: "{{ env }}.learning.events.failed" - kafka_broker_host: "{{ ingestion_kafka_brokers }}" - zookeeper_quorum: "{{ ingestion_zookeepers }}" - max_file_size: "100000000" - max_file_age: "3600" - ostrich_port: "9979" - partition_prefix_enabled: "true" - partition_prefix_key: "jobName" - partition_prefix_mapping: "{\"publish-pipeline\":\"publish_pipeline\",\"composite-search-indexer\":\"cs_index\",\"DEFAULT\":\"failed_events\"}" - output_file_pattern: "{partition}-{currentTimestamp}.json" - service_description: "Learning Failed Backup" - service_monitor_schedule_hr: "23" - service_monitor_schedule_min: "52" - message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" - assess-events-backup: - consumer_group: "{{ env }}.telemetry.assess.events.backup" - base_path: "telemetry-batch-assess" - timestamp_key: "assessmentTs" - fallback_timestamp_key: "assessmentTs" - topic: "{{ env }}.telemetry.assess" - kafka_broker_host: "{{ ingestion_kafka_brokers }}" - zookeeper_quorum: "{{ ingestion_zookeepers }}" + topic: "{{ env }}.issue.certificate.request" + kafka_broker_host: "{{groups['ingestion-cluster-kafka']|join(':9092,')}}:9092" + zookeeper_quorum: "{{groups['ingestion-cluster-zookeepers']|join(':2181,')}}:2181" max_file_size: "100000000" - max_file_age: "21600" - ostrich_port: "9978" + max_file_age: "14400" + ostrich_port: "9976" partition_prefix_enabled: "false" partition_prefix_key: "" partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" - service_description: "Telemetry Batch assess events backup" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" + service_description: "Issue Certificate Events Backup" service_monitor_schedule_hr: "23" - service_monitor_schedule_min: "50" + service_monitor_schedule_min: "55" + message_channel_identifier: "" message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" - ingestion-cluster-telemetry-backup: - consumer_group: "{{ env }}.telemetry.ingestion.events.backup" - base_path: "ingestion-telemetry" - timestamp_key: "syncts" - fallback_timestamp_key: "@timestamp" - topic: "{{ telemetry_ingestion_topic }}" - kafka_broker_host: "{{ ingestion_kafka_brokers }}" - zookeeper_quorum: "{{ ingestion_zookeepers }}" + pipeline-metrics-backup: + consumer_group: "{{ env }}.pipeline_metrics" + base_path: "data-pipeline-metrics" + timestamp_key: "metricts" + fallback_timestamp_key: "metricts" + topic: "{{ env }}.pipeline_metrics" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" max_file_size: "100000000" max_file_age: "14400" - ostrich_port: "9977" + ostrich_port: "9991" partition_prefix_enabled: "false" partition_prefix_key: "" partition_prefix_mapping: "{}" - output_file_pattern: "{partition}-{currentTimestamp}.json" - service_description: "Ingestion cluster Telemetry Backup" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" + service_description: "Pipeline metrics backup" service_monitor_schedule_hr: "23" - service_monitor_schedule_min: "54" - message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + service_monitor_schedule_min: "50" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" \ No newline at end of file diff --git a/ansible/roles/secor-telemetry-backup-deploy/tasks/main.yml b/ansible/roles/secor-telemetry-backup-deploy/tasks/main.yml index cc8c316a13..3ce8bc86c6 100644 --- a/ansible/roles/secor-telemetry-backup-deploy/tasks/main.yml +++ b/ansible/roles/secor-telemetry-backup-deploy/tasks/main.yml @@ -25,7 +25,7 @@ - name: Unarchive secor libraries unarchive: - src: "{{ analytics_user_home }}/secor-0.25-SNAPSHOT-bin.tar.gz" + src: "{{ analytics_user_home }}/secor-{{ secor.artifact_ver }}-SNAPSHOT-bin.tar.gz" dest: "/mount/{{ item.key }}" copy: no owner: "{{ analytics_user }}" @@ -38,6 +38,13 @@ - "{{ secor_service_name }}" - "{{ secor.properties }}" +- name: Write GCP service account file + copy: + content: "{{ gcp_artifcats_upload_service_account }}" + dest: /mount/{{ item }}/gcp_service_account.json + mode: "0640" + with_items: "{{ secor_service_name }}" + - name: Clean cron jobs command: crontab -r ignore_errors: yes diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/log4j.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/log4j.j2 index d7dfd8870e..6076711c1e 100644 --- a/ansible/roles/secor-telemetry-backup-deploy/templates/log4j.j2 +++ b/ansible/roles/secor-telemetry-backup-deploy/templates/log4j.j2 @@ -11,7 +11,7 @@ log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] (%C:%L) %-5p %m log4j.appender.ROLLINGFILE=org.apache.log4j.DailyRollingFileAppender log4j.appender.ROLLINGFILE.Threshold=INFO -log4j.appender.ROLLINGFILE.File=/mount/{{ secor_service_name }}/logs/secor.log +log4j.appender.ROLLINGFILE.File=/mount/{{ item.0 }}/logs/secor.log log4j.appender.ROLLINGFILE.DatePattern='.'yyyy-MM-dd log4j.appender.ROLLINGFILE.MaxBackupIndex=30 log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.aws.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.aws.j2 new file mode 100644 index 0000000000..c0fe5b4c63 --- /dev/null +++ b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.aws.j2 @@ -0,0 +1,101 @@ + +# Choose what to fill according to the service you are using +# in the choice option you can fill S3, GS, Swift or Azure +cloud.service=S3 + +# AWS authentication credentials. +# Leave empty if using IAM role-based authentication with s3a filesystem. +aws.access.key={{ s3_access_key }} +aws.secret.key={{ s3_secret_key }} +aws.role= + +# Optional Proxy Setting. Set to true to enable proxy +# Only applicable to S3UploadManager +aws.proxy.isEnabled=false +aws.proxy.http.host= +aws.proxy.http.port= + +# AWS region or endpoint. region should be a known region name (eg. +# us-east-1). endpoint should be a known S3 endpoint url. If neither +# are specified, then the default region (us-east-1) is used. If both +# are specified then endpoint is used. +# +# Only apply if the the S3UploadManager is used - see +# secor.upload.manager.class. +# +# http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region +aws.region={{ s3_region }} +aws.endpoint={{ s3_endpoint }} + +# Toggle the AWS S3 client between virtual host style access and path style +# access. See http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html +aws.client.pathstyleaccess={{ s3_path_style_access }} + +########################### +# START AWS S3 ENCRYPTION # +########################### + +# AWS specify type of server-side encryption, if any +# set to S3 to enable S3-managed encryption +# set to KMS to enable AWS KMS-managed encryption (see aws.sse.kms.key) +# set to customer to enable customer-managed encryption (see aws.sse.customer.key) +# set empty to disable encryption +aws.sse.type= + +# Key to use for S3 server-side encryption, base64-encoded +# Note: requires aws.sse.type to be set to customer to be used +aws.sse.customer.key= + +# KMS Key to use for S3 server-side encryption, base64-encoded +# Leave empty to use default generated key +# Note: requires aws.sse.type to be set to KMS to be used +aws.sse.kms.key= + +######################### +# END AWS S3 ENCRYPTION # +######################### + +# Hadoop filesystem to use. Choices are s3n or s3a. +# See https://wiki.apache.org/hadoop/AmazonS3 for details. +secor.s3.filesystem=s3n + +# Class that will manage uploads. Default is to use the hadoop +# interface to S3. +secor.upload.manager.class=com.pinterest.secor.uploader.S3UploadManager + + +# Set below property to true if you want to have the md5hash appended to your s3 path. +# This helps for better partitioning of the data on s3. Which gives better performance while reading and writing on s3 +secor.s3.prefix.md5hash=false + +# After the given date, secor will upload files to the supplied s3 alternative path +secor.s3.alter.path.date= + +# An alternative S3 path for secor to upload files to +secor.s3.alternative.path= + + +############### +# Using S3 # +############### + +# Name of the s3 bucket where log files are stored. +secor.s3.bucket={{ s3_container_name }} + +# S3 path where sequence files are stored. +secor.s3.path={{ s3_base_path }}/{{ secor_service_name[item[0]].base_path }} + +# Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz. +# Available placeholders are +# topic - The topic name the data is being fetched +# partition - The partition name +# generation - Generation +# kafkaPartition - The kafka partition +# fmOffset - First Message offset in the file. +# randomHex - A 4 character random hex to append to the file name +# currentTimestamp - Time of upload in epoch format +# currentTime - Time of upload in HH-mm format +# currentDate - Time of upload in YYYYMMDD format +# folder - Folder to use based on message id map lookup +secor.s3.output_file_pattern={{ secor_service_name[item[0]].output_file_pattern }} + diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.azure.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.azure.j2 index f44174563c..f62a976abd 100644 --- a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.azure.j2 +++ b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.azure.j2 @@ -1,5 +1,13 @@ include=secor.properties + +# Choose what to fill according to the service you are using +# in the choice option you can fill S3, GS, Swift or Azure +cloud.service=Azure + +# Azure path where sequence files are stored. +secor.azure.path={{ secor_service_name[item[0]].base_path }} + # Configure upload manager class to use Azure blob storage upload manager secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager @@ -12,16 +20,18 @@ secor.azure.endpoints.protocol=https # Microsoft Azure authentication credentials. # https://azure.microsoft.com/en-us/documentation/articles/storage-create-storage-account -secor.azure.account.name={{sunbird_private_storage_account_name}} -secor.azure.account.key={{sunbird_private_storage_account_key}} +secor.azure.account.name= +secor.azure.account.key= # Microsoft Azure blob storage container name. Container is a grouping of a set # of blobs. https://msdn.microsoft.com/en-us/library/dd135715.aspx -secor.azure.container.name={{ azure_container_name }} +secor.azure.container.name=secor # Microsoft Azure blob storage path where files are stored within the container. +# Azure path where sequence files are stored. secor.azure.path={{ secor_service_name[item[0]].base_path }} + ################ # END MUST SET # ################ diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.common.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.common.j2 index c0f76c6d1f..5e7852e907 100644 --- a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.common.j2 +++ b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.common.j2 @@ -21,71 +21,10 @@ secor.kafka.topic_filter={{ secor_service_name[item[0]].topic }} secor.kafka.topic_blacklist= -# Choose what to fill according to the service you are using -# in the choice option you can fill S3, GS, Swift or Azure -cloud.service=Azure - -# AWS authentication credentials. -# Leave empty if using IAM role-based authentication with s3a filesystem. -aws.access.key= -aws.secret.key= -aws.role= - -# Optional Proxy Setting. Set to true to enable proxy -# Only applicable to S3UploadManager -aws.proxy.isEnabled=false -aws.proxy.http.host= -aws.proxy.http.port= - ################ # END MUST SET # ################ - -# AWS region or endpoint. region should be a known region name (eg. -# us-east-1). endpoint should be a known S3 endpoint url. If neither -# are specified, then the default region (us-east-1) is used. If both -# are specified then endpoint is used. -# -# Only apply if the the S3UploadManager is used - see -# secor.upload.manager.class. -# -# http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region -aws.region= -aws.endpoint= - -# Toggle the AWS S3 client between virtual host style access and path style -# access. See http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html -aws.client.pathstyleaccess=false - -########################### -# START AWS S3 ENCRYPTION # -########################### - -# AWS specify type of server-side encryption, if any -# set to S3 to enable S3-managed encryption -# set to KMS to enable AWS KMS-managed encryption (see aws.sse.kms.key) -# set to customer to enable customer-managed encryption (see aws.sse.customer.key) -# set empty to disable encryption -aws.sse.type= - -# Key to use for S3 server-side encryption, base64-encoded -# Note: requires aws.sse.type to be set to customer to be used -aws.sse.customer.key= - -# KMS Key to use for S3 server-side encryption, base64-encoded -# Leave empty to use default generated key -# Note: requires aws.sse.type to be set to KMS to be used -aws.sse.kms.key= - -######################### -# END AWS S3 ENCRYPTION # -######################### - -# Hadoop filesystem to use. Choices are s3n or s3a. -# See https://wiki.apache.org/hadoop/AmazonS3 for details. -secor.s3.filesystem=s3n - # Swift config, MUST configure if cloud.service=Swift # Swift Login Details: @@ -102,26 +41,6 @@ swift.password= # only needed if "swift.use.get.auth" = true swift.api.key= -# GS config, MUST configure if gcloud.service=GS - -# Name of the Google cloud storage bucket where log files are stored. -secor.gs.bucket=secor_gs - -# Google cloud storage path where files are stored within the bucket. -secor.gs.path=data - -# Use direct uploads -# WARNING: disables resumable uploads, files are uploaded in a single request -# This may help prevent IOException: insufficient data written, -# see https://github.com/pinterest/secor/issues/177 -# https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload -secor.gs.upload.direct=false - -# Application credentials configuration file -# https://developers.google.com/identity/protocols/application-default-credentials -# It can be empty when secor running in Google Cloud VMs with proper scopes -secor.gs.credentials.path= - # Zookeeper config. zookeeper.session.timeout.ms=3000 zookeeper.sync.time.ms=200 @@ -165,7 +84,7 @@ kafka.fetch.min.bytes= kafka.fetch.wait.max.ms= # Port of the broker serving topic partition metadata. -kafka.seed.broker.port=9092 +#kafka.seed.broker.port=9092 # Zookeeper path at which kafka is registered. In Zookeeper parlance, this is referred # to as the chroot. @@ -355,10 +274,6 @@ secor.file.writer.Delimiter=\n # Default is 0.1 MB secor.max.message.size.bytes=100000 -# Class that will manage uploads. Default is to use the hadoop -# interface to S3. -secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager - #Set below property to your timezone, and the events will be parsed and converted to the timezone specified secor.message.timezone=UTC @@ -368,16 +283,6 @@ secor.parser.timezone=Asia/Kolkata # Transformer class that transform message accordingly. secor.message.transformer.class=com.pinterest.secor.transformer.IdentityMessageTransformer -# Set below property to true if you want to have the md5hash appended to your s3 path. -# This helps for better partitioning of the data on s3. Which gives better performance while reading and writing on s3 -secor.s3.prefix.md5hash=false - -# After the given date, secor will upload files to the supplied s3 alternative path -secor.s3.alter.path.date= - -# An alternative S3 path for secor to upload files to -secor.s3.alternative.path= - # If enabled, add calls will be made to qubole, otherwise, skip qubole call for finalization secor.enable.qubole=true @@ -420,3 +325,76 @@ parquet.validation=false secor.orc.message.schema.*=struct\,f:array\,g:int> # Below config used for defining ORC schema provider class name. User can use the custom implementation for orc schema provider secor.orc.schema.provider=com.pinterest.secor.util.orc.schema.DefaultORCSchemaProvider + +# Changes related to 0.29 upgrade + +# If true, runs a final upload when the JVM is shut down (eg, from SIGTERM). +# You may want to ensure that whatever system you use to run secor sets an +# appropriate grace period to allow a full upload before a forced termination. +secor.upload.on.shutdown=false + +# If true, uploads lastSeen offset onto ZK, see https://github.com/pinterest/secor/issues/600 for details +secor.upload.secor.upload.last.seen.offset=false + +# If true, uploads are entirely deterministic, which can avoid some race conditions +# which can lead to messages being backed up multiple times. This is incompatible with +# secor.upload.on.shutdown=true, and ignores the values of secor.max.file.size.bytes, +# secor.max.file.age.seconds, and secor.upload.minute_mark. +# +# In deterministic mode, you must set one or both of secor.max.file.timestamp.range.millis and +# secor.max.input.payload.size.bytes. These determine when to upload, and are ignored outside +# of deterministic mode. +secor.upload.deterministic=false + +# Classname for the message iterator you want to use. The message iterator determines what kind of consumer +# secor will use to communicate with kafka. com.pinterest.secor.reader.LegacyKafkaMessageIterator uses +# the old kafka consumer written scala. Its not recommended to use the legacy iterator with kafka version >= 1.0 since it +# does not support the new broker protocols. You may face significant performance degradation on your brokers if you use it +kafka.message.iterator.className=com.pinterest.secor.reader.SecorKafkaMessageIterator + +# Classname for the kafka client used by utility classes like PartitionFinalizer and Progress monitor. +# Legacy client is kept for compatibility purposes and is deprecated +kafka.client.className=com.pinterest.secor.common.SecorKafkaClient + + +######################################## +# Optional: Kafka New Consumer Configs # +######################################## + +# Same as old configuration. Except accepted values are earliest and latest instead of smallest +# and largest +kafka.new.consumer.auto.offset.reset=earliest + +# Comma-separated list of topics to consume. Please note that this is not a regular expression. +# If that's what you want, you can use "secor.kafka.topic_filter" instead. +kafka.new.consumer.topic.list={{ secor_service_name[item[0]].topic }} + +kafka.new.consumer.poll.timeout.seconds=10 +kafka.new.consumer.request.timeout.ms=10000 +kafka.new.consumer.ssl.key.password= +kafka.new.consumer.ssl.keystore.location= +kafka.new.consumer.ssl.keystore.password= +kafka.new.consumer.ssl.truststore.location= +kafka.new.consumer.ssl.truststore.password= +kafka.new.consumer.isolation.level= +kafka.new.consumer.max.poll.interval.ms= +kafka.new.consumer.max.poll.records= +kafka.new.consumer.sasl.client.callback.handler.class= +kafka.new.consumer.sasl.jaas.config= +kafka.new.consumer.sasl.kerberos.service.name= +kafka.new.consumer.sasl.login.callback.handler.class= +kafka.new.consumer.sasl.login.class= +kafka.new.consumer.sasl.mechanism= +kafka.new.consumer.security.protocol= +kafka.new.consumer.ssl.enabled.protocols= +kafka.new.consumer.ssl.keystore.type= +kafka.new.consumer.ssl.protocol= +kafka.new.consumer.ssl.provider= +kafka.new.consumer.ssl.truststore.type= +kafka.new.consumer.partition.assignment.strategy.class= + +######################################## +# End: Kafka New Consumer Configs # +######################################## + +kafka.fetch.max.bytes= \ No newline at end of file diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.j2 index eb243d293a..508b6a1757 100644 --- a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.j2 +++ b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.j2 @@ -4,14 +4,6 @@ include=secor.common.properties # MUST SET # ############ -# Fill the section which fits your needs -############### -# Using S3 # -############### - -# Name of the s3 bucket where log files are stored. -secor.s3.bucket= - ############### # Using Swift # ############### @@ -29,7 +21,7 @@ secor.swift.container=logsContainer ################ kafka.seed.broker.host={{ secor_service_name[item[0]].kafka_broker_host }} -kafka.seed.broker.port= +kafka.seed.broker.port=9092 zookeeper.quorum={{ secor_service_name[item[0]].zookeeper_quorum }} diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.partition.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.partition.j2 index a9f2247aa5..2dc16ef6c0 100644 --- a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.partition.j2 +++ b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.partition.j2 @@ -14,7 +14,53 @@ # limitations under the License. include=secor.properties -include=secor.azure.properties + + +# Choose what to fill according to the service you are using +# in the choice option you can fill S3, GS, Swift or Azure +cloud.service={{ secor_cloud_service }} + + +# Class that will manage uploads. Default is to use the hadoop +# interface to S3. +secor.upload.manager.class={{ cloud_storage_uploadmanager }} + + +# Name of the Google cloud storage bucket where log files are stored. +secor.gs.bucket={{ cloud_storage_bucket_secor }} + +# Google cloud storage path where files are stored within the bucket. +secor.gs.path={{ gs_base_path }}/{{ secor_service_name[item[0]].base_path }} + +# Use direct uploads +# WARNING: disables resumable uploads, files are uploaded in a single request +# This may help prevent IOException: insufficient data written, +# see https://github.com/pinterest/secor/issues/177 +# https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload +secor.gs.upload.direct=true + +# Application credentials configuration file +# https://developers.google.com/identity/protocols/application-default-credentials +# It can be empty when secor running in Google Cloud VMs with proper scopes +secor.gs.credentials.path=/mount/{{ item.0 }}/gcp_service_account.json + + +# Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz. +# Available placeholders are +# topic - The topic name the data is being fetched +# partition - The partition name +# generation - Generation +# kafkaPartition - The kafka partition +# fmOffset - First Message offset in the file. +# randomHex - A 4 character random hex to append to the file name +# currentTimestamp - Time of upload in epoch format +# currentTime - Time of upload in HH-mm format +# currentDate - Time of upload in YYYYMMDD format +# folder - Folder to use based on message id map lookup +secor.s3.output_file_pattern={{ secor_service_name[item[0]].output_file_pattern }} + + + # Name of the Kafka consumer group. secor.kafka.group={{ secor_service_name[item[0]].consumer_group }} @@ -22,15 +68,9 @@ secor.kafka.group={{ secor_service_name[item[0]].consumer_group }} # Parser class that extracts s3 partitions from consumed messages. secor.message.parser.class={{ secor_service_name[item[0]].message_parser }} -# S3 path where sequence files are stored. -secor.s3.path= - # Swift path where sequence files are stored. secor.swift.path=secor_dev/partition -# Azure path where sequence files are stored. -secor.azure.path={{ secor_service_name[item[0]].base_path }} - # Local path where sequence files are stored before they are uploaded to s3. secor.local.path=/mount/{{ item.0 }}/message_logs/partition @@ -50,16 +90,4 @@ secor.partition.prefix.mapping={{ secor_service_name[item[0]].partition_prefix_m secor.max.file.age.policy=oldest -# Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz. -# Available placeholders are -# topic - The topic name the data is being fetched -# partition - The partition name -# generation - Generation -# kafkaPartition - The kafka partition -# fmOffset - First Message offset in the file. -# randomHex - A 4 character random hex to append to the file name -# currentTimestamp - Time of upload in epoch format -# currentTime - Time of upload in HH-mm format -# currentDate - Time of upload in YYYYMMDD format -# folder - Folder to use based on message id map lookup -secor.s3.output_file_pattern={{ secor_service_name[item[0]].output_file_pattern }} +secor.partition.message.channel.identifier={{ secor_service_name[item[0]].message_channel_identifier }} diff --git a/ansible/roles/setup-kafka/defaults/main.yml b/ansible/roles/setup-kafka/defaults/main.yml index 08e13e4038..f8c2c7068d 100644 --- a/ansible/roles/setup-kafka/defaults/main.yml +++ b/ansible/roles/setup-kafka/defaults/main.yml @@ -3,271 +3,317 @@ kafka_partition_override_size: 5242880 dp_downstream_kafka_max_message_bytes: 1572864 ingestion_kafka_topics: - - name: telemetry.ingest + - name: telemetry.ingestion + num_of_partitions: 16 + replication_factor: 2 + - name: events.deviceprofile num_of_partitions: 4 replication_factor: 1 ingestion_kafka_overriden_topics: - - name: telemetry.ingest - retention_time: 604800000 - replication_factor: 1 + - name: telemetry.ingestion + retention_time: 259200000 + replication_factor: 2 max_message_bytes: 5242880 + - name: events.deviceprofile + retention_time: 172800000 + replication_factor: 1 processing_kafka_overriden_topics: - - name: telemetry.raw + - name: dashboards.course retention_time: 172800000 - replication_factor: 1 - - name: telemetry.valid - retention_time: 21600000 - replication_factor: 1 - - name: telemetry.unique + replication_factor: 2 + - name: dashboards.role.count retention_time: 172800000 - replication_factor: 1 - - name: telemetry.duplicate - retention_time: 21600000 - replication_factor: 1 - - name: telemetry.sink - retention_time: 21600000 - replication_factor: 1 - - name: telemetry.with_location - retention_time: 86400000 - replication_factor: 1 - - name: telemetry.audit - retention_time: 86400000 - replication_factor: 1 - - name: telemetry.denorm - retention_time: 86400000 - replication_factor: 1 - - name: telemetry.denorm.valid - retention_time: 86400000 - replication_factor: 1 - - name: telemetry.de_normalized - retention_time: 21600000 - replication_factor: 1 - - name: telemetry.log - retention_time: 21600000 - replication_factor: 1 + replication_factor: 2 + - name: dashboards.org.role.count + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.user.course.program.progress + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.competency.frac + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.competency.course + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.competency.expected + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.competency.declared + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.competency.gap + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.user.org + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.org + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.assessment + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.user.assessment + retention_time: 172800000 + replication_factor: 2 + - name: dashboards.acbp.enrolment + retention_time: 172800000 + replication_factor: 2 - name: analytics.job_queue - retention_time: 28800000 - replication_factor: 1 - - name: learning.graph.events - retention_time: 86400000 - replication_factor: 1 - - name: pipeline_metrics - retention_time: 86400000 - replication_factor: 1 - - name: prom.monitoring.metrics - retention_time: 604800000 - replication_factor: 1 - - name: metrics - retention_time: 604800000 - replication_factor: 1 - - name: learning.graph.events.fail - retention_time: 1296000000 - replication_factor: 1 + retention_time: 172800000 + replication_factor: 2 - name: analytics_metrics - retention_time: 21600000 - replication_factor: 1 - - name: learning.job.request - retention_time: 86400000 - replication_factor: 1 - - name: learning.job.request.fail - retention_time: 1296000000 - replication_factor: 1 - - name: learning.republish.job.request - retention_time: 1296000000 - replication_factor: 1 - - name: telemetry.derived - retention_time: 21600000 - replication_factor: 1 - - name: events.telemetry - retention_time: 86400000 - replication_factor: 1 - - name: events.summary - retention_time: 86400000 + retention_time: 172800000 + replication_factor: 2 + - name: druid.events.error + retention_time: 172800000 + replication_factor: 2 + - name: druid.events.log + retention_time: 172800000 + replication_factor: 2 + - name: druid.events.summary + retention_time: 172800000 + replication_factor: 2 + - name: druid.events.telemetry + retention_time: 172800000 + replication_factor: 2 + - name: events.deviceprofile + retention_time: 172800000 replication_factor: 1 - - name: events.log - retention_time: 86400000 - - name: events.error - retention_time: 86400000 + - name: prom.monitoring.metrics + retention_time: 172800000 + replication_factor: 2 - name: telemetry.assess - retention_time: 604800000 - replication_factor: 1 - - name: telemetry.assess.redact - retention_time: 86400000 - replication_factor: 1 + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.assess.failed + retention_time: 172800000 + replication_factor: 2 - name: telemetry.assess.raw - retention_time: 86400000 - replication_factor: 1 + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.audit + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.denorm + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.derived + retention_time: 172800000 + replication_factor: 2 - name: telemetry.derived.unique - retention_time: 21600000 - replication_factor: 1 - # Error/Failed Topics - - name: telemetry.failed - retention_time: 86400000 - replication_factor: 1 - - name: telemetry.malformed - retention_time: 21600000 - replication_factor: 1 + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.duplicate + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.error + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.extractor.duplicate + retention_time: 172800000 + replication_factor: 2 - name: telemetry.extractor.failed - retention_time: 21600000 - replication_factor: 1 - - name: events.deviceprofile - retention_time: 604800000 - replication_factor: 1 - - name: system.command - retention_time: 3600 - replication_factor: 1 - - name: lms.audit.events retention_time: 172800000 - replication_factor: 1 - - name: lms.sso.events + replication_factor: 2 + - name: telemetry.failed retention_time: 172800000 - replication_factor: 1 - - name: lms.user.account.merge + replication_factor: 2 + - name: telemetry.ingest retention_time: 172800000 - replication_factor: 1 + replication_factor: 2 - name: telemetry.metrics - retention_time: 604800000 - replication_factor: 1 - - name: telemetry.assess.failed - retention_time: 2592000000 - replication_factor: 1 - - name: telemetry.share - retention_time: 2592000000 - replication_factor: 1 + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.raw + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.unique + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.unique.latest + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.unique.primary + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.unique.secondary + retention_time: 172800000 + replication_factor: 2 + - name: ml.observation.raw + retention_time: 172800000 + replication_factor: 2 + - name: ml.observation.druid + retention_time: 172800000 + replication_factor: 2 + - name: telemetry.cb.audit + retention_time: 172800000 + replication_factor: 2 + - name: druid.cb.audit + retention_time: 172800000 + replication_factor: 2 + - name: druid.cb.work.order.row + retention_time: 172800000 + replication_factor: 2 + - name: druid.cb.work.order.officer + retention_time: 172800000 + replication_factor: 2 + - name: rating.event + retention_time: 172800000 + replication_factor: 2 + - name: rating.failedRating + retention_time: 172800000 + replication_factor: 2 processing_kafka_topics: - - name: system.command - retention_time: 3600 - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.raw + - name: dashboards.course num_of_partitions: 4 - replication_factor: 1 - - name: telemetry.valid + replication_factor: 2 + - name: dashboards.role.count num_of_partitions: 4 - replication_factor: 1 - - name: telemetry.unique - num_of_partitions: 8 - replication_factor: 1 - - name: telemetry.duplicate - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.sink - num_of_partitions: 8 - replication_factor: 1 - - name: telemetry.with_location - num_of_partitions: 8 - replication_factor: 1 - - name: telemetry.audit - num_of_partitions: 2 - replication_factor: 1 - - name: telemetry.denorm + replication_factor: 2 + - name: dashboards.org.role.count num_of_partitions: 4 - replication_factor: 1 - - name: telemetry.denorm.valid - num_of_partitions: 2 - replication_factor: 1 - - name: telemetry.de_normalized + replication_factor: 2 + - name: dashboards.user.course.program.progress num_of_partitions: 4 - replication_factor: 1 - - name: events.telemetry + replication_factor: 2 + - name: dashboards.competency.frac num_of_partitions: 4 - replication_factor: 1 - - name: events.summary + replication_factor: 2 + - name: dashboards.competency.course num_of_partitions: 4 - replication_factor: 1 - - name: events.log - num_of_partitions: 2 - replication_factor: 1 - - name: events.error - num_of_partitions: 2 - replication_factor: 1 - - name: telemetry.log - num_of_partitions: 1 - replication_factor: 1 + replication_factor: 2 + - name: dashboards.competency.expected + num_of_partitions: 4 + replication_factor: 2 + - name: dashboards.competency.declared + num_of_partitions: 4 + replication_factor: 2 + - name: dashboards.competency.gap + num_of_partitions: 4 + replication_factor: 2 + - name: dashboards.user.org + num_of_partitions: 4 + replication_factor: 2 + - name: dashboards.org + num_of_partitions: 4 + replication_factor: 2 + - name: dashboards.assessment + num_of_partitions: 4 + replication_factor: 2 + - name: dashboards.user.assessment + num_of_partitions: 4 + replication_factor: 2 + - name: dashboards.acbp.enrolment + num_of_partitions: 4 + replication_factor: 2 - name: analytics.job_queue - num_of_partitions: 1 - replication_factor: 1 - - name: learning.graph.events - num_of_partitions: 1 - replication_factor: 1 - - name: pipeline_metrics - num_of_partitions: 1 - replication_factor: 1 - - name: metrics - num_of_partitions: 1 - replication_factor: 1 - - name: learning.graph.events.fail - num_of_partitions: 1 - replication_factor: 1 + num_of_partitions: 4 + replication_factor: 2 - name: analytics_metrics - num_of_partitions: 1 - replication_factor: 1 - - name: learning.job.request - num_of_partitions: 1 - replication_factor: 1 - - name: learning.job.request.fail - num_of_partitions: 1 - replication_factor: 1 - - name: learning.republish.job.request - num_of_partitions: 1 - replication_factor: 1 - - name: qrimage.request - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.derived - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.failed - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.malformed - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.extractor.failed - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.indexer.failed - num_of_partitions: 1 - replication_factor: 1 - - name: learning.events.failed - num_of_partitions: 1 - replication_factor: 1 + num_of_partitions: 4 + replication_factor: 2 + - name: druid.events.error + num_of_partitions: 16 + replication_factor: 2 + - name: druid.events.log + num_of_partitions: 16 + replication_factor: 2 + - name: druid.events.summary + num_of_partitions: 16 + replication_factor: 2 + - name: druid.events.telemetry + num_of_partitions: 16 + replication_factor: 2 - name: events.deviceprofile - num_of_partitions: 1 - replication_factor: 1 - - name: lms.audit.events - num_of_partitions: 1 - replication_factor: 1 - - name: lms.sso.events num_of_partitions: 4 replication_factor: 1 - - name: lms.user.account.merge - num_of_partitions: 1 - replication_factor: 1 + - name: prom.monitoring.metrics + num_of_partitions: 4 + replication_factor: 2 - name: telemetry.assess - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.assess.redact - num_of_partitions: 1 - replication_factor: 1 + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.assess.failed + num_of_partitions: 16 + replication_factor: 2 - name: telemetry.assess.raw - num_of_partitions: 1 - replication_factor: 1 + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.audit + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.denorm + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.derived + num_of_partitions: 16 + replication_factor: 2 - name: telemetry.derived.unique - num_of_partitions: 1 - replication_factor: 1 + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.duplicate + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.error + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.extractor.duplicate + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.extractor.failed + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.failed + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.ingest + num_of_partitions: 16 + replication_factor: 2 - name: telemetry.metrics - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.assess.failed - num_of_partitions: 1 - replication_factor: 1 - - name: telemetry.share - num_of_partitions: 1 - replication_factor: 1 - - name: prom.monitoring.metrics - num_of_partitions: 1 - replication_factor: 1 + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.raw + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.unique + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.unique.latest + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.unique.primary + num_of_partitions: 16 + replication_factor: 2 + - name: telemetry.unique.secondary + num_of_partitions: 16 + replication_factor: 2 + - name: ml.observation.raw + num_of_partitions: 4 + replication_factor: 2 + - name: ml.observation.druid + num_of_partitions: 4 + replication_factor: 2 + - name: telemetry.cb.audit + num_of_partitions: 16 + replication_factor: 2 + - name: druid.cb.audit + num_of_partitions: 16 + replication_factor: 2 + - name: druid.cb.work.order.row + num_of_partitions: 16 + replication_factor: 2 + - name: druid.cb.work.order.officer + num_of_partitions: 16 + replication_factor: 2 + - name: rating.event + num_of_partitions: 4 + replication_factor: 2 + - name: rating.failedRating + num_of_partitions: 4 + replication_factor: 2 + diff --git a/ansible/roles/setup-kafka/tasks/main.yml b/ansible/roles/setup-kafka/tasks/main.yml index 1e34906e90..d868f47173 100644 --- a/ansible/roles/setup-kafka/tasks/main.yml +++ b/ansible/roles/setup-kafka/tasks/main.yml @@ -1,50 +1,69 @@ +- name: Get list of existing Kafka topics + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{ingestion_zookeeper_ip}}:2181 --list + register: existing_ingestion_kafka_topics + tags: + - ingestion-kafka + - name: create topics - command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic {{env}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{ingestion_zookeeper_ip}}:2181 --create --topic "{{env}}.{{ item.name}}" --partitions {{ item.num_of_partitions}} --replication-factor {{ item.replication_factor}} with_items: "{{ingestion_kafka_topics}}" ignore_errors: true - when: kafka_id=="1" + when: kafka_id == "1" and "{{env}}.{{ item.name }}" not in existing_ingestion_kafka_topics.stdout_lines tags: - ingestion-kafka - name: override retention time - command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic {{env}}.{{item.name}} --config retention.ms={{ item.retention_time }} + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{ingestion_zookeeper_ip}}:2181 --alter --topic {{env}}.{{item.name}} --config retention.ms={{ item.retention_time }} with_items: "{{ingestion_kafka_overriden_topics}}" when: kafka_id=="1" and item.retention_time is defined tags: - ingestion-kafka +- name: Get list of existing Kafka topics + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{processing_zookeeper_ip}}:2181 --list + register: existing_processing_kafka_topics + tags: + - processing-kafka + - name: create topics - command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic {{env}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} - with_items: "{{processing_kafka_topics}}" + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{processing_zookeeper_ip}}:2181 --create --topic "{{env}}.{{ item.name}}" --partitions {{ item.num_of_partitions}} --replication-factor {{ item.replication_factor}} ignore_errors: true - when: kafka_id=="1" + when: kafka_id == "1" and "{{env}}.{{ item.name }}" not in existing_processing_kafka_topics.stdout_lines + with_items: "{{processing_kafka_topics}}" tags: - processing-kafka - name: override retention time - command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic {{env}}.{{item.name}} --config retention.ms={{ item.retention_time }} + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{processing_zookeeper_ip}}:2181 --alter --topic {{env}}.{{item.name}} --config retention.ms={{ item.retention_time }} with_items: "{{processing_kafka_overriden_topics}}" - when: kafka_id=="1" and item.retention_time is defined + when: kafka_id=="1" and item.retention_time is defined tags: - processing-kafka - name: overriding default partition size - shell: "/opt/kafka_2.12-1.1.0/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic {{item}} --config max.message.bytes={{kafka_partition_override_size}}" + shell: "/opt/kafka/bin/kafka-topics.sh --zookeeper {{processing_zookeeper_ip}}:2181 --alter --topic {{item}} --config max.message.bytes={{kafka_partition_override_size}}" with_items: - "{{env}}.telemetry.ingest" - "{{env}}.telemetry.extractor.failed" - name: overriding data pipeline downstream topic size - shell: "/opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic {{item}} --config max.message.bytes={{dp_downstream_kafka_max_message_bytes}}" + shell: "/opt/kafka/bin/kafka-topics.sh --zookeeper {{processing_zookeeper_ip}}:2181 --alter --topic {{item}} --config max.message.bytes={{dp_downstream_kafka_max_message_bytes}}" with_items: - - "{{env}}.telemetry.valid" - - "{{env}}.telemetry.unique" - - "{{env}}.telemetry.duplicate" - - "{{env}}.telemetry.sink" - - "{{env}}.telemetry.with_location" + - "{{env}}.druid.events.error" + - "{{env}}.druid.events.log" + - "{{env}}.druid.events.summary" + - "{{env}}.druid.events.telemetry" + - "{{env}}.telemetry.assess.raw" + - "{{env}}.telemetry.audit" - "{{env}}.telemetry.denorm" - - "{{env}}.telemetry.denorm.valid" - - "{{env}}.telemetry.log" - - "{{env}}.events.telemetry" - - "{{env}}.events.summary" - - "{{env}}.events.log" + - "{{env}}.telemetry.duplicate" + - "{{env}}.telemetry.error" + - "{{env}}.telemetry.extractor.duplicate" + - "{{env}}.telemetry.extractor.failed" + - "{{env}}.telemetry.failed" + - "{{env}}.telemetry.ingest" + - "{{env}}.telemetry.raw" + - "{{env}}.telemetry.unique" + - "{{env}}.telemetry.unique.latest" + - "{{env}}.telemetry.unique.primary" + - "{{env}}.telemetry.unique.secondary" diff --git a/ansible/roles/yarn/defaults/main.yml b/ansible/roles/yarn/defaults/main.yml deleted file mode 100644 index 4060075e8e..0000000000 --- a/ansible/roles/yarn/defaults/main.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -yarn_deploy_dir: /home/ecosystem/.deploy -repo_folder: /home/hduser/Ecosystem-Platform -hadoop_tarball: hadoop-{{hadoop_version}}.tar.gz -hadoop_download_url: https://archive.apache.org/dist/hadoop/common/hadoop-{{hadoop_version}}/{{hadoop_tarball}} -scala_tarball: scala-{{scala_version}}.tgz -scala_download_url: http://www.scala-lang.org/files/archive/{{scala_tarball}} -hadoop_yarn_home: /usr/local/hadoop-{{hadoop_version}} -hadoop_version: 2.7.2 -scala_version: 2.10.4 - -yarn_config_override: true -yarn_vmem_check_enabled: false -yarn_vmem_pmem_ratio: 2.1 -yarn_vcores: 16 -yarn_resource_memory: 20000 diff --git a/ansible/roles/yarn/files/truncate_logs.sh b/ansible/roles/yarn/files/truncate_logs.sh deleted file mode 100644 index 7dddd9d702..0000000000 --- a/ansible/roles/yarn/files/truncate_logs.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Truncate hadoop/yarn userlogs and keep the last 100 lines - -HADOOP_LOGS_HOME=/usr/local/hadoop/logs/userlogs - -for d in $HADOOP_LOGS_HOME/*/*/ ; do (cd $d && tail -n 100 stdout > stdout.tmp && cat stdout.tmp > stdout && rm stdout.tmp); done - -LOGSTASH_LOGS=/var/log/logstash -tail -n 100 $LOGSTASH_LOGS/logstash.stdout > $LOGSTASH_LOGS/logstash.stdout.tmp -cat $LOGSTASH_LOGS/logstash.stdout.tmp > $LOGSTASH_LOGS/logstash.stdout -rm $LOGSTASH_LOGS/logstash.stdout.tmp - -HADOOP_TMP_USERLOGS=/usr/local/hadoop/logs/userlogs - -for g in $HADOOP_TMP_USERLOGS/*/*/ do - cd $g - tail -n 100 stdout > stdout.tmp - cat stdout.tmp > stdout - rm stdout.tmp -done diff --git a/ansible/roles/yarn/tasks/common.yml b/ansible/roles/yarn/tasks/common.yml deleted file mode 100644 index 343aa54274..0000000000 --- a/ansible/roles/yarn/tasks/common.yml +++ /dev/null @@ -1,76 +0,0 @@ -- name: Common tasks for yarn master and slave - block: - - name: Download and extract hadoop tarball - unarchive: - src: "{{hadoop_download_url}}" - dest: "/usr/local/" - owner: hduser - group: hadoop - creates: "{{hadoop_yarn_home}}" - remote_src: yes - - - name: Creates symlink - file: - src: /usr/local/hadoop-{{hadoop_version}} - dest: /usr/local/hadoop - owner: hduser - group: hadoop - state: link - - - name: creating conf dir - file: - path: "{{hadoop_yarn_home}}/conf" - owner: hduser - group: hadoop - recurse: yes - state: directory - - - name: Templating configs - template: - src: "{{item}}" - dest: "{{hadoop_yarn_home}}/conf/{{item}}" - owner: hduser - group: hadoop - with_items: - - yarn-site.xml - - capacity-scheduler.xml - - core-site.xml - - log4j.properties - - hadoop-env.sh - - - name: Downloading artifacts - get_url: - url: "http://search.maven.org/remotecontent?filepath=org/{{item}}" - dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/" - owner: hduser - group: hadoop - with_items: - - clapper/grizzled-slf4j_2.10/1.0.1/grizzled-slf4j_2.10-1.0.1.jar - - apache/samza/samza-yarn_2.10/0.8.0/samza-yarn_2.10-0.8.0.jar - - apache/samza/samza-core_2.10/0.8.0/samza-core_2.10-0.8.0.jar - - - name: Download and extract scala - unarchive: - src: "{{scala_download_url}}" - dest: "/usr/local/" - owner: hduser - group: hadoop - remote_src: yes - - - name: Creates symlink - file: - src: "/usr/local/scala-{{scala_version}}" - dest: /usr/local/scala - owner: hduser - group: hadoop - state: link - - - name: copying scala files - copy: - src: "/usr/local/scala-{{scala_version}}/lib/{{item}}" - dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/" - remote_src: true - with_items: - - scala-compiler.jar - - scala-library.jar - delegate_to: "{{slave|default(inventory_hostname)}}" diff --git a/ansible/roles/yarn/tasks/main.yml b/ansible/roles/yarn/tasks/main.yml deleted file mode 100644 index a297e9a951..0000000000 --- a/ansible/roles/yarn/tasks/main.yml +++ /dev/null @@ -1,55 +0,0 @@ -- name: Debian | Install Maven - apt: - pkg: "{{item}}" - update_cache: yes - state: latest - install_recommends: yes - with_items: - - maven - - git - -# Running common tasks in master -- include: common.yml - -- lineinfile: - dest: /home/hduser/.bashrc - state: present - regexp: '^HADOOP_YARN_HOME' - line: 'HADOOP_YARN_HOME={{hadoop_yarn_home}}' - -- lineinfile: - dest: /home/hduser/.bashrc - state: present - regexp: '^HADOOP_CONF_DIR' - line: 'HADOOP_CONF_DIR=$HADOOP_YARN_HOME/conf' - -- file: - path: "/{{hadoop_yarn_home}}/conf/slaves" - state: touch - -- lineinfile: - dest: "/{{hadoop_yarn_home}}/conf/slaves" - state: present - regexp: "^{{item}}" - line: "{{item}}" - with_items: "{{yarn_slaves}}" - -# Running common tasks in slaves -- name: Running common tasks in slaves - include: common.yml - with_items: "{{yarn_slaves}}" - loop_control: - loop_var: slave - -- name: Copy truncate_files.sh - copy: - src: truncate_logs.sh - dest: /usr/local/bin - mode: 755 - -- name: Add truncate logs to cron - cron: - name: "Truncate yarn logs" - minute: "0" - job: "/usr/local/bin/truncate_logs.sh" - backup: yes diff --git a/ansible/roles/yarn/templates/capacity-scheduler.xml b/ansible/roles/yarn/templates/capacity-scheduler.xml deleted file mode 100644 index 4161b7ac03..0000000000 --- a/ansible/roles/yarn/templates/capacity-scheduler.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - - yarn.scheduler.capacity.maximum-applications - 10000 - - Maximum number of applications that can be pending and running. - - - - - yarn.scheduler.capacity.maximum-am-resource-percent - 0.5 - - Maximum percent of resources in the cluster which can be used to run - application masters i.e. controls number of concurrent running - applications. - - - - - yarn.scheduler.capacity.resource-calculator - org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator - - The ResourceCalculator implementation to be used to compare - Resources in the scheduler. - The default i.e. DefaultResourceCalculator only uses Memory while - DominantResourceCalculator uses dominant-resource to compare - multi-dimensional resources such as Memory, CPU etc. - - - - - yarn.scheduler.capacity.root.queues - default - - The queues at the this level (root is the root queue). - - - - - yarn.scheduler.capacity.root.default.capacity - 100 - Default queue target capacity. - - - - yarn.scheduler.capacity.root.default.user-limit-factor - 1 - - Default queue user limit a percentage from 0.0 to 1.0. - - - - - yarn.scheduler.capacity.root.default.maximum-capacity - 100 - - The maximum capacity of the default queue. - - - - - yarn.scheduler.capacity.root.default.state - RUNNING - - The state of the default queue. State can be one of RUNNING or STOPPED. - - - - - yarn.scheduler.capacity.root.default.acl_submit_applications - * - - The ACL of who can submit jobs to the default queue. - - - - - yarn.scheduler.capacity.root.default.acl_administer_queue - * - - The ACL of who can administer jobs on the default queue. - - - - - yarn.scheduler.capacity.node-locality-delay - -1 - - Number of missed scheduling opportunities after which the CapacityScheduler - attempts to schedule rack-local containers. - Typically this should be set to number of racks in the cluster, this - feature is disabled by default, set to -1. - - - - diff --git a/ansible/roles/yarn/templates/config.j2 b/ansible/roles/yarn/templates/config.j2 deleted file mode 100644 index f1b411ed5f..0000000000 --- a/ansible/roles/yarn/templates/config.j2 +++ /dev/null @@ -1,3 +0,0 @@ -Host {{item}} - IdentityFile ~/.ssh/hadoop_rsa - StrictHostKeyChecking no diff --git a/ansible/roles/yarn/templates/core-site.xml b/ansible/roles/yarn/templates/core-site.xml deleted file mode 100644 index d5a0752d52..0000000000 --- a/ansible/roles/yarn/templates/core-site.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - fs.http.impl - org.apache.samza.util.hadoop.HttpFileSystem - - diff --git a/ansible/roles/yarn/templates/hadoop-env.sh b/ansible/roles/yarn/templates/hadoop-env.sh deleted file mode 100644 index fd200c373c..0000000000 --- a/ansible/roles/yarn/templates/hadoop-env.sh +++ /dev/null @@ -1,2 +0,0 @@ -#export JAVA_HOME=/usr/lib/jvm/java-8-oracle -export JAVA_HOME=/opt/jdk1.8.0_121 diff --git a/ansible/roles/yarn/templates/log4j.properties b/ansible/roles/yarn/templates/log4j.properties deleted file mode 100644 index 4181560fcc..0000000000 --- a/ansible/roles/yarn/templates/log4j.properties +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hadoop.root.logger=INFO,console -hadoop.log.dir=. -hadoop.log.file=hadoop.log - -# Define the root logger to the system property "hadoop.root.logger". -log4j.rootLogger=${hadoop.root.logger}, EventCounter - -# Logging Threshold -log4j.threshold=ALL - -# Null Appender -log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender - -# -# Rolling File Appender - cap space usage at 5gb. -# -hadoop.log.maxfilesize=25MB -hadoop.log.maxbackupindex=1 -log4j.appender.RFA=org.apache.log4j.RollingFileAppender -log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} - -log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} -log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} - -log4j.appender.RFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -# Debugging Pattern format -#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - - -# -# Daily Rolling File Appender -# - -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -log4j.appender.DRFA.MaxBackupIndex=1 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -# Debugging Pattern format -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - - -# -# console -# Add "console" to rootlogger above if you want to use this -# - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n - -# -# TaskLog Appender -# - -#Default values -hadoop.tasklog.taskid=null -hadoop.tasklog.iscleanup=false -hadoop.tasklog.noKeepSplits=4 -hadoop.tasklog.totalLogFileSize=100 -hadoop.tasklog.purgeLogSplits=true -hadoop.tasklog.logsRetainHours=1 - -log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender -log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} -log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} -log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} - -log4j.appender.TLA.layout=org.apache.log4j.PatternLayout -log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -# -# HDFS block state change log from block manager -# -# Uncomment the following to suppress normal block state change -# messages from BlockManager in NameNode. -#log4j.logger.BlockStateChange=WARN - -# -#Security appender -# -hadoop.security.logger=INFO,NullAppender -hadoop.security.log.maxfilesize=256MB -hadoop.security.log.maxbackupindex=20 -log4j.category.SecurityLogger=${hadoop.security.logger} -hadoop.security.log.file=SecurityAuth-${user.name}.audit -log4j.appender.RFAS=org.apache.log4j.RollingFileAppender -log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} -log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} -log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} - -# -# Daily Rolling Security appender -# -log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} -log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd - -# -# hadoop configuration logging -# - -# Uncomment the following line to turn off configuration deprecation warnings. -# log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN - -# -# hdfs audit logging -# -hdfs.audit.logger=INFO,NullAppender -hdfs.audit.log.maxfilesize=25MB -hdfs.audit.log.maxbackupindex=1 -log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} -log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false -log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender -log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log -log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} -log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} - -# -# mapred audit logging -# -mapred.audit.logger=INFO,NullAppender -mapred.audit.log.maxfilesize=256MB -mapred.audit.log.maxbackupindex=20 -log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} -log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false -log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender -log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log -log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize} -log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex} - -# Custom Logging levels - -#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG -#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG -#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG - -# Jets3t library -log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR - -# -# Event Counter Appender -# Sends counts of logging messages at different severity levels to Hadoop Metrics. -# -log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter - -# -# Job Summary Appender -# -# Use following logger to send summary to separate file defined by -# hadoop.mapreduce.jobsummary.log.file : -# hadoop.mapreduce.jobsummary.logger=INFO,JSA -# -hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} -hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log -hadoop.mapreduce.jobsummary.log.maxfilesize=256MB -hadoop.mapreduce.jobsummary.log.maxbackupindex=20 -log4j.appender.JSA=org.apache.log4j.RollingFileAppender -log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} -log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} -log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} -log4j.appender.JSA.layout=org.apache.log4j.PatternLayout -log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n -log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} -log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false - -# -# Yarn ResourceManager Application Summary Log -# -# Set the ResourceManager summary log filename -yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log -# Set the ResourceManager summary log level and appender -yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} -#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY - -# To enable AppSummaryLogging for the RM, -# set yarn.server.resourcemanager.appsummary.logger to -# ,RMSUMMARY in hadoop-env.sh - -# Appender for ResourceManager Application Summary Log -# Requires the following properties to be set -# - hadoop.log.dir (Hadoop Log directory) -# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) -# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) - -log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} -log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false -log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender -log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} -log4j.appender.RMSUMMARY.MaxFileSize=25MB -log4j.appender.RMSUMMARY.MaxBackupIndex=1 -log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout -log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n - -# HS audit log configs -#mapreduce.hs.audit.logger=INFO,HSAUDIT -#log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} -#log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false -#log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender -#log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log -#log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout -#log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -#log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd - -# Http Server Request Logs -#log4j.logger.http.requests.namenode=INFO,namenoderequestlog -#log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log -#log4j.appender.namenoderequestlog.RetainDays=1 - -#log4j.logger.http.requests.datanode=INFO,datanoderequestlog -#log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log -#log4j.appender.datanoderequestlog.RetainDays=3 - -#log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog -#log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log -#log4j.appender.resourcemanagerrequestlog.RetainDays=3 - -#log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog -#log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log -#log4j.appender.jobhistoryrequestlog.RetainDays=3 - -#log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog -#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log -#log4j.appender.nodemanagerrequestlog.RetainDays=3 diff --git a/ansible/roles/yarn/templates/yarn-site.xml b/ansible/roles/yarn/templates/yarn-site.xml deleted file mode 100644 index 796f6450c3..0000000000 --- a/ansible/roles/yarn/templates/yarn-site.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - - yarn.resourcemanager.hostname - - {{resourcemanager}} - - - yarn.nodemanager.resource.memory-mb - {{yarn_resource_memory}} - - - yarn.scheduler.minimum-allocation-mb - 128 - - - mapreduce.job.userlog.retain.hours - 240 - - - yarn.log-aggregation-enable - false - - - yarn.nodemanager.log.retain-seconds - 3600 - - - yarn.nodemanager.recovery.enabled - true - - - yarn.nodemanager.address - 0.0.0.0:45454 - - - yarn.nodemanager.resource.cpu-vcores - {{yarn_vcores}} - - {% if yarn_config_override is defined %} - - yarn.nodemanager.vmem-check-enabled - {{yarn_vmem_check_enabled}} - - - yarn.nodemanager.vmem-pmem-ratio - {{yarn_vmem_pmem_ratio}} - - {% endif %} - - diff --git a/ansible/roles/zookeeper-upgrade/defaults/main.yml b/ansible/roles/zookeeper-upgrade/defaults/main.yml index efb0a39a40..75bc05dfbc 100644 --- a/ansible/roles/zookeeper-upgrade/defaults/main.yml +++ b/ansible/roles/zookeeper-upgrade/defaults/main.yml @@ -1,6 +1,6 @@ --- -zookeeper_version: 3.4.12 -zookeeper_url: http://www.us.apache.org/dist/zookeeper/zookeeper-{{zookeeper_version}}/zookeeper-{{zookeeper_version}}.tar.gz +zookeeper_version: 3.6.3 +zookeeper_url: "https://archive.apache.org/dist/zookeeper/zookeeper-{{ zookeeper_version }}/apache-zookeeper-{{ zookeeper_version }}-bin.tar.gz" zookeeper_port: 2181 zk_jvm_opts: "-Xms128m -Xmx128m" verify: True @@ -10,10 +10,13 @@ sync_limit: 2 tick_time: 2000 zoo_id: 1 maxclinetconnection_limit: 200 -data_dir: /var/lib/zookeeper +zookeeper_data_dir: /opt/zookeeper log_dir: /var/log/zookeeper client_port: "{{ zookeeper_port }}" -zookeeper_group: "{{ groups['zookeeper'] }}" +zookeeper_group: "{{ groups[remote] }}" +zookeeper_user: zookeeper +zookeeper_dir: /opt +zookeeper_systemd_service: /etc/systemd/system/zookeeper.service # List of dict (i.e. {zookeeper_hosts:[{host:,id:},{host:,id:},...]}) # zookeeper_hosts: # - host: "{{inventory_hostname}}" # the machine running diff --git a/ansible/roles/zookeeper-upgrade/handlers/main.yml b/ansible/roles/zookeeper-upgrade/handlers/main.yml index 3818cc1092..377aac88a9 100644 --- a/ansible/roles/zookeeper-upgrade/handlers/main.yml +++ b/ansible/roles/zookeeper-upgrade/handlers/main.yml @@ -1,3 +1,4 @@ --- -- name: Restart zookeeper - service: name=zookeeper state=restarted +- name: Restart zookeeper + systemd: name=zookeeper state=restarted + become: yes diff --git a/ansible/roles/zookeeper-upgrade/tasks/Debian.yml b/ansible/roles/zookeeper-upgrade/tasks/Debian.yml index c0b7efca7f..4d7bffc92e 100644 --- a/ansible/roles/zookeeper-upgrade/tasks/Debian.yml +++ b/ansible/roles/zookeeper-upgrade/tasks/Debian.yml @@ -1,41 +1,68 @@ ---- -- name: Update apt cache - apt: update_cache=yes cache_valid_time={{apt_cache_timeout}} +- name: download and unarchive zookeeper binary + become: yes + # become_user: "{{ zookeeper_user }}" + unarchive: + src: "{{ zookeeper_url }}" + dest: "{{ zookeeper_dir }}" + mode: 0755 + group: "{{ zookeeper_user }}" + owner: "{{ zookeeper_user }}" + remote_src: True -- name: Apt install required system packages. - apt: pkg={{item}} state=present - with_items: - - zookeeper - - zookeeperd +- name: check if the zookeeper dir already exists + stat: + path: "{{ zookeeper_dir }}/zookeeper" + register: folder_exists + +- name: rename the dir name + become: yes + shell: mv "apache-zookeeper-{{ zookeeper_version }}-bin" zookeeper + #group: "{{ zookeeper_user }}" + #owner: "{{ zookeeper_user }}" + args: + chdir: "{{ zookeeper_dir }}" + when: folder_exists.stat.exists == false - name: Registering server_id set_fact: - server_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}" + server_id: "{% for servername in ansible_play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}" - name: Overwrite myid file. - template: src=myid.j2 dest=/etc/zookeeper/conf/myid + template: src=myid.j2 dest={{ zookeeper_dir }}/zookeeper/myid notify: - Restart zookeeper - name: Overwrite log4j file. - template: src=log4j.properties.j2 dest=/etc/zookeeper/conf/log4j.properties + template: src=log4j.properties.j2 dest={{ zookeeper_dir }}/zookeeper/conf/log4j.properties notify: - Restart zookeeper - name: Overwrite env file. - template: src=environment.j2 dest=/etc/zookeeper/conf/environment + template: src=environment.j2 dest={{ zookeeper_dir }}/zookeeper/conf/environment notify: - Restart zookeeper - name: Overwrite default config file - template: src=zoo.cfg.j2 dest=/etc/zookeeper/conf/zoo.cfg + template: src=zoo.cfg.j2 dest={{ zookeeper_dir }}/zookeeper/conf/zoo.cfg notify: - Restart zookeeper -- name: Start zookeeper service - service: name=zookeeper state=started enabled=yes +- name: Change ownership of zookeeper installation + become: yes + file: path={{ zookeeper_dir }}/zookeeper owner={{ zookeeper_user }} group={{ zookeeper_user }} state=directory recurse=yes + +- name: create systemd config + template: dest={{zookeeper_systemd_service}} owner=root group=root mode=644 src=zookeeper.service.j2 + notify: + - Restart zookeeper + +- name: just force systemd to reread configs + systemd: daemon_reload=yes + +- name: Enable and Start zookeeper + systemd: name=zookeeper state=started enabled=yes + become: yes - name: wait for zookeeper port wait_for: host={{zookeeper_listen_address| default('localhost')}} port={{zookeeper_port}} state=started timeout=30 when: verify - diff --git a/ansible/roles/zookeeper-upgrade/tasks/main.yml b/ansible/roles/zookeeper-upgrade/tasks/main.yml index 3e8a8c09e3..2b1cf6892a 100644 --- a/ansible/roles/zookeeper-upgrade/tasks/main.yml +++ b/ansible/roles/zookeeper-upgrade/tasks/main.yml @@ -1,4 +1,11 @@ --- + +- name: setup group + group: name={{ zookeeper_user }} system=yes + +- name: Setup user + user: name={{ zookeeper_user }} system=yes group={{ zookeeper_user }} + - include: Debian.yml when: ansible_os_family == 'Debian' diff --git a/ansible/roles/zookeeper-upgrade/templates/log4j.properties.j2 b/ansible/roles/zookeeper-upgrade/templates/log4j.properties.j2 index 4b68d06d9d..28c255a1d5 100644 --- a/ansible/roles/zookeeper-upgrade/templates/log4j.properties.j2 +++ b/ansible/roles/zookeeper-upgrade/templates/log4j.properties.j2 @@ -50,3 +50,8 @@ log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1} # log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout ### Notice we are including log4j's NDC here (%x) # log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n +log4j.rootLogger=INFO, CONSOLE +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Target=System.out +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c %x - %m%n diff --git a/ansible/roles/zookeeper-upgrade/templates/zoo.cfg.j2 b/ansible/roles/zookeeper-upgrade/templates/zoo.cfg.j2 index 10eb5c0e78..4614e452aa 100644 --- a/ansible/roles/zookeeper-upgrade/templates/zoo.cfg.j2 +++ b/ansible/roles/zookeeper-upgrade/templates/zoo.cfg.j2 @@ -1,12 +1,14 @@ tickTime={{ tick_time }} -dataDir={{ data_dir }} -dataLogDir={{ log_dir }} +dataDir={{ zookeeper_data_dir }} +dataLogDir={{ zookeeper_data_dir }} clientPort={{ client_port }} initLimit={{ init_limit }} syncLimit={{ sync_limit }} maxClientCnxns={{ maxclinetconnection_limit }} - - +admin.enableServer={{ admin_enableserver | default('false') }} +admin.serverPort=8081 +{% if zookeeper_group | length > 1 %} {% for host in zookeeper_group %} server.{{hostvars[host]['server_id']}}={{ host }}:2888:3888 {% endfor %} +{% endif %} \ No newline at end of file diff --git a/ansible/roles/zookeeper-upgrade/templates/zookeeper.service.j2 b/ansible/roles/zookeeper-upgrade/templates/zookeeper.service.j2 new file mode 100644 index 0000000000..c33698d2b6 --- /dev/null +++ b/ansible/roles/zookeeper-upgrade/templates/zookeeper.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=ZooKeeper Service +Documentation=http://zookeeper.apache.org +Requires=network.target +After=network.target + +[Service] +Type=forking +User=zookeeper +Group=zookeeper +ExecStart={{ zookeeper_dir }}/zookeeper/bin/zkServer.sh start {{ zookeeper_dir }}/zookeeper/conf/zoo.cfg +ExecStop={{ zookeeper_dir }}/zookeeper/bin/zkServer.sh stop {{ zookeeper_dir }}/zookeeper/conf/zoo.cfg +ExecReload={{ zookeeper_dir }}/zookeeper/bin/zkServer.sh restart {{ zookeeper_dir }}/zookeeper/conf/zoo.cfg + +[Install] +WantedBy=default.target diff --git a/ansible/samza_deploy.yml b/ansible/samza_deploy.yml deleted file mode 100644 index 9ac34de840..0000000000 --- a/ansible/samza_deploy.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: Move the selected samza app tar files to another dir - hosts: localhost - tasks: - - name: find the selected samza app tar files path - find: - paths: "{{job_workspace}}/{{ samza_tar_files_localpath }}/allfiles" - patterns: "{{ job_names['%s'|format(item)].job_file_name }}*" - recurse: yes - with_items: "{{ job_names_to_kill.split(',') }}" - register: existing_files - - - name: Copy the selected samza app tar files to jobs folder - copy: - src: "{{ item }}" - dest: "{{job_workspace}}/{{ samza_tar_files_localpath }}/jobs" - with_items: - - "{{ existing_files | json_query('results[].files[].path') }}" - -- name: "Start Nodemanager on Slaves if stopped" - hosts: "yarn-slave" - vars: - hadoop_version: 2.7.2 - become: yes - pre_tasks: - - name: Ensure yarn nodemanager is running - become_user: hduser - shell: | - (ps aux | grep yarn-hduser-nodemanager | grep -v grep) \ - || /usr/local/hadoop/sbin/yarn-daemon.sh --config /usr/local/hadoop-{{ hadoop_version }}/conf/ start nodemanager \ - || sleep 10 - - - name: install imagemagick - apt: name=imagemagick state=present update_cache=yes - -- name: "Deploy Samza jobs" - hosts: "yarn-master" - become: yes - become_user: hduser - vars_files: - - "{{ inventory_dir }}/secrets.yml" - pre_tasks: - - name: Ensure yarn resource manager is running - shell: | - (ps aux | grep yarn-hduser-resourcemanager | grep -v grep) \ - || /usr/local/hadoop/sbin/yarn-daemon.sh --config /usr/local/hadoop-{{ hadoop_version }}/conf/ start resourcemanager - roles: - - samza-jobs diff --git a/ansible/secor_telemetry_backup_deploy.yaml b/ansible/secor_telemetry_backup_deploy.yaml index 7d10d41c61..62bf4f7011 100644 --- a/ansible/secor_telemetry_backup_deploy.yaml +++ b/ansible/secor_telemetry_backup_deploy.yaml @@ -3,19 +3,19 @@ become: yes vars_files: - "{{inventory_dir}}/secrets.yml" - pre_tasks: - - name: checking the list of installed services - service_facts: +# pre_tasks: +# - name: checking the list of installed services +# service_facts: - - name: Stop the monit - service: name=monit state=stopped - become: yes - when: ansible_facts.services.monit is defined +# - name: Stop the monit +# service: name=monit state=stopped +# become: yes +# when: ansible_facts.services.monit is defined roles: - secor-telemetry-backup-deploy - post_tasks: - - name: Start the monit - service: name=monit state=started - become: yes - when: ansible_facts.services.monit is defined +# post_tasks: +# - name: Start the monit +# service: name=monit state=started +# become: yes +# when: ansible_facts.services.monit is defined diff --git a/ansible/spark-cluster-job-submit.yml b/ansible/spark-cluster-job-submit.yml new file mode 100644 index 0000000000..ba4e017a23 --- /dev/null +++ b/ansible/spark-cluster-job-submit.yml @@ -0,0 +1,11 @@ +--- +- hosts: local + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}" + AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}" + roles: + - data-products-deploy + diff --git a/data-pipeline-flink/assessment-aggregator/pom.xml b/data-pipeline-flink/assessment-aggregator/pom.xml index 39f46e176e..2c4ca6f665 100644 --- a/data-pipeline-flink/assessment-aggregator/pom.xml +++ b/data-pipeline-flink/assessment-aggregator/pom.xml @@ -27,7 +27,7 @@ org.apache.flink - flink-streaming-scala_${scala.version} + flink-streaming-scala_${scala.maj.version} ${flink.version} provided @@ -46,13 +46,13 @@ org.apache.flink flink-test-utils_2.12 - 1.10.0 + ${flink.version} test org.apache.flink flink-runtime_2.12 - 1.10.0 + ${flink.version} test tests @@ -65,7 +65,7 @@ org.apache.flink flink-streaming-java_2.12 - 1.10.0 + ${flink.version} test tests @@ -96,7 +96,7 @@ org.cassandraunit cassandra-unit - 3.1.1.0 + 3.11.2.0 test @@ -105,6 +105,14 @@ src/main/scala src/test/scala + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + org.apache.maven.plugins maven-shade-plugin @@ -153,10 +161,11 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 4.4.0 - 1.8 - 1.8 + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} false diff --git a/data-pipeline-flink/assessment-aggregator/src/main/resources/assessment-aggregator.conf b/data-pipeline-flink/assessment-aggregator/src/main/resources/assessment-aggregator.conf index 1cd170e190..dcc342732a 100644 --- a/data-pipeline-flink/assessment-aggregator/src/main/resources/assessment-aggregator.conf +++ b/data-pipeline-flink/assessment-aggregator/src/main/resources/assessment-aggregator.conf @@ -4,16 +4,33 @@ kafka { input.topic = ${job.env}".telemetry.assess" failed.topic= ${job.env}".telemetry.assess.failed" groupId = ${job.env}"-assessment-aggregator-group" + output.certissue.topic = ${job.env}".issue.certificate.request" } task { + consumer.parallelism = 1 + downstream.parallelism = 1 assessaggregator { parallelism = 1 } + scoreaggregator { + parallelism = 1 + } } lms-cassandra { keyspace = "sunbird_courses" table = "assessment_aggregator" questionudttype= "question" + enrolmentstable = "user_enrolments" + activitytable = "user_activity_agg" +} +redis { + database { + relationCache.id = 10 + contentCache.id = 5 + } } +assessment.skip.missingRecords = false +content.read.api = "http://dev.sunbirded.org/api/content/v1/read/" +user.activity.agg.type="attempt_metrics" diff --git a/data-pipeline-flink/assessment-aggregator/src/main/resources/log4j.properties b/data-pipeline-flink/assessment-aggregator/src/main/resources/log4j.properties deleted file mode 100644 index dda4b21056..0000000000 --- a/data-pipeline-flink/assessment-aggregator/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=telemetry-extractor.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/domain/Event.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/domain/Event.scala index 60623bc77e..d77a325926 100644 --- a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/domain/Event.scala +++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/domain/Event.scala @@ -1,8 +1,6 @@ package org.sunbird.dp.assessment.domain import java.util - -import com.google.gson.internal.LinkedTreeMap import org.sunbird.dp.core.domain.{Events, EventsPath} class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { @@ -10,42 +8,40 @@ class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { private val jobName = "AssessmentAggregator" def assessmentEts: Long = { - val ets = telemetry.read[Double]("assessmentTs").get - ets.asInstanceOf[Double].longValue + val ets = telemetry.read[Long]("assessmentTs").get + ets.longValue() } - def courseId: String = { - telemetry.read("courseId").get + telemetry.read[String]("courseId").get } def contentId: String = { - telemetry.read("contentId").get + telemetry.read[String]("contentId").get } def batchId: String = { - telemetry.read("batchId").get + telemetry.read[String]("batchId").get } def attemptId: String = { - telemetry.read("attemptId").get + telemetry.read[String]("attemptId").get } def userId: String = { - telemetry.read("userId").get + telemetry.read[String]("userId").get } - def assessEvents: util.List[LinkedTreeMap[String, AnyRef]] = { - telemetry.read("events").get + def assessEvents: util.ArrayList[util.Map[String, AnyRef]] = { + telemetry.read[util.ArrayList[util.Map[String, AnyRef]]]("events").getOrElse(null) } def markFailed(errorMsg: String): Unit = { telemetry.addFieldIfAbsent(EventsPath.FLAGS_PATH, new util.HashMap[String, Boolean]) - telemetry.addFieldIfAbsent("metadata", new util.HashMap[String, AnyRef]) + telemetry.addFieldIfAbsent("metadata", new util.HashMap[String, AnyRef]) telemetry.add("metadata.validation_error", errorMsg) telemetry.add("metadata.src", jobName) - } } diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/AssessmentAggregatorFunction.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/AssessmentAggregatorFunction.scala index e027e632be..0bbfa3ec1b 100644 --- a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/AssessmentAggregatorFunction.scala +++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/AssessmentAggregatorFunction.scala @@ -5,11 +5,10 @@ import java.math.BigDecimal import java.sql.Timestamp import java.text.DecimalFormat import java.util - +import java.util.UUID import com.datastax.driver.core.querybuilder.QueryBuilder import com.datastax.driver.core.{Row, UDTValue, UserType} import com.google.gson.Gson -import com.google.gson.internal.LinkedTreeMap import com.google.gson.reflect.TypeToken import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration @@ -18,146 +17,289 @@ import org.joda.time.{DateTime, DateTimeZone} import org.slf4j.LoggerFactory import org.sunbird.dp.assessment.domain.Event import org.sunbird.dp.assessment.task.AssessmentAggregatorConfig +import org.sunbird.dp.contentupdater.core.util.RestUtil +import org.sunbird.dp.core.cache.{DataCache, RedisConnect} import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics} -import org.sunbird.dp.core.util.CassandraUtil +import org.sunbird.dp.core.util.{CassandraUtil, JSONUtil} import scala.collection.JavaConverters._ import scala.collection.mutable -case class Question(id: String, maxscore: Double, params: util.List[LinkedTreeMap[String, Any]], title: String, `type`: String, desc: String) +case class Question(id: String, maxscore: Double, params: util.List[util.HashMap[String, Any]], title: String, `type`: String, desc: String) -case class QuestionData(resvalues: util.List[LinkedTreeMap[String, Any]], duration: Double, score: Double, item: Question) +case class QuestionData(resvalues: util.List[util.HashMap[String, Any]], duration: Double, score: Double, item: Question) -case class AssessEvent(ets: Double, edata: QuestionData) +case class AssessEvent(ets: Long, edata: QuestionData) case class Aggregate(totalScore: Double, totalMaxScore: Double, grandTotal: String, questionsList: List[UDTValue]) - class AssessmentAggregatorFunction(config: AssessmentAggregatorConfig, @transient var cassandraUtil: CassandraUtil = null )(implicit val mapTypeInfo: TypeInformation[Event]) extends BaseProcessFunction[Event, Event](config) { - val mapType: Type = new TypeToken[util.Map[String, AnyRef]]() {}.getType - private[this] val logger = LoggerFactory.getLogger(classOf[AssessmentAggregatorFunction]) - var questionType: UserType = _ - private val df = new DecimalFormat("0.0#") - - override def metricsList() = List(config.dbUpdateCount, config.dbReadCount, config.failedEventCount, config.batchSuccessCount, config.skippedEventCount) - - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) - questionType = cassandraUtil.getUDTType(config.dbKeyspace, config.dbudtType) - } - - override def close(): Unit = { - super.close() + val mapType: Type = new TypeToken[util.Map[String, AnyRef]]() {}.getType + private[this] val logger = LoggerFactory.getLogger(classOf[AssessmentAggregatorFunction]) + private var dataCache: DataCache = _ + private var contentCache: DataCache = _ + var questionType: UserType = _ + private var restUtil: RestUtil = _ + + + override def metricsList() = List(config.dbUpdateCount, config.dbReadCount, + config.failedEventCount, config.batchSuccessCount, + config.skippedEventCount, config.cacheHitCount, config.cacheHitMissCount, config.certIssueEventsCount, config.apiHitFailedCount, config.apiHitSuccessCount, config.ignoredEventsCount, config.recomputeAggEventCount) + + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.relationCacheNode, List()) + dataCache.init() + contentCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.contentCacheNode, List()) + contentCache.init() + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + questionType = cassandraUtil.getUDTType(config.dbKeyspace, config.dbudtType) + restUtil = new RestUtil() + } + + override def close(): Unit = { + super.close() + } + + def getListValues(values: util.List[util.HashMap[String, Any]]): mutable.Buffer[util.Map[String, Any]] = { + values.asScala.map { res => + res.asScala.map { + case (key, value) => key -> (if (null != value && !value.isInstanceOf[String]) new Gson().toJson(value) else value) + }.toMap.asJava } - - def getListValues(values: util.List[LinkedTreeMap[String, Any]]): mutable.Buffer[util.Map[String, Any]] = { - values.asScala.map { res => - res.asScala.map { - case (key, value) => key -> (if (null != value && !value.isInstanceOf[String]) new Gson().toJson(value) else value) - }.toMap.asJava - } - } - - /** - * Method to write the assess event to cassandra table - * - * @param event - Assess Batch Events - * @param context - Process Context - */ - override def processElement(event: Event, - context: ProcessFunction[Event, Event]#Context, - metrics: Metrics): Unit = { - try { - var totalScore = 0.0 - var totalMaxScore = 0.0 - val assessment = getAssessment(event) - val assessEvents = event.assessEvents.asScala - - val sortAndFilteredEvents = assessEvents.map(event => { - AssessEvent(event.get("ets").asInstanceOf[Double], new Gson().fromJson(new Gson().toJson(event.get("edata")), - classOf[QuestionData])) - }).sortWith(_.ets > _.ets).groupBy(_.edata.item.id).map(_._2.head) - - val result = sortAndFilteredEvents.map(event => { - totalScore = totalScore + event.edata.score - totalMaxScore = totalMaxScore + event.edata.item.maxscore - getQuestion(event.edata, event.ets.longValue()) - }) - - val grandTotal = String.format("%s/%s", df.format(totalScore), df.format(totalMaxScore)) - - if (null == assessment) { - saveAssessment(event, Aggregate(totalScore, totalMaxScore, grandTotal, result.toList), new DateTime().getMillis) - metrics.incCounter(config.dbUpdateCount) - metrics.incCounter(config.batchSuccessCount) - } - else { - metrics.incCounter(config.dbReadCount) - if (event.assessmentEts > assessment.getTimestamp("last_attempted_on").getTime) { - saveAssessment(event, Aggregate(totalScore, totalMaxScore, grandTotal, result.toList), - assessment.getTimestamp("created_on").getTime) - metrics.incCounter(config.dbUpdateCount) - metrics.incCounter(config.batchSuccessCount) - } - else { - metrics.incCounter(config.skippedEventCount) - } - } - } catch { - case ex: Exception => - logger.info("Assessment Failed with exception :", ex) - event.markFailed(ex.getMessage) + } + + /** + * Method to write the assess event to cassandra table + * + * @param event - Assess Batch Events + * @param context - Process Context + */ + override def processElement(event: Event, + context: ProcessFunction[Event, Event]#Context, + metrics: Metrics): Unit = { + try { + // Validating the contentId + if (isValidContent(event.courseId, event.contentId)(metrics)) { + val assessEvents = event.assessEvents.asScala + if(null != assessEvents && !assessEvents.isEmpty) { + val sortAndFilteredEvents: List[AssessEvent] = getUniqueQuestions(assessEvents = assessEvents.toList) + if (config.skipMissingRecords) { // Skip configuration to enable the force filter of duplicate questions based on the question meta + val totalQuestions = getTotalQuestionsCount(event.contentId)(metrics) + logger.info(s"The total Question count value - ContentId:${event.contentId}, TotalQuestionCount:$totalQuestions") + if (totalQuestions == null) { + updateDB(scoreMetrics = computeScoreMetrics(sortAndFilteredEvents), event = event)(metrics, context) + } else { + /** + * If the totalQuestions from the content Meta and Events count are matching + * Then we are computing the score metrics and updating the table + */ + if (sortAndFilteredEvents.size <= totalQuestions.asInstanceOf[Int]) { + updateDB(scoreMetrics = computeScoreMetrics(sortAndFilteredEvents), event = event)(metrics, context) + } else { + /** + * if the totalQuestions Count value is not matching with the event size then job should skip those events and increase the metrics value + */ context.output(config.failedEventsOutputTag, event) - metrics.incCounter(config.failedEventCount) + metrics.incCounter(config.ignoredEventsCount) + } + } + } else { + /** + * If the config.skipMissingRecords = false then it will work as previous logic, no further validation + */ + updateDB(scoreMetrics = computeScoreMetrics(sortAndFilteredEvents), event = event)(metrics, context) + } + } else { + recomputeAggregates(event)(metrics, context) } + + } else { + context.output(config.failedEventsOutputTag, event) + metrics.incCounter(config.failedEventCount) + } + } catch { + case ex: Exception => + ex.printStackTrace() + logger.info("Assessment Failed with exception :", ex.getMessage) + event.markFailed(ex.getMessage) + context.output(config.failedEventsOutputTag, event) + metrics.incCounter(config.failedEventCount) } - - def getAssessment(event: Event): Row = { - - val query = QueryBuilder.select("last_attempted_on", "created_on") - .from(config.dbKeyspace, config.dbTable). - where(QueryBuilder.eq("attempt_id", event.attemptId)) - .and(QueryBuilder.eq("batch_id", event.batchId)) - .and(QueryBuilder.eq("user_id", event.userId)) - .and(QueryBuilder.eq("content_id", event.contentId)) - .and(QueryBuilder.eq("course_id", event.courseId)).toString - - cassandraUtil.findOne(query) - + } + + def updateDB(scoreMetrics: Aggregate, event: Event)(metrics: Metrics, context: ProcessFunction[Event, Event]#Context): Unit = { + val assessmentDBResult = getAssessment(event) + if (null == assessmentDBResult) { + saveAssessment(event, Aggregate(scoreMetrics.totalScore, scoreMetrics.totalMaxScore, scoreMetrics.grandTotal, scoreMetrics.questionsList), new DateTime().getMillis) + metrics.incCounter(config.dbUpdateCount) + metrics.incCounter(config.batchSuccessCount) + context.output(config.scoreAggregateTag, event) + createIssueCertEvent(event, context, metrics) } - - - def saveAssessment(batchEvent: Event, aggregate: Aggregate, createdOn: Long): Unit = { - val query = QueryBuilder.insertInto(config.dbKeyspace, config.dbTable) - .value("course_id", batchEvent.courseId).value("batch_id", batchEvent.batchId).value("user_id", batchEvent.userId) - .value("content_id", batchEvent.contentId).value("attempt_id", batchEvent.attemptId) - .value("updated_on", new DateTime(DateTimeZone.UTC).getMillis).value("created_on", createdOn) - .value("last_attempted_on", batchEvent.assessmentEts).value("total_score", aggregate.totalScore) - .value("total_max_score", aggregate.totalMaxScore) - .value("question", aggregate.questionsList.asJava).value("grand_total", aggregate.grandTotal).toString - - cassandraUtil.upsert(query) - logger.info("Successfully Aggregated the batch event - batchid: " - + batchEvent.batchId + " ,userid: " + batchEvent.userId + " ,couserid: " - + batchEvent.courseId + " ,contentid: " + batchEvent.contentId, "attempid" + batchEvent.attemptId) + else { + metrics.incCounter(config.dbReadCount) + if (event.assessmentEts > assessmentDBResult.getTimestamp("last_attempted_on").getTime) { + saveAssessment(event, Aggregate(scoreMetrics.totalScore, scoreMetrics.totalMaxScore, scoreMetrics.grandTotal, scoreMetrics.questionsList), + assessmentDBResult.getTimestamp("created_on").getTime) + metrics.incCounter(config.dbUpdateCount) + metrics.incCounter(config.batchSuccessCount) + context.output(config.scoreAggregateTag, event) + createIssueCertEvent(event, context, metrics) + } + else { + metrics.incCounter(config.skippedEventCount) + } } - - def getQuestion(questionData: QuestionData, assessTs: Long): UDTValue = - - questionType.newValue().setString("id", questionData.item.id).setDouble("max_score", questionData.item.maxscore) - .setDouble("score", questionData.score) - .setString("type", questionData.item.`type`) - .setString("title", questionData.item.title) - .setList("resvalues", getListValues(questionData.resvalues).asJava).setList("params", getListValues(questionData.item.params).asJava) - .setString("description", questionData.item.desc) - .setDecimal("duration", BigDecimal.valueOf(questionData.duration)).setTimestamp("assess_ts", new Timestamp(assessTs)) + } + + def getUniqueQuestions(assessEvents: List[util.Map[String, AnyRef]]): List[AssessEvent] = { + assessEvents.map(event => { + AssessEvent(event.get("ets").asInstanceOf[Long], new Gson().fromJson(new Gson().toJson(event.get("edata")), classOf[QuestionData])) + }).sortWith(_.ets > _.ets).groupBy(_.edata.item.id).map(_._2.head).toList + } + + def computeScoreMetrics(events: List[AssessEvent]): Aggregate = { + var totalScore = 0.0 + var totalMaxScore = 0.0 + val df = new DecimalFormat("0.0#") + val questions = events.map(event => { + totalScore = totalScore + event.edata.score + totalMaxScore = totalMaxScore + event.edata.item.maxscore + getQuestion(event.edata, event.ets.longValue()) + }) + val grandTotal = String.format("%s/%s", df.format(totalScore), df.format(totalMaxScore)) + Aggregate(totalScore = totalScore, totalMaxScore = totalMaxScore, grandTotal = grandTotal, questionsList = questions) + } + + def getAssessment(event: Event): Row = { + + val query = QueryBuilder.select("last_attempted_on", "created_on") + .from(config.dbKeyspace, config.dbTable). + where(QueryBuilder.eq("attempt_id", event.attemptId)) + .and(QueryBuilder.eq("batch_id", event.batchId)) + .and(QueryBuilder.eq("user_id", event.userId)) + .and(QueryBuilder.eq("content_id", event.contentId)) + .and(QueryBuilder.eq("course_id", event.courseId)).toString + + cassandraUtil.findOne(query) + + } + def isValidContent(courseId: String, contentId: String)(metrics: Metrics): Boolean = { + val leafNodes = dataCache.getKeyMembers(key = s"$courseId:$courseId:leafnodes") + if (!leafNodes.isEmpty) { + metrics.incCounter(config.cacheHitCount) + leafNodes.contains(contentId) + } else { + metrics.incCounter(config.cacheHitMissCount) + true + } + } + + def saveAssessment(batchEvent: Event, aggregate: Aggregate, createdOn: Long): Unit = { + val query = QueryBuilder.insertInto(config.dbKeyspace, config.dbTable) + .value("course_id", batchEvent.courseId).value("batch_id", batchEvent.batchId).value("user_id", batchEvent.userId) + .value("content_id", batchEvent.contentId).value("attempt_id", batchEvent.attemptId) + .value("updated_on", new DateTime(DateTimeZone.UTC).getMillis).value("created_on", createdOn) + .value("last_attempted_on", batchEvent.assessmentEts).value("total_score", aggregate.totalScore) + .value("total_max_score", aggregate.totalMaxScore) + .value("question", aggregate.questionsList.asJava).value("grand_total", aggregate.grandTotal).toString + + cassandraUtil.upsert(query) + logger.info("Successfully Aggregated the batch event - batchid: " + + batchEvent.batchId + " ,userid: " + batchEvent.userId + " ,couserid: " + + batchEvent.courseId + " ,contentid: " + batchEvent.contentId + "attempid" + batchEvent.attemptId) + } + + def getQuestion(questionData: QuestionData, assessTs: Long): UDTValue = { + + questionType.newValue().setString("id", questionData.item.id).setDouble("max_score", questionData.item.maxscore) + .setDouble("score", questionData.score) + .setString("type", questionData.item.`type`) + .setString("title", questionData.item.title) + .setList("resvalues", getListValues(questionData.resvalues).asJava).setList("params", getListValues(questionData.item.params).asJava) + .setString("description", questionData.item.desc) + .setDecimal("duration", BigDecimal.valueOf(questionData.duration)).setTimestamp("assess_ts", new Timestamp(assessTs)) + } + + /** + * Generation of Certificate Issue event for the enrolment completed users to validate and generate certificate. + * + * @param batchEvent + * @param context + * @param metrics + */ + def createIssueCertEvent(batchEvent: Event, context: ProcessFunction[Event, Event]#Context, + metrics: Metrics): Unit = { + val ets = System.currentTimeMillis + val mid = s"""LP.${ets}.${UUID.randomUUID}""" + val event = + s"""{"eid": "BE_JOB_REQUEST", + |"ets": ${ets}, + |"mid": "${mid}", + |"actor": {"id": "Course Certificate Generator","type": "System"}, + |"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}}, + |"object": {"id": "${batchEvent.batchId}_${batchEvent.courseId}","type": "CourseCertificateGeneration"}, + |"edata": {"userIds": ["${batchEvent.userId}"],"action": "issue-certificate","iteration": 1, "trigger": "auto-issue","batchId": "${batchEvent.batchId}","reIssue": false,"courseId": "${batchEvent.courseId}"}}""" + .stripMargin.replaceAll("\n", "") + context.output(config.certIssueOutputTag, event) + metrics.incCounter(config.certIssueEventsCount) + } + + def getQuestionCountFromCache(contentId: String)(metrics: Metrics) = { + val result = contentCache.getWithRetry(contentId) + metrics.incCounter(config.cacheHitCount) + result.getOrElse("totalQuestions", 0.0).asInstanceOf[Double].toInt + } + + def getQuestionCountFromAPI(contentId: String)(metrics: Metrics) = { + val contentReadResp = JSONUtil.deserialize[util.HashMap[String, AnyRef]](restUtil.get(config.contentReadAPI.concat(contentId))) + if (contentReadResp.get("responseCode").asInstanceOf[String].toUpperCase.equalsIgnoreCase("OK")) { + metrics.incCounter(config.apiHitSuccessCount) + val result = contentReadResp.getOrDefault("result", new util.HashMap()).asInstanceOf[util.Map[String, AnyRef]] + val content = result.getOrDefault("content", new util.HashMap()).asInstanceOf[util.Map[String, Any]] + val totalQuestions = content.getOrDefault("totalQuestions", null) + logger.info(s"Fetched the totalQuestion Value from the Content Read API - ContentId:$contentId, TotalQuestionCount:$totalQuestions") + totalQuestions + } else { + metrics.incCounter(config.apiHitFailedCount) + logger.info(s"API Failed to Fetch the TotalQuestion Count - ContentId:$contentId, ResponseCode - ${contentReadResp.get("responseCode")} ") + throw new Exception(s"Failed to fetch the content meta for the content ID: $contentId") // Job should stop if the api has failed + } + } + + def getTotalQuestionsCount(contentId: String)(metrics: Metrics) = { + val totalQuestionsCountFromCache = getQuestionCountFromCache(contentId)(metrics) + logger.info(s" Total Question Count Value From Redis - ContentId:$contentId, TotalQuestionCount - ${totalQuestionsCountFromCache} ") + if (isNegligibleValue(totalQuestionsCountFromCache)) { + metrics.incCounter(config.cacheHitMissCount) + getQuestionCountFromAPI(contentId)(metrics) + } else { + totalQuestionsCountFromCache + } + } + + def isNegligibleValue(value: Int): Boolean = { + if (value == 0) true else false + } + + /** + * Pushes event to compute the score aggregates and pushes certificate issue event to kafka + * @param event + * @param metrics + * @param context + */ + def recomputeAggregates(event: Event)(metrics: Metrics, context: ProcessFunction[Event, Event]#Context) = { + metrics.incCounter(config.recomputeAggEventCount) + context.output(config.scoreAggregateTag, event) + createIssueCertEvent(event, context, metrics) + } } diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/UserScoreAggregateFunction.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/UserScoreAggregateFunction.scala new file mode 100644 index 0000000000..fabda4fbf8 --- /dev/null +++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/UserScoreAggregateFunction.scala @@ -0,0 +1,110 @@ +package org.sunbird.dp.assessment.functions + +import java.lang.reflect.Type +import java.util +import java.util.Date + +import com.datastax.driver.core.Row +import com.datastax.driver.core.querybuilder.QueryBuilder +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.dp.assessment.domain.Event +import org.apache.flink.configuration.Configuration +import org.sunbird.dp.assessment.task.AssessmentAggregatorConfig +import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics} +import org.sunbird.dp.core.util.CassandraUtil + +import scala.collection.JavaConverters._ + +case class AggDetails(attempt_id: String, last_attempted_on: Date, score: Double, content_id: String, max_score: Double, `type`: String) + +case class UserActivityAgg(aggregates: Map[String, Double], aggDetails: List[String]) + +class UserScoreAggregateFunction(config: AssessmentAggregatorConfig, + @transient var cassandraUtil: CassandraUtil = null + )(implicit val mapTypeInfo: TypeInformation[Event]) + extends BaseProcessFunction[Event, Event](config) { + + val mapType: Type = new TypeToken[util.Map[String, AnyRef]]() {}.getType + private[this] val logger = LoggerFactory.getLogger(classOf[UserScoreAggregateFunction]) + + override def metricsList() = List(config.dbScoreAggUpdateCount, config.dbScoreAggReadCount, + config.failedEventCount, config.batchSuccessCount, + config.skippedEventCount) + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + } + + override def close(): Unit = { + super.close() + } + + def getAggregateDetails(assessAggRows: List[Row]): List[String] = { + if (null != assessAggRows && !assessAggRows.isEmpty) { + assessAggRows.map { row => + val aggMap = AggDetails(row.getString("attempt_id"), row.getTimestamp("last_attempted_on"), + row.getDouble("score"), row.getString("content_id"), row.getDouble("total_max_score"), config.aggType) + new Gson().toJson(aggMap) + }.toList + } else List() + } + + def getAggregates(assessAggRows: List[Row]): Map[String, Double] = { + if (null != assessAggRows && !assessAggRows.isEmpty) { + val attemptGroupList = assessAggRows.groupBy(row => row.getString("content_id")).values + attemptGroupList.map(row => { + val scoreRow = row.maxBy(r => r.getDouble("score")) + + Map(s"score:${scoreRow.getString("content_id")}" -> scoreRow.getDouble("score"), + s"max_score:${scoreRow.getString("content_id")}" -> scoreRow.getDouble("total_max_score"), + s"attempts_count:${scoreRow.getString("content_id")}" -> row.length.toDouble) + }).flatten.toMap + } else Map[String, Double]() + } + + def getBestScore(event: Event): UserActivityAgg = { + val query = QueryBuilder.select().column("content_id").column("attempt_id").column("last_attempted_on").column("total_max_score").column("total_score").as("score").from(config.dbKeyspace, config.dbTable) + .where(QueryBuilder.eq("course_id", event.courseId)).and(QueryBuilder.eq("batch_id", event.batchId)) + .and(QueryBuilder.eq("user_id", event.userId)) + val rows = cassandraUtil.find(query.toString).asScala.toList + + UserActivityAgg(aggregates = getAggregates(rows), aggDetails = getAggregateDetails(rows)) + } + + def updateUserActivity(event: Event, score: UserActivityAgg): Unit = { + val scoreLastUpdatedTime: Map[String, Long] = score.aggregates.map(m => m._1 -> System.currentTimeMillis()) + val updateQuery = QueryBuilder.update(config.dbKeyspace, config.activityTable) + .`with`(QueryBuilder.putAll(config.aggregates, score.aggregates.asJava)) + .and(QueryBuilder.set(config.aggDetails, score.aggDetails.asJava)) + .and(QueryBuilder.putAll(config.aggLastUpdated, scoreLastUpdatedTime.asJava)) + .where(QueryBuilder.eq(config.activityId, event.courseId)) + .and(QueryBuilder.eq(config.activityType, "Course")) + .and(QueryBuilder.eq(config.contextId, "cb:" + event.batchId)) + .and(QueryBuilder.eq(config.activityUser, event.userId)) + cassandraUtil.upsert(updateQuery.toString) + logger.info("Successfully updated scores in user activity - batchid: " + + event.batchId + " ,userid: " + event.userId + " ,couserid: " + + event.courseId) + } + + override def processElement(event: Event, + context: ProcessFunction[Event, Event]#Context, + metrics: Metrics): Unit = { + val score: UserActivityAgg = getBestScore(event) + metrics.incCounter(config.dbScoreAggReadCount) + if (score.aggregates.nonEmpty || score.aggDetails.nonEmpty) { + updateUserActivity(event, score) + metrics.incCounter(config.dbScoreAggUpdateCount) + } else { + logger.info("No scores to update for batchid: " + + event.batchId + " ,userid: " + event.userId + " ,couserid: " + + event.courseId) + } + } + +} diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorConfig.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorConfig.scala index 1a37bfcb06..2fc795681a 100644 --- a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorConfig.scala +++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorConfig.scala @@ -9,44 +9,83 @@ import org.sunbird.dp.core.job.BaseJobConfig class AssessmentAggregatorConfig(override val config: Config) extends BaseJobConfig(config, jobName = "AssessmentAggregatorJob") { - private val serialVersionUID = 2905979434303791379L - - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - - // Kafka Topics Configurationval kafkaInputTopic: String = config.getString("kafka.input.topic") - - val assessAggregatorParallelism: Int = config.getInt("task.assessaggregator.parallelism") - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val kafkaFailedTopic: String = config.getString("kafka.failed.topic") - - // Metric List - val dbUpdateCount = "db-update-count" - val dbReadCount = "db-read-count" - val batchSuccessCount = "batch-success-event-count" - val failedEventCount = "failed-event-count" - val skippedEventCount = "skipped-event-count" - - - //Cassandra - - val dbTable: String = config.getString("lms-cassandra.table") - val dbKeyspace: String = config.getString("lms-cassandra.keyspace") - val dbHost: String = config.getString("lms-cassandra.host") - val dbPort: Int = config.getInt("lms-cassandra.port") - val dbudtType: String = config.getString("lms-cassandra.questionudttype") - - val FAILED_EVENTS_OUTPUT_TAG = "failed-events" - - val failedEventsOutputTag: OutputTag[Event] = OutputTag[Event]("assess-failed-events") - - // Consumers - val assessmentAggConsumer = "assessment-agg-consumer" - - // Functions - val assessmentAggregatorFunction = "AssessmentAggregatorFunction" - - // Producers - val assessFailedEventsSink = "assess-failed-events-sink" + private val serialVersionUID = 2905979434303791379L + + implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + + + // Kafka Topics Configurationval kafkaInputTopic: String = config.getString("kafka.input.topic") + // Parallelism configs + val assessAggregatorParallelism: Int = config.getInt("task.assessaggregator.parallelism") + val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.parallelism") + val scoreAggregatorParallelism: Int = config.getInt("task.scoreaggregator.parallelism") + override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") + + val kafkaInputTopic: String = config.getString("kafka.input.topic") + val kafkaFailedTopic: String = config.getString("kafka.failed.topic") + val kafkaCertIssueTopic: String = config.getString("kafka.output.certissue.topic") + + // Metric List + val dbUpdateCount = "db-update-count" + val dbReadCount = "db-read-count" + val batchSuccessCount = "batch-success-event-count" + val failedEventCount = "failed-event-count" + val ignoredEventsCount = "ignored-event-count" + val skippedEventCount = "skipped-event-count" + val cacheHitCount = "cache-hit-count" + val cacheHitMissCount = "cache-hit-miss-count" + val certIssueEventsCount = "cert-issue-events-count" + val dbScoreAggUpdateCount = "db-score-update-count" + val dbScoreAggReadCount = "db-score-read-count" + val apiHitSuccessCount = "api-hit-success-count" + val apiHitFailedCount = "api-hit-failed-count" + val recomputeAggEventCount = "recompute-agg-event-count" + + + //Cassandra + + val dbTable: String = config.getString("lms-cassandra.table") + val dbKeyspace: String = config.getString("lms-cassandra.keyspace") + val dbHost: String = config.getString("lms-cassandra.host") + val dbPort: Int = config.getInt("lms-cassandra.port") + val dbudtType: String = config.getString("lms-cassandra.questionudttype") + val enrolmentTable: String = config.getString("lms-cassandra.enrolmentstable") + val activityTable: String = config.getString("lms-cassandra.activitytable") + + val FAILED_EVENTS_OUTPUT_TAG = "failed-events" + + val failedEventsOutputTag: OutputTag[Event] = OutputTag[Event]("assess-failed-events") + val certIssueOutputTagName = "certificate-issue-events" + val certIssueOutputTag: OutputTag[String] = OutputTag[String](certIssueOutputTagName) + + // Consumers + val assessmentAggConsumer = "assessment-agg-consumer" + + // Functions + val assessmentAggregatorFunction = "AssessmentAggregatorFunction" + + // Producers + val assessFailedEventsSink = "assess-failed-events-sink" + val certIssueEventSink = "certificate-issue-event-sink" + val userScoreAggregateFn = "user-score-aggregator" + + // Cache + val relationCacheNode:Int = config.getInt("redis.database.relationCache.id") + val contentCacheNode:Int = config.getInt("redis.database.contentCache.id") + + //UserActivityAgg + val scoreAggregateTag: OutputTag[Event] = OutputTag[Event]("score-aggregate-events") + val activityType = "activity_type" + val activityId = "activity_id" + val contextId = "context_id" + val activityUser = "user_id" + val aggLastUpdated = "agg_last_updated" + val aggDetails = "agg_details" + val aggregates = "aggregates" + + val aggType = config.getString("user.activity.agg.type") + + val skipMissingRecords: Boolean = config.getBoolean("assessment.skip.missingRecords") + val contentReadAPI: String = config.getString("content.read.api") } diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorStreamTask.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorStreamTask.scala index a3dcbb953f..bb6a3399aa 100644 --- a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorStreamTask.scala +++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorStreamTask.scala @@ -6,10 +6,9 @@ import com.typesafe.config.ConfigFactory import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.sunbird.dp.assessment.domain.Event -import org.sunbird.dp.assessment.functions.AssessmentAggregatorFunction +import org.sunbird.dp.assessment.functions.{AssessmentAggregatorFunction, UserScoreAggregateFunction} import org.sunbird.dp.core.job.FlinkKafkaConnector import org.sunbird.dp.core.util.FlinkUtil @@ -48,15 +47,21 @@ class AssessmentAggregatorStreamTask(config: AssessmentAggregatorConfig, kafkaCo implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) val source = kafkaConnector.kafkaEventSource[Event](config.kafkaInputTopic) - val aggregatorStream: SingleOutputStreamOperator[Event] = - env.addSource(source, config.assessmentAggConsumer).uid(config.assessmentAggConsumer) - .rebalance() + val aggregatorStream = + env.addSource(source, config.assessmentAggConsumer) + .uid(config.assessmentAggConsumer).setParallelism(config.kafkaConsumerParallelism).rebalance() .process(new AssessmentAggregatorFunction(config)) .name(config.assessmentAggregatorFunction).uid(config.assessmentAggregatorFunction) .setParallelism(config.assessAggregatorParallelism) aggregatorStream.getSideOutput(config.failedEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaFailedTopic)) .name(config.assessFailedEventsSink).uid(config.assessFailedEventsSink) + .setParallelism(config.downstreamOperatorsParallelism) + aggregatorStream.getSideOutput(config.certIssueOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaCertIssueTopic)) + .name(config.certIssueEventSink).uid(config.certIssueEventSink) + .setParallelism(config.downstreamOperatorsParallelism) + aggregatorStream.getSideOutput(config.scoreAggregateTag).process(new UserScoreAggregateFunction(config)) + .name(config.userScoreAggregateFn).uid(config.userScoreAggregateFn).setParallelism(config.scoreAggregatorParallelism) env.execute(config.jobName) } } diff --git a/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf b/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf new file mode 100644 index 0000000000..290d24809a --- /dev/null +++ b/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf @@ -0,0 +1,40 @@ +include "base-test.conf" + +kafka { + input.topic = "flink.telemetry.assess" + groupId = "flink-assessment-aggregator-group" + failed.topic= "telemetry.assess.failed" + output.certissue.topic = "issue.certificate.request" +} + +task { + parallelism = 1 + checkpointing.interval = 60000 + assessaggregator { + parallelism = 1 + } + scoreaggregator { + parallelism = 1 + } + consumer.parallelism = 1 + downstream.parallelism = 1 +} + +lms-cassandra { + keyspace = "sunbird_courses" + table = "assessment_aggregator" + questionudttype= "question" + enrolmentstable = "user_enrolments" + activitytable = "user_activity_agg" +} + +redis { + database { + relationCache.id = 10 + contentCache.id = 5 + } +} + +assessment.skip.missingRecords = true +content.read.api = "http://dev.sunbirded.org/api/content/v1/read/" +user.activity.agg.type="attempt_metrics" \ No newline at end of file diff --git a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf index 243d024576..8e1b8bb5d1 100644 --- a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf +++ b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf @@ -4,6 +4,7 @@ kafka { input.topic = "flink.telemetry.assess" groupId = "flink-assessment-aggregator-group" failed.topic= "telemetry.assess.failed" + output.certissue.topic = "issue.certificate.request" } task { @@ -12,10 +13,27 @@ task { assessaggregator { parallelism = 1 } + scoreaggregator { + parallelism = 1 + } + consumer.parallelism = 1 + downstream.parallelism = 1 } lms-cassandra { keyspace = "sunbird_courses" table = "assessment_aggregator" questionudttype= "question" -} \ No newline at end of file + enrolmentstable = "user_enrolments" + activitytable = "user_activity_agg" +} + +redis { + database { + relationCache.id = 10 + contentCache.id = 5 + } +} +assessment.skip.missingRecords = false +content.read.api = "http://dev.sunbirded.org/api/content/v1/read/" +user.activity.agg.type="attempt_metrics" \ No newline at end of file diff --git a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.cql b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.cql index 99b0f0a92f..a401643c44 100644 --- a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.cql +++ b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.cql @@ -1,4 +1,4 @@ -CREATE KEYSPACE sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1}; +CREATE KEYSPACE IF NOT EXISTS sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1}; CREATE TYPE IF NOT EXISTS sunbird_courses.question ( @@ -16,11 +16,11 @@ duration decimal CREATE TABLE IF NOT EXISTS sunbird_courses.assessment_aggregator ( +user_id text, course_id text, batch_id text, content_id text, attempt_id text, -user_id text, created_on timestamp, last_attempted_on timestamp, total_max_score double, @@ -28,10 +28,10 @@ grand_total text, question list>, total_score double, updated_on timestamp, -PRIMARY KEY (course_id, batch_id, content_id, attempt_id, user_id) +PRIMARY KEY ((user_id, course_id), batch_id, content_id, attempt_id) ); -CREATE INDEX assessment_aggregator_last_attempted_on_idx ON sunbird_courses.assessment_aggregator (last_attempted_on); +CREATE INDEX IF NOT EXISTS assessment_aggregator_last_attempted_on_idx ON sunbird_courses.assessment_aggregator (last_attempted_on); /*INSERT INTO sunbird_courses.assessment_aggregator (course_id,batch_id,user_id,content_id,attempt_id,created_on,grand_total @@ -40,3 +40,37 @@ CREATE INDEX assessment_aggregator_last_attempted_on_idx ON sunbird_courses.asse '0afe2cc175ffc95b83a4dc8d1af2c47b','2019-11-07 05:38:49.640+0000','2/5','2019-11-07 05:38:05.076+0000',null, 5,2,'2019-11-07 05:38:49.642+0000');*/ +CREATE TABLE IF NOT EXISTS sunbird_courses.user_enrolments ( +userid text, +courseid text, +batchid text, +active boolean, +addedby text, +certificates list>>, +completedon timestamp, +completionpercentage int, +contentstatus map, +datetime timestamp, +enrolleddate text, +issued_certificates list>>, +lastreadcontentid text, +lastreadcontentstatus int, +progress int, +status int, +PRIMARY KEY (userid, courseid, batchid)); + +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, status) VALUES ('ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f', 'do_2128415652377067521125', '012846671379595264119', 2); +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, status) VALUES ('d0d8a341-9637-484c-b871-0c27015af238', 'do_2128410273679114241112', '01284169026368307244', 1); + + +CREATE TABLE IF NOT EXISTS sunbird_courses.user_activity_agg ( +activity_type text, +activity_id text, +user_id text, +context_id text, +agg map, +agg_last_updated map, +aggregates map, +agg_details list, +PRIMARY KEY ((activity_type, activity_id, user_id), context_id) +) WITH CLUSTERING ORDER BY (context_id ASC); \ No newline at end of file diff --git a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala index cf714de51d..a5e048e865 100644 --- a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala +++ b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala @@ -2,15 +2,42 @@ package org.sunbird.dp.fixture object EventFixture { - val BATCH_ASSESS_EVENT ="""{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" + val BATCH_ASSESS_EVENT = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" - val LATEST_BATCH_ASSESS_EVENT ="""{"assessmentTs":1587988642000,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":1,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8756","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362dr776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":1,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" + val SECOND_ATTEMPT_BATCH_ASSESS_EVENT = """{"assessmentTs":1568891729576,"attemptId":"9dd87e24df268ad09a8b0060c0a40262","batchId":"012846671379595264119","contentId":"do_212686723743318016173","courseId":"do_2128415652377067521125","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:cb00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"aqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEE","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521125","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:235815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"aqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEE","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521125","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:7ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"aqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEE","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521125","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:118f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"aqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEE","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521125","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1567073236195,"ver":"3.1","mid":"ASSESS:59d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n items ∑(χ - χ̅ )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}],"userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f"}""" - val BATCH_ASSESS__OLDER_EVENT: String = """{"courseId":"do_2128415652377067521125","batchId":"012846671379595264119","contentId":"do_212686723743318016173","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","assessmentTs":1567073236195,"attemptId":"8cd87e24df268ad09a8b0060c0a40271","events":[{"eid":"ASSESS","ets":1567687351000,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":10,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n items ∑(χ - χ̅ )="},"index":1,"pass":"Yes","score":5,"resvalues":[{"∑χ":"true"}],"duration":9}},{"eid":"ASSESS","ets":1567073236195,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n items ∑(χ - χ̅ )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}]}""" + // CourseId does not have this contentId do_4957349 in the leafnodes list + val BATCH_ASSESS_EVENT_WITHOUT_CACHE = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_4957349","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_4957349","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_4957349","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_4957349","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_4957349","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" - val BATCH_ASSESS_FAIL_EVENT: String = """{"courseId":"do_312712196780204032110117","batchId":"01271220181664563270","contentId":"505c7c48ac6dc1edc9b08f21db5a571d","userId":"b3541347e18ab916c06ed76aeb0ce57f","assessmentTs":1567073236195,"attemptId":"attempt1","events":[{"eid":"ASSESS","ets":1567687351000,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":10,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n items ∑(χ - χ̅ )="},"index":1,"pass":"Yes","score":5,"resvalues":[{"∑χ":"true"}],"duration":9}},{"eid":"ASSESS","ets":"1567073236195","ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n items ∑(χ - χ̅ )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}]}""" + val LATEST_BATCH_ASSESS_EVENT = """{"assessmentTs":1587988642000,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":1,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362dr776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":1,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" - val BATCH_DUPLICATE_QUESTION_EVENT: String = """{"assessmentTs":1569304757079,"batchId":"01284169026368307244","courseId":"do_2128410273679114241112","userId":"d0d8a341-9637-484c-b871-0c27015af238","attemptId":"90e1a0d12542806389a1a52aaf1fc622","contentId":"do_2128373396098744321673","events":[{"eid":"ASSESS","ets":1569304758743,"ver":"3.1","mid":"ASSESS:5b2e689446886f3cee13de44fec8c02f","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"1":"{\"text\":\"normal distribution\\n\"}"}],"duration":2}},{"eid":"ASSESS","ets":1569304761105,"ver":"3.1","mid":"ASSESS:0cba50019b880fc064a343a6d01d3d1a","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"binomial distribution\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1569304763323,"ver":"3.1","mid":"ASSESS:ace9f977fdb8254e709180777ff81ba6","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"4":"{\"text\":\"uniform distribution\\n\"}"}],"duration":6}},{"eid":"ASSESS","ets":1569304765515,"ver":"3.1","mid":"ASSESS:b4b19e35aed5b32cf240650aa09ec558","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"3":"{\"text\":\"Poisson distribution\\n\"}"}],"duration":8}},{"eid":"ASSESS","ets":1569304767576,"ver":"3.1","mid":"ASSESS:366e140cf5fddf850d1e548644a35729","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"759b8138-4c47-4724-8cd6-cae6a7d1ad18","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"descriptive statistic\\n\"}"},{"2":"{\"text\":\"probability function\\n\"}"},{"3":"{\"text\":\"variance\\n\"}"},{"4":"{\"text\":\"random variable\\n\"}"},{"answer":"{\"correct\":[\"4\"]}"}],"uri":"","title":"A numerical description of the outcome of an experiment is called a\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0,"resvalues":[{"3":"{\"text\":\"variance\\n\"}"}],"duration":1}},{"eid":"ASSESS","ets":1569304769433,"ver":"3.1","mid":"ASSESS:834b83c5205b657a208e552d5964a872","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"759b8138-4c47-4724-8cd6-cae6a7d1ad18","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"descriptive statistic\\n\"}"},{"2":"{\"text\":\"probability function\\n\"}"},{"3":"{\"text\":\"variance\\n\"}"},{"4":"{\"text\":\"random variable\\n\"}"},{"answer":"{\"correct\":[\"4\"]}"}],"uri":"","title":"A numerical description of the outcome of an experiment is called a\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"Yes","score":1,"resvalues":[{"4":"{\"text\":\"random variable\\n\"}"}],"duration":3}}]}""" + val BATCH_ASSESS__OLDER_EVENT: String = """{"courseId":"do_2128415652377067521125","batchId":"012846671379595264119","contentId":"do_212686723743318016173","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","assessmentTs":1567073236195,"attemptId":"8cd87e24df268ad09a8b0060c0a40271","events":[{"eid":"ASSESS","ets":1567687351000,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":10,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n items ∑(χ - χ̅ )="},"index":1,"pass":"Yes","score":5,"resvalues":[{"∑χ":"true"}],"duration":9}},{"eid":"ASSESS","ets":1567073236195,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n items ∑(χ - χ̅ )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}]}""" - val QUESTION_EVENT_RES_VALUES = """{"assessmentTs":1578663044164,"userId":"50a9e3fc-d047-4fa5-a37b-67501b8933db","contentId":"do_3129323935897108481169","courseId":"do_3129323995959541761169","batchId":"0129324118211215362","attemptId":"702ae8b81e37c94448d1fa117678d68c","events":[{"eid":"ASSESS","ets":1578663044164,"ver":"3.0","mid":"ASSESS:0b5c1d06920337df9d662b69d53f48bf","actor":{"id":"50a9e3fc-d047-4fa5-a37b-67501b8933db","type":"User"},"context":{"channel":"0126684405014528002","pdata":{"id":"prod.diksha.app","ver":"2.6.203","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"11059dcc-0a69-4d92-90b3-c1eb97f5f93b","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[{"id":"83f8c374dc0e4d91ec4c48d6e5d10b4c","type":"AttemptId"},{"id":"streaming","type":"PlayerLaunch"},{"id":"6a7809b4eccf75ae62b18c224a4cfeb5","type":"ContentSession"},{"id":"48e8e20e8c6fbc660f4ca317497fa7a6","type":"PlaySession"}],"rollup":{"l1":"0126684405014528002"}},"object":{"id":"do_3129323935897108481169","type":"Content","ver":"1","rollup":{"l1":"do_3129323995959541761169","l2":"do_31293239986759270411495"}},"tags":[],"edata":{"item":{"id":"QMCQ02115","maxscore":1,"exlength":0,"params":[{"item":{"keywords":["mdd"],"subject":"Mathematics","channel":"in.ekstep","language":["English"],"type":"ftb","editorState":null,"body":null,"question_audio":"","gradeLevel":["Class 4"],"appId":"prod.diksha.app","used_for":"worksheet","model":{"hintMsg":"Observe the key value and find the number of aircrafts which flies in the morning.","numericLangId":"en","langId":"en","variables":{"$aircraft1":"random(100,100)","$aircraft2":"random(70,70)","$aircraft3":"random(80,80)","$aircraft4":"random(30,30)"},"dropDowns":[{"identifier":1,"options":[{"text":"OPT_1_0","mh":null,"mmc":["DH277","DH278"],"answer":true,"image":null},{"text":"OPT_1_1","mh":"MH_1_1","mmc":["DH277","DH278"],"answer":false,"image":null},{"text":"OPT_1_2","mh":"MH_1_2","mmc":["DH277","DH278"],"answer":false,"image":null},{"text":"OPT_1_3","mh":"MH_1_3","mmc":["DH277","DH278"],"answer":false,"image":null}]},{"identifier":2,"options":[]},{"identifier":3,"options":[]},{"identifier":4,"options":[]}],"variablesProcessed":true},"state":"Verified","identifier":"QMCQ02115","level":2,"author":"funtoot","consumerId":"1762141d-8681-458b-9bd1-c6af98c0d989","solutions":null,"portalOwner":"562","version":1,"i18n":{"en":{"HINT":"Hint","MICROHINT":"Micro Hint","SOLUTION":"Solution","HELP":"Help","REDUCE_FRACTION":"Check the solution again. Remove the common factors in the fraction and reduce it to its lowest term.","MIXED_FRACTION":"Convert the mixed fraction to an improper fraction.","IMPROPER_FRACTION":"Convert the improper fraction to an mixed fraction.","LIKEFRACTIONADDITION_NUMERATOR":"Add the numerator of the given fractions correctly.","LIKEFRACTIONADDITION_DENOMINATOR":"For addition of like fractions the denominator remains the same.","LIKEFRACTIONADDITION_FULL":"Check the addition of the numerator. The denominator remains the same.","UNLIKEFRACTIONADDITION_NUMERATOR":"First convert the unlike fractions to like fractions. Then add the numerator of the like fractions.","UNLIKEFRACTIONADDITION_DENOMINATOR":"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.","UNLIKEFRACTIONADDITION_FULL":"Convert the given fractions to like fractions and then carefully do the addition.","LIKEFRACTIONSUBTRACTION_NUMERATOR":"Subtract the numerator of the given fractions correctly.","LIKEFRACTIONSUBTRACTION_DENOMINATOR":"For subtraction of like fractions the denominator remains the same as that of the question.","LIKEFRACTIONSUBTRACTION_FULL":"Subtract the numerator of the fractions. Denominator remains the same.","UNLIKEFRACTIONSUBTRACTION_NUMERATOR":"First convert the unlike fractions to like fractions. Then subtract the numerator of the like fractions.","UNLIKEFRACTIONSUBTRACTION_DENOMINATOR":"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.","UNLIKEFRACTIONSUBTRACTION_FULL":"Convert the given fractions to like fractions and then carefully do the subtraction.","LIKEMIXEDFRACTIONADDITION_WHOLE":"Add the whole number part of both the fractions carefully. Check for the conversion of the improper fraction to proper fractions in the solution.","LIKEMIXEDFRACTIONADDITION_NUMERATOR":"Add the numerator of the like fractions. If the fractional part is an improper fraction, then convert it to a proper fraction.","LIKEMIXEDFRACTIONADDITION_DENOMINATOR":"The denominator of the solution fraction does not change for like fractions.","LIKEMIXEDFRACTIONADDITION_FULL":"Convert the mixed fractions to improper fractions and then add them. The final solution should again be as a mixed fraction.","LIKEMIXEDFRACTIONSUBTRACTION_WHOLE":"Subtract the whole number part of both the fractions carefully.","LIKEMIXEDFRACTIONSUBTRACTION_NUMERATOR":"Subtract the numerator of the like fractions.","LIKEMIXEDFRACTIONSUBTRACTION_DENOMINATOR":"The denominator of the solution fraction does not change for like fractions.","LIKEMIXEDFRACTIONSUBTRACTION_FULL":"Convert the mixed fractions to improper fractions and then subtract them. The final solution should again be as a mixed fraction.","WHOLEANDPROPERFRACTIONADDITION_WHOLE":"The integer added becomes the whole number part of the mixed fraction.","WHOLEANDPROPERFRACTIONADDITION_NUMERATOR":"The numerator of the solution fraction remains the same. ","WHOLEANDPROPERFRACTIONADDITION_DENOMINATOR":"The denominator of the solution fraction remains the same as given in the problem.","WHOLEANDPROPERFRACTIONADDITION_FULL":"The integer and the proper fraction when added becomes the mixed fraction.","WHOLEANDPROPERFRACTIONSUBTRACTION_WHOLE":"Convert the whole number to an equivalent fraction. Then find the difference between the like fractions.","WHOLEANDPROPERFRACTIONSUBTRACTION_NUMERATOR":"First find the least common factor and convert them to difference between like fractions. Then subtract the numerators carefully.","WHOLEANDPROPERFRACTIONSUBTRACTION_DENOMINATOR":"The denominator of the solution fraction remains the same as given in the problem. ","WHOLEANDPROPERFRACTIONSUBTRACTION_FULL":"When the proper fraction is subtracted from the integer becomes the mixed fraction.","WHOLEANDFRACTIONMULTIPLICATION_NUMERATOR":"Multiply the numerator and the whole number carefully.","WHOLEANDFRACTIONMULTIPLICATION_DENOMINATOR":"Multiply the denominators of the fractions carefully. ","WHOLEANDFRACTIONMULTIPLICATION_FULL":"Multiply the integer with the fraction.","FRACTIONMULTIPLICATION_NUMERATOR":"Multiply the numerators of the fractions carefully.","FRACTIONMULTIPLICATION_DENOMINATOR":"Multiply the denominators of the fractions carefully. ","FRACTIONMULTIPLICATION_FULL":"Multiply the numerators and denominators of the fractions.","FRACTIONDIVISION":"First find the reciprocal of the divisor. Then multiply the fraction with the reciprocal of the divisor to get the solution.","SELECT":"Select","EXPRESSIONS":"$aircraft1=random(100,100)\n$aircraft2=random(70,70)\n$aircraft3=random(80,80)\n$aircraft4=random(30,30)","NO_HINT":"There is no hint for this question.","NO_ANSWER":"Please answer.","SOLUTION_ID":"a. The tallest bar shows the subject which is most liked by the students i.e., Social Studies. 40 students like Social Studiesb. There are 5 subjects mentioned in the graph, namely English, Mathematics, Hindi, Social Studies, and General Science.c. The shortest bar represents the subject which is least liked by the students i.e., Mathematics. Only 10 students like Mathematics.d. Number of students who like Hindi = 20Number of students who like Mathematics = 10We can clearly see that the number of students who like Hindi is double that of Mathematics. Hence the statement is true.","HINT_ID":"Observe the key value and find the number of aircrafts which flies in the morning.","QUESTION_TEXT":"The pictograph represents the number of aircrafts flying from India to foreign countries at different timings. Airlines decided to fly $aircraft1 aircrafts in the morning. How many aircrafts should they fly more? __1__","MH_1_3":"Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_3":"$aircraft1","MH_1_2":"According to the pictograph the number of aircrafts which flies in the afternoon is $aircraft3. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_2":"$aircraft3","MH_1_1":"According to the pictograph the number of aircrafts which flies in the morning is $aircraft2. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_1":"$aircraft2","OPT_1_0":"$aircraft4"}},"tags":["mdd"],"concepts":["C421"],"grade":["4"],"domain":"Numeracy","name":"QMCQ02115","bloomsTaxonomyLevel":"Understand","status":"Live","itemType":"UNIT","code":"QMCQ02115","qtype":"mdd","qlevel":"MEDIUM","flags":"{\"isZoomable\":true}","questionImage":"org.ekstep.funtoot.QMCQ02115.image.9074280266192281","media":[{"id":"org.ekstep.funtoot.QMCQ02115.image.9074280266192281","src":"https://ekstep-public-prod.s3-ap-south-1.amazonaws.com/content/org.ekstep.funtoot.qmcq02115.image.9886103695997586/artifact/org.ekstep.funtoot.qmcq02115.image.9886103695997586_1514092203316.png","type":"image"}],"title":"","qid":"QMCQ02115","createdOn":"2017-12-24T05:10:03.956+0000","qindex":1,"lastUpdatedOn":"2018-08-16T14:08:14.323+0000","subLevel":"","question":"QUESTION_TEXT","versionKey":"1534428494323","framework":"NCF","answer":{},"max_score":5,"sublevel":3,"template_id":"org.ekstep.funtoot.template.01","category":"MCQ","isSelected":true,"$$hashKey":"object:1540","template":"funtoot.template.01","maxAttempts":2,"curAttempt":0},"config":{"count":1,"selectedConfig":{},"title":"","type":"items","var":"item"}}],"uri":"","mmc":[],"mc":[null],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"random(30,30)"}],"duration":6}}]}""" + val BATCH_ASSESS_FAIL_EVENT: String = """{"courseId":"do_312712196780204032110117","batchId":"01271220181664563270","contentId":"505c7c48ac6dc1edc9b08f21db5a571d","userId":"b3541347e18ab916c06ed76aeb0ce57f","assessmentTs":1567073236195,"attemptId":"attempt1","events":[{"eid":"ASSESS","ets":1567687351000,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":10,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n items ∑(χ - χ̅ )="},"index":1,"pass":"Yes","score":5,"resvalues":[{"∑χ":"true"}],"duration":9}},{"eid":"ASSESS","ets":"1567073236195","ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n items ∑(χ - χ̅ )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}]}""" + + val BATCH_DUPLICATE_QUESTION_EVENT: String = """{"assessmentTs":1569304757079,"batchId":"01284169026368307244","courseId":"do_2128410273679114241112","userId":"d0d8a341-9637-484c-b871-0c27015af238","attemptId":"90e1a0d12542806389a1a52aaf1fc622","contentId":"do_2128373396098744321673","events":[{"eid":"ASSESS","ets":1569304758743,"ver":"3.1","mid":"ASSESS:5b2e689446886f3cee13de44fec8c02f","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"1":"{\"text\":\"normal distribution\\n\"}"}],"duration":2}},{"eid":"ASSESS","ets":1569304761105,"ver":"3.1","mid":"ASSESS:0cba50019b880fc064a343a6d01d3d1a","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"binomial distribution\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1569304763323,"ver":"3.1","mid":"ASSESS:ace9f977fdb8254e709180777ff81ba6","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"4":"{\"text\":\"uniform distribution\\n\"}"}],"duration":6}},{"eid":"ASSESS","ets":1569304765515,"ver":"3.1","mid":"ASSESS:b4b19e35aed5b32cf240650aa09ec558","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"3":"{\"text\":\"Poisson distribution\\n\"}"}],"duration":8}},{"eid":"ASSESS","ets":1569304767576,"ver":"3.1","mid":"ASSESS:366e140cf5fddf850d1e548644a35729","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"759b8138-4c47-4724-8cd6-cae6a7d1ad18","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"descriptive statistic\\n\"}"},{"2":"{\"text\":\"probability function\\n\"}"},{"3":"{\"text\":\"variance\\n\"}"},{"4":"{\"text\":\"random variable\\n\"}"},{"answer":"{\"correct\":[\"4\"]}"}],"uri":"","title":"A numerical description of the outcome of an experiment is called a\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0,"resvalues":[{"3":"{\"text\":\"variance\\n\"}"}],"duration":1}},{"eid":"ASSESS","ets":1569304769433,"ver":"3.1","mid":"ASSESS:834b83c5205b657a208e552d5964a872","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"759b8138-4c47-4724-8cd6-cae6a7d1ad18","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"descriptive statistic\\n\"}"},{"2":"{\"text\":\"probability function\\n\"}"},{"3":"{\"text\":\"variance\\n\"}"},{"4":"{\"text\":\"random variable\\n\"}"},{"answer":"{\"correct\":[\"4\"]}"}],"uri":"","title":"A numerical description of the outcome of an experiment is called a\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"Yes","score":1,"resvalues":[{"4":"{\"text\":\"random variable\\n\"}"}],"duration":3}}]}""" + + val QUESTION_EVENT_RES_VALUES = """{"assessmentTs":1578663044164,"userId":"50a9e3fc-d047-4fa5-a37b-67501b8933db","contentId":"do_3129323935897108481169","courseId":"do_3129323995959541761169","batchId":"0129324118211215362","attemptId":"702ae8b81e37c94448d1fa117678d68c","events":[{"eid":"ASSESS","ets":1578663044164,"ver":"3.0","mid":"ASSESS:0b5c1d06920337df9d662b69d53f48bf","actor":{"id":"50a9e3fc-d047-4fa5-a37b-67501b8933db","type":"User"},"context":{"channel":"0126684405014528002","pdata":{"id":"prod.diksha.app","ver":"2.6.203","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"11059dcc-0a69-4d92-90b3-c1eb97f5f93b","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[{"id":"83f8c374dc0e4d91ec4c48d6e5d10b4c","type":"AttemptId"},{"id":"streaming","type":"PlayerLaunch"},{"id":"6a7809b4eccf75ae62b18c224a4cfeb5","type":"ContentSession"},{"id":"48e8e20e8c6fbc660f4ca317497fa7a6","type":"PlaySession"}],"rollup":{"l1":"0126684405014528002"}},"object":{"id":"do_3129323935897108481169","type":"Content","ver":"1","rollup":{"l1":"do_3129323995959541761169","l2":"do_31293239986759270411495"}},"tags":[],"edata":{"item":{"id":"QMCQ02115","maxscore":1,"exlength":0,"params":[{"item":{"keywords":["mdd"],"subject":"Mathematics","channel":"in.ekstep","language":["English"],"type":"ftb","editorState":null,"body":null,"question_audio":"","gradeLevel":["Class 4"],"appId":"prod.diksha.app","used_for":"worksheet","model":{"hintMsg":"Observe the key value and find the number of aircrafts which flies in the morning.","numericLangId":"en","langId":"en","variables":{"$aircraft1":"random(100,100)","$aircraft2":"random(70,70)","$aircraft3":"random(80,80)","$aircraft4":"random(30,30)"},"dropDowns":[{"identifier":1,"options":[{"text":"OPT_1_0","mh":null,"mmc":["DH277","DH278"],"answer":true,"image":null},{"text":"OPT_1_1","mh":"MH_1_1","mmc":["DH277","DH278"],"answer":false,"image":null},{"text":"OPT_1_2","mh":"MH_1_2","mmc":["DH277","DH278"],"answer":false,"image":null},{"text":"OPT_1_3","mh":"MH_1_3","mmc":["DH277","DH278"],"answer":false,"image":null}]},{"identifier":2,"options":[]},{"identifier":3,"options":[]},{"identifier":4,"options":[]}],"variablesProcessed":true},"state":"Verified","identifier":"QMCQ02115","level":2,"author":"funtoot","consumerId":"1762141d-8681-458b-9bd1-c6af98c0d989","solutions":null,"portalOwner":"562","version":1,"i18n":{"en":{"HINT":"Hint","MICROHINT":"Micro Hint","SOLUTION":"Solution","HELP":"Help","REDUCE_FRACTION":"Check the solution again. Remove the common factors in the fraction and reduce it to its lowest term.","MIXED_FRACTION":"Convert the mixed fraction to an improper fraction.","IMPROPER_FRACTION":"Convert the improper fraction to an mixed fraction.","LIKEFRACTIONADDITION_NUMERATOR":"Add the numerator of the given fractions correctly.","LIKEFRACTIONADDITION_DENOMINATOR":"For addition of like fractions the denominator remains the same.","LIKEFRACTIONADDITION_FULL":"Check the addition of the numerator. The denominator remains the same.","UNLIKEFRACTIONADDITION_NUMERATOR":"First convert the unlike fractions to like fractions. Then add the numerator of the like fractions.","UNLIKEFRACTIONADDITION_DENOMINATOR":"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.","UNLIKEFRACTIONADDITION_FULL":"Convert the given fractions to like fractions and then carefully do the addition.","LIKEFRACTIONSUBTRACTION_NUMERATOR":"Subtract the numerator of the given fractions correctly.","LIKEFRACTIONSUBTRACTION_DENOMINATOR":"For subtraction of like fractions the denominator remains the same as that of the question.","LIKEFRACTIONSUBTRACTION_FULL":"Subtract the numerator of the fractions. Denominator remains the same.","UNLIKEFRACTIONSUBTRACTION_NUMERATOR":"First convert the unlike fractions to like fractions. Then subtract the numerator of the like fractions.","UNLIKEFRACTIONSUBTRACTION_DENOMINATOR":"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.","UNLIKEFRACTIONSUBTRACTION_FULL":"Convert the given fractions to like fractions and then carefully do the subtraction.","LIKEMIXEDFRACTIONADDITION_WHOLE":"Add the whole number part of both the fractions carefully. Check for the conversion of the improper fraction to proper fractions in the solution.","LIKEMIXEDFRACTIONADDITION_NUMERATOR":"Add the numerator of the like fractions. If the fractional part is an improper fraction, then convert it to a proper fraction.","LIKEMIXEDFRACTIONADDITION_DENOMINATOR":"The denominator of the solution fraction does not change for like fractions.","LIKEMIXEDFRACTIONADDITION_FULL":"Convert the mixed fractions to improper fractions and then add them. The final solution should again be as a mixed fraction.","LIKEMIXEDFRACTIONSUBTRACTION_WHOLE":"Subtract the whole number part of both the fractions carefully.","LIKEMIXEDFRACTIONSUBTRACTION_NUMERATOR":"Subtract the numerator of the like fractions.","LIKEMIXEDFRACTIONSUBTRACTION_DENOMINATOR":"The denominator of the solution fraction does not change for like fractions.","LIKEMIXEDFRACTIONSUBTRACTION_FULL":"Convert the mixed fractions to improper fractions and then subtract them. The final solution should again be as a mixed fraction.","WHOLEANDPROPERFRACTIONADDITION_WHOLE":"The integer added becomes the whole number part of the mixed fraction.","WHOLEANDPROPERFRACTIONADDITION_NUMERATOR":"The numerator of the solution fraction remains the same. ","WHOLEANDPROPERFRACTIONADDITION_DENOMINATOR":"The denominator of the solution fraction remains the same as given in the problem.","WHOLEANDPROPERFRACTIONADDITION_FULL":"The integer and the proper fraction when added becomes the mixed fraction.","WHOLEANDPROPERFRACTIONSUBTRACTION_WHOLE":"Convert the whole number to an equivalent fraction. Then find the difference between the like fractions.","WHOLEANDPROPERFRACTIONSUBTRACTION_NUMERATOR":"First find the least common factor and convert them to difference between like fractions. Then subtract the numerators carefully.","WHOLEANDPROPERFRACTIONSUBTRACTION_DENOMINATOR":"The denominator of the solution fraction remains the same as given in the problem. ","WHOLEANDPROPERFRACTIONSUBTRACTION_FULL":"When the proper fraction is subtracted from the integer becomes the mixed fraction.","WHOLEANDFRACTIONMULTIPLICATION_NUMERATOR":"Multiply the numerator and the whole number carefully.","WHOLEANDFRACTIONMULTIPLICATION_DENOMINATOR":"Multiply the denominators of the fractions carefully. ","WHOLEANDFRACTIONMULTIPLICATION_FULL":"Multiply the integer with the fraction.","FRACTIONMULTIPLICATION_NUMERATOR":"Multiply the numerators of the fractions carefully.","FRACTIONMULTIPLICATION_DENOMINATOR":"Multiply the denominators of the fractions carefully. ","FRACTIONMULTIPLICATION_FULL":"Multiply the numerators and denominators of the fractions.","FRACTIONDIVISION":"First find the reciprocal of the divisor. Then multiply the fraction with the reciprocal of the divisor to get the solution.","SELECT":"Select","EXPRESSIONS":"$aircraft1=random(100,100)\n$aircraft2=random(70,70)\n$aircraft3=random(80,80)\n$aircraft4=random(30,30)","NO_HINT":"There is no hint for this question.","NO_ANSWER":"Please answer.","SOLUTION_ID":"a. The tallest bar shows the subject which is most liked by the students i.e., Social Studies. 40 students like Social Studiesb. There are 5 subjects mentioned in the graph, namely English, Mathematics, Hindi, Social Studies, and General Science.c. The shortest bar represents the subject which is least liked by the students i.e., Mathematics. Only 10 students like Mathematics.d. Number of students who like Hindi = 20Number of students who like Mathematics = 10We can clearly see that the number of students who like Hindi is double that of Mathematics. Hence the statement is true.","HINT_ID":"Observe the key value and find the number of aircrafts which flies in the morning.","QUESTION_TEXT":"The pictograph represents the number of aircrafts flying from India to foreign countries at different timings. Airlines decided to fly $aircraft1 aircrafts in the morning. How many aircrafts should they fly more? __1__","MH_1_3":"Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_3":"$aircraft1","MH_1_2":"According to the pictograph the number of aircrafts which flies in the afternoon is $aircraft3. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_2":"$aircraft3","MH_1_1":"According to the pictograph the number of aircrafts which flies in the morning is $aircraft2. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_1":"$aircraft2","OPT_1_0":"$aircraft4"}},"tags":["mdd"],"concepts":["C421"],"grade":["4"],"domain":"Numeracy","name":"QMCQ02115","bloomsTaxonomyLevel":"Understand","status":"Live","itemType":"UNIT","code":"QMCQ02115","qtype":"mdd","qlevel":"MEDIUM","flags":"{\"isZoomable\":true}","questionImage":"org.ekstep.funtoot.QMCQ02115.image.9074280266192281","media":[{"id":"org.ekstep.funtoot.QMCQ02115.image.9074280266192281","src":"https://ekstep-public-prod.s3-ap-south-1.amazonaws.com/content/org.ekstep.funtoot.qmcq02115.image.9886103695997586/artifact/org.ekstep.funtoot.qmcq02115.image.9886103695997586_1514092203316.png","type":"image"}],"title":"","qid":"QMCQ02115","createdOn":"2017-12-24T05:10:03.956+0000","qindex":1,"lastUpdatedOn":"2018-08-16T14:08:14.323+0000","subLevel":"","question":"QUESTION_TEXT","versionKey":"1534428494323","framework":"NCF","answer":{},"max_score":5,"sublevel":3,"template_id":"org.ekstep.funtoot.template.01","category":"MCQ","isSelected":true,"$$hashKey":"object:1540","template":"funtoot.template.01","maxAttempts":2,"curAttempt":0},"config":{"count":1,"selectedConfig":{},"title":"","type":"items","var":"item"}}],"uri":"","mmc":[],"mc":[null],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"random(30,30)"}],"duration":6}}]}""" + + // Cache Does not have leafnodes for this course // is_valid content true + val INVALID_CONTENT_ID_EVENT= """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125-invalid","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173-invalid","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" + + //Recompute aggregate event + val RECOMPUTE_ASSESS_EVENT = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","contentId":"do_212686723743318016173"}""" + + val courseLeafNodes_1 = Map("do_2128415652377067521125:do_2128415652377067521125:leafnodes" -> "do_212686723743318016173") + val courseLeafNodes_2 = Map("do_312712196780204032110117:do_312712196780204032110117:leafnodes" -> "505c7c48ac6dc1edc9b08f21db5a571d") + val courseLeafNodes_3 = Map("do_2128410273679114241112:do_2128410273679114241112:leafnodes" -> "do_2128373396098744321673") + val courseLeafNodes_4 = Map("do_3129323995959541761169:do_3129323995959541761169:leafnodes" -> "do_3129323935897108481169") + val courseLeafNodes_5 = Map("do_873264782364827482:do_873264782364827482:leafnodes" -> "do_313026415363981312122") + val courseLeafNodes_6 = Map("do_87326478236482748244:do_87326478236482748244:leafnodes" -> "do_1131998128479272961991") + val leafNodesList = List(courseLeafNodes_1, courseLeafNodes_2, courseLeafNodes_3, courseLeafNodes_4, courseLeafNodes_6) + + val DUPLICATE_BATCH_ASSESS_EVENTS_1 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_873264782364827482","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_313026415363981312122","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" + val DUPLICATE_BATCH_ASSESS_EVENTS_2 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_873264782364827482","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_11307972307046400011917","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776_1","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776_2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776_3","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" + val DUPLICATE_BATCH_ASSESS_EVENTS_3 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_873264782364827482","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_313026415363981312122123","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" + val DUPLICATE_BATCH_ASSESS_EVENTS_4 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_87326478236482748244","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_1131998128479272961991","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}""" + val contentCacheData_1 = Map("do_313026415363981312122" -> "{\n \"ownershipType\": [\n \"createdBy\"\n ],\n \"copyright\": \"EKSTEP\",\n \"previewUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_313026415363981312122-latest\",\n \"plugins\": [\n {\n \"identifier\": \"org.ekstep.stage\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.questionset\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.navigation\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.reorder\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.sequence\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionset.quiz\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.iterator\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.keyboard\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.ftb\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.mtf\",\n \"semanticVersion\": \"1.2\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.mcq\",\n \"semanticVersion\": \"1.3\"\n },\n {\n \"identifier\": \"org.ekstep.summary\",\n \"semanticVersion\": \"1.0\"\n }\n ],\n \"subject\": [\n \"CBSE Training\"\n ],\n \"channel\": \"0123221758376673287017\",\n \"downloadUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_313026415363981312122/vjjjj_1590142788385_do_313026415363981312122_1.0.ecar\",\n \"questions\": [\n {\n \"name\": \"Arrange the Given sentence \\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026448412631040111\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"Test data mark is 10\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026416457089024115\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"Arrange the given words in proper sentence\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026446885158912120\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"'The tree ____ is ____ and ____ mark is 3\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026423737737216117\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"Match the following\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_31302641810772787218\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n }\n ],\n \"organisation\": [\n \"EKSTEP\"\n ],\n \"language\": [\n \"English\"\n ],\n \"mimeType\": \"application/vnd.ekstep.ecml-archive\",\n \"variants\": {\n \"spine\": {\n \"ecarUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_313026415363981312122/vjjjj_1590142788484_do_313026415363981312122_1.0_spine.ecar\",\n \"size\": 44094\n }\n },\n \"editorState\": \"{\\\"plugin\\\":{\\\"noOfExtPlugins\\\":11,\\\"extPlugins\\\":[{\\\"plugin\\\":\\\"org.ekstep.contenteditorfunctions\\\",\\\"version\\\":\\\"1.2\\\"},{\\\"plugin\\\":\\\"org.ekstep.keyboardshortcuts\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.richtext\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.iterator\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.navigation\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.reviewercomments\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.mtf\\\",\\\"version\\\":\\\"1.2\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.3\\\"},{\\\"plugin\\\":\\\"org.ekstep.keyboard\\\",\\\"version\\\":\\\"1.1\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.reorder\\\",\\\"version\\\":\\\"1.1\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.sequence\\\",\\\"version\\\":\\\"1.1\\\"}]},\\\"stage\\\":{\\\"noOfStages\\\":1,\\\"currentStage\\\":\\\"371a0b37-e8ab-4e8e-b83e-3775a2ed927d\\\",\\\"selectedPluginObject\\\":\\\"49469445-2ca2-479d-95b7-360aa07ee3bd\\\"},\\\"sidebar\\\":{\\\"selectedMenu\\\":\\\"settings\\\"}}\",\n \"objectType\": \"Content\",\n \"gradeLevel\": [\n \"Class 1\"\n ],\n \"appIcon\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_313026415363981312122/artifact/10_1560927487910.thumb.jpg\",\n \"primaryCategory\": \"Course Assessment\",\n \"collections\": [\n {\n \"name\": \"Copy of SelfServiceable\",\n \"relation\": \"hasSequenceMember\",\n \"identifier\": \"do_313026450799894528150\",\n \"description\": \"Execution\",\n \"objectType\": \"Collection\",\n \"status\": \"Retired\"\n }\n ],\n \"appId\": \"prod.diksha.portal\",\n \"contentEncoding\": \"gzip\",\n \"artifactUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_313026415363981312122/artifact/1590142788151_do_313026415363981312122.zip\",\n \"lockKey\": \"0708a132-4774-4126-b2e5-df0e83f6dd40\",\n \"sYS_INTERNAL_LAST_UPDATED_ON\": \"2020-05-22T10:19:50.208+0000\",\n \"contentType\": \"SelfAssess\",\n \"lastUpdatedBy\": \"2dded955-5656-4dc2-a683-e7e42082aa8c\",\n \"identifier\": \"do_313026415363981312122\",\n \"audience\": [\n \"Teacher\"\n ],\n \"visibility\": \"Default\",\n \"consumerId\": \"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\",\n \"mediaType\": \"content\",\n \"osId\": \"org.ekstep.quiz.app\",\n \"lastPublishedBy\": \"5c627fa2-a7a4-490f-b9c8-bc0f42559562\",\n \"languageCode\": [\n \"en\"\n ],\n \"version\": 2,\n \"license\": \"CC BY 4.0\",\n \"prevState\": \"Review\",\n \"size\": 547518,\n \"lastPublishedOn\": \"2020-05-22T10:19:48.378+0000\",\n \"name\": \"Vjjjj\",\n \"status\": \"Retired\",\n \"totalQuestions\": 2,\n \"code\": \"org.sunbird.lfSsbq\",\n \"prevStatus\": \"Processing\",\n \"description\": \"Enter description for Assessment\",\n \"streamingUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_313026415363981312122-latest\",\n \"medium\": [\n \"English\"\n ],\n \"posterImage\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_3127871179775098881294/artifact/10_1560927487910.jpg\",\n \"idealScreenSize\": \"normal\",\n \"createdOn\": \"2020-05-22T09:09:54.237+0000\",\n \"copyrightYear\": 2020,\n \"contentDisposition\": \"inline\",\n \"licenseterms\": \"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.\",\n \"lastUpdatedOn\": \"2020-05-22T11:59:41.479+0000\",\n \"dialcodeRequired\": \"No\",\n \"lastStatusChangedOn\": \"2020-05-22T11:59:41.479+0000\",\n \"createdFor\": [\n \"0123221758376673287017\"\n ],\n \"creator\": \"content_creator sahu\",\n \"os\": [\n \"All\"\n ],\n \"totalScore\": 5,\n \"pkgVersion\": 1,\n \"versionKey\": \"1590142785805\",\n \"idealScreenDensity\": \"hdpi\",\n \"framework\": \"ekstep_ncert_k-12\",\n \"s3Key\": \"ecar_files/do_313026415363981312122/vjjjj_1590142788385_do_313026415363981312122_1.0.ecar\",\n \"lastSubmittedOn\": \"2020-05-22T10:19:07.606+0000\",\n \"createdBy\": \"2dded955-5656-4dc2-a683-e7e42082aa8c\",\n \"compatibilityLevel\": 2,\n \"board\": \"CBSE\"\n }") + val contentCacheData_2 = Map("do_313026415363981312122123" -> "{\"totalQuestions\":1}") + val contentCacheList = List(contentCacheData_1, contentCacheData_2) } diff --git a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala index 5e25f62abd..171f7e4f71 100644 --- a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala +++ b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala @@ -1,9 +1,10 @@ package org.sunbird.dp.spec -import java.util +import com.datastax.driver.core.Row +import java.util +import com.google.common.reflect.TypeToken import com.google.gson.Gson -import com.google.gson.internal.LinkedTreeMap import com.typesafe.config.{Config, ConfigFactory} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.TypeExtractor @@ -18,125 +19,250 @@ import org.cassandraunit.utils.EmbeddedCassandraServerHelper import org.mockito.Mockito import org.mockito.Mockito._ import org.sunbird.dp.assessment.domain.Event +import org.sunbird.dp.assessment.functions.{AggDetails, UserActivityAgg} import org.sunbird.dp.assessment.task.{AssessmentAggregatorConfig, AssessmentAggregatorStreamTask} +import org.sunbird.dp.core.cache.RedisConnect import org.sunbird.dp.core.job.FlinkKafkaConnector -import org.sunbird.dp.core.util.CassandraUtil +import org.sunbird.dp.core.util.{CassandraUtil, JSONUtil} import org.sunbird.dp.fixture.EventFixture import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} -import org.junit.Assert.{assertEquals, assertNotNull} +import redis.embedded.RedisServer -import collection.JavaConverters._ class AssessmentAggregatorTaskTestSpec extends BaseTestSpec { - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) + implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) + val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() + .setConfiguration(testConfiguration()) + .setNumberSlotsPerTaskManager(1) + .setNumberTaskManagers(1) + .build) + var redisServer: RedisServer = _ + val config: Config = ConfigFactory.load("test.conf") + val assessmentConfig: AssessmentAggregatorConfig = new AssessmentAggregatorConfig(config) + val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) + val gson = new Gson() - val config: Config = ConfigFactory.load("test.conf") - val assessmentConfig: AssessmentAggregatorConfig = new AssessmentAggregatorConfig(config) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val gson = new Gson() + var cassandraUtil: CassandraUtil = _ - var cassandraUtil: CassandraUtil = _ + override protected def beforeAll(): Unit = { + super.beforeAll() + redisServer = new RedisServer(6340) + redisServer.start() + EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) + cassandraUtil = new CassandraUtil(assessmentConfig.dbHost, assessmentConfig.dbPort) + val session = cassandraUtil.session + setupRedisTestData() - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(assessmentConfig.dbHost, assessmentConfig.dbPort) - val session = cassandraUtil.session + val dataLoader = new CQLDataLoader(session) + dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)); + // Clear the metrics + testCassandraUtil(cassandraUtil) + BaseMetricsReporter.gaugeMetrics.clear() + flinkCluster.before() + } - val dataLoader = new CQLDataLoader(session); - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)); - // Clear the metrics - testCassandraUtil(cassandraUtil) - BaseMetricsReporter.gaugeMetrics.clear() + override protected def afterAll(): Unit = { + super.afterAll() + redisServer.stop() + try { + EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() + } catch { + case ex: Exception => { - flinkCluster.before() + } } + flinkCluster.after() + } - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - }catch { - case ex: Exception =>{ - } - } - flinkCluster.after() - } + "AssessmentAggregator " should "Update event to db" in { + when(mockKafkaUtil.kafkaEventSource[Event](assessmentConfig.kafkaInputTopic)).thenReturn(new AssessmentAggreagatorEventSource) + when(mockKafkaUtil.kafkaEventSink[Event](assessmentConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink) + when(mockKafkaUtil.kafkaStringSink(assessmentConfig.kafkaCertIssueTopic)).thenReturn(new certificateIssuedEventsSink) + val task = new AssessmentAggregatorStreamTask(assessmentConfig, mockKafkaUtil) + task.process() + assert(FailedEventsSink.values.get(0).getTelemetry.getMap.containsKey("metadata")) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.skippedEventCount}").getValue() should be(1) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbReadCount}").getValue() should be(2) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbUpdateCount}").getValue() should be(6) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.failedEventCount}").getValue() should be(2) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.batchSuccessCount}").getValue() should be(6) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.cacheHitCount}").getValue() should be(8) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.cacheHitMissCount}").getValue() should be(1) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.certIssueEventsCount}").getValue() should be(6) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbScoreAggUpdateCount}").getValue() should be(6) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbScoreAggReadCount}").getValue() should be(6) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.recomputeAggEventCount}").getValue() should be(0) + val test_row1 = cassandraUtil.findOne("select total_score,total_max_score from sunbird_courses.assessment_aggregator where user_id='d0d8a341-9637-484c-b871-0c27015af238' and course_id='do_2128410273679114241112'") + assert(test_row1.getDouble("total_score") == 2.0) + assert(test_row1.getDouble("total_max_score") == 2.0) + val test_row2: java.util.List[Row] = cassandraUtil.find("select attempt_id,total_score,total_max_score from sunbird_courses.assessment_aggregator where user_id='ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f' and course_id='do_2128415652377067521125'") - "AssessmentAggregator " should "Update event to db" in { - when(mockKafkaUtil.kafkaEventSource[Event](assessmentConfig.kafkaInputTopic)).thenReturn(new AssessmentAggreagatorEventSource) - when(mockKafkaUtil.kafkaEventSink[Event](assessmentConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink) - val task = new AssessmentAggregatorStreamTask(assessmentConfig, mockKafkaUtil) - task.process() - val failedEvent = gson.fromJson(gson.toJson(FailedEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala - assert(failedEvent.get("map").get.asInstanceOf[LinkedTreeMap[String, AnyRef]].containsKey("metadata")) - BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.skippedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbReadCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbUpdateCount}").getValue() should be(4) - BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.failedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.batchSuccessCount}").getValue() should be(4) - val test_row1 = cassandraUtil.findOne("select total_score,total_max_score from sunbird_courses.assessment_aggregator where course_id='do_2128410273679114241112'") - assert(test_row1.getDouble("total_score") == 2.0) - assert(test_row1.getDouble("total_max_score") == 2.0) - - val test_row2 = cassandraUtil.findOne("select total_score,total_max_score from sunbird_courses.assessment_aggregator where course_id='do_2128415652377067521125'") - assert(test_row2.getDouble("total_score") == 3.0) - assert(test_row2.getDouble("total_max_score") == 4.0) - } + assert(test_row2.get(0).getString("attempt_id") == "8cd87e24df268ad09a8b0060c0a40271") + assert(test_row2.get(0).getDouble("total_score") == 2.0) + assert(test_row2.get(0).getDouble("total_max_score") == 3.0) - def testCassandraUtil(cassandraUtil: CassandraUtil): Unit ={ - cassandraUtil.reconnect() - val response = cassandraUtil.find("SELECT * FROM sunbird_courses.assessment_aggregator;") - response should not be(null) - } + assert(test_row2.get(1).getString("attempt_id") == "9dd87e24df268ad09a8b0060c0a40262") + assert(test_row2.get(1).getDouble("total_score") == 1.33) + assert(test_row2.get(1).getDouble("total_max_score") == 3.0) + + val test_row3 = cassandraUtil.findOne("select aggregates from sunbird_courses.user_activity_agg where activity_type='Course' and activity_id='do_2128410273679114241112' and user_id='d0d8a341-9637-484c-b871-0c27015af238'") + val resultMap3 = test_row3.getMap("aggregates", new TypeToken[String]() {}, new TypeToken[java.lang.Double]() {}) + assert(null != resultMap3) + assert(2 == resultMap3.getOrDefault("score:do_2128373396098744321673", 0)) + assert(2 == resultMap3.getOrDefault("max_score:do_2128373396098744321673", 0)) + assert(1 == resultMap3.getOrDefault("attempts_count:do_2128373396098744321673", 0)) + + val test_row6 = cassandraUtil.findOne("select agg_details from sunbird_courses.user_activity_agg where activity_type='Course' and activity_id='do_2128415652377067521125' and user_id='ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f'") + val aggDetail1 = new Gson().fromJson(test_row6.getList("agg_details", new TypeToken[String](){}).get(0), classOf[AggDetails]) + assert(null != aggDetail1) + assert("8cd87e24df268ad09a8b0060c0a40271" == aggDetail1.attempt_id) + assert("do_212686723743318016173" == aggDetail1.content_id) + assert(2 == aggDetail1.score) + assert(3 == aggDetail1.max_score) + assert("attempt_metrics" == aggDetail1.`type`) + + val aggDetail2 = new Gson().fromJson(test_row6.getList("agg_details", new TypeToken[String](){}).get(1), classOf[AggDetails]) + assert(null != aggDetail2) + assert("9dd87e24df268ad09a8b0060c0a40262" == aggDetail2.attempt_id) + assert("do_212686723743318016173" == aggDetail2.content_id) + assert(1.33 == aggDetail2.score) + assert(3 == aggDetail2.max_score) + assert("attempt_metrics" == aggDetail2.`type`) + + val test_row4 = cassandraUtil.findOne("select aggregates from sunbird_courses.user_activity_agg where activity_type='Course' and activity_id='do_2128415652377067521125' and user_id='ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f'") + val resultMap4 = test_row4.getMap("aggregates", new TypeToken[String]() {}, new TypeToken[java.lang.Double]() {}) + assert(null != resultMap4) + assert(2 == resultMap4.getOrDefault("score:do_212686723743318016173", 0)) + assert(3 == resultMap4.getOrDefault("max_score:do_212686723743318016173", 0)) + assert(2 == resultMap4.getOrDefault("attempts_count:do_212686723743318016173", 0)) + + val test_row5 = cassandraUtil.findOne("select aggregates from sunbird_courses.user_activity_agg where activity_type='Course' and activity_id='do_3129323995959541761169' and user_id='50a9e3fc-d047-4fa5-a37b-67501b8933db'") + val resultMap5 = test_row5.getMap("aggregates", new TypeToken[String]() {}, new TypeToken[java.lang.Double]() {}) + assert(null != resultMap5) + assert(1 == resultMap5.getOrDefault("score:do_3129323935897108481169", 0)) + assert(1 == resultMap5.getOrDefault("max_score:do_3129323935897108481169", 0)) + assert(1 == resultMap5.getOrDefault("attempts_count:do_3129323935897108481169", 0)) + } + + "AssessmentAggregator " should "Skip the missing records from the event" in { + val forceValidationAssessmentConfig: AssessmentAggregatorConfig = new AssessmentAggregatorConfig(ConfigFactory.load("forcevalidate.conf")) + when(mockKafkaUtil.kafkaEventSource[Event](forceValidationAssessmentConfig.kafkaInputTopic)).thenReturn(new AssessmentAggreagatorEventSourceForceValidation) + when(mockKafkaUtil.kafkaEventSink[Event](forceValidationAssessmentConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink) + when(mockKafkaUtil.kafkaStringSink(forceValidationAssessmentConfig.kafkaCertIssueTopic)).thenReturn(new certificateIssuedEventsSink) + val task = new AssessmentAggregatorStreamTask(forceValidationAssessmentConfig, mockKafkaUtil) + task.process() + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.batchSuccessCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.cacheHitCount}").getValue() should be(5) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.apiHitSuccessCount}").getValue() should be(2) + BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.ignoredEventsCount}").getValue() should be(1) + } + + def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { + cassandraUtil.reconnect() + val response = cassandraUtil.find("SELECT * FROM sunbird_courses.assessment_aggregator;") + response should not be (null) + } + + def setupRedisTestData() { + val redisConnect = new RedisConnect(assessmentConfig.metaRedisHost, assessmentConfig.metaRedisPort, assessmentConfig) + val jedis = redisConnect.getConnection(assessmentConfig.relationCacheNode) + EventFixture.leafNodesList.map(nodes => { + nodes.map(node => { + jedis.sadd(node._1, node._2) + }) + }) + + // Setup content Cache + val contentCache = redisConnect.getConnection(assessmentConfig.contentCacheNode) + EventFixture.contentCacheList.map(nodes => { + nodes.map(node => { + contentCache.set(node._1, node._2) + }) + }) + } } + + class AssessmentAggreagatorEventSource extends SourceFunction[Event] { - override def run(ctx: SourceContext[Event]) { - val gson = new Gson() - - val eventMap1 = gson.fromJson(EventFixture.BATCH_ASSESS_EVENT, new util.LinkedHashMap[String, Any]().getClass) - val eventMap2 = gson.fromJson(EventFixture.BATCH_ASSESS__OLDER_EVENT, new util.LinkedHashMap[String, Any]().getClass) - val eventMap3 = gson.fromJson(EventFixture.BATCH_ASSESS_FAIL_EVENT, new util.LinkedHashMap[String, Any]().getClass) - val eventMap4 = gson.fromJson(EventFixture.QUESTION_EVENT_RES_VALUES, new util.LinkedHashMap[String, Any]().getClass) - val eventMap5 = gson.fromJson(EventFixture.LATEST_BATCH_ASSESS_EVENT, new util.LinkedHashMap[String, Any]().getClass) - val eventMap6 = gson.fromJson(EventFixture.BATCH_DUPLICATE_QUESTION_EVENT, new util.LinkedHashMap[String, Any]().getClass) - ctx.collect(new Event(eventMap1)) - ctx.collect(new Event(eventMap2)) - ctx.collect(new Event(eventMap3)) - ctx.collect(new Event(eventMap4)) - ctx.collect(new Event(eventMap5)) - ctx.collect(new Event(eventMap6)) - } + override def run(ctx: SourceContext[Event]) { + val gson = new Gson() + + //val eventMap1 = gson.fromJson(EventFixture.BATCH_ASSESS_EVENT, new util.LinkedHashMap[String, Any]().getClass) + + val eventMap1 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_ASSESS_EVENT) + val eventMap2 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_ASSESS__OLDER_EVENT) + val eventMap3 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_ASSESS_FAIL_EVENT) + val eventMap4 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.QUESTION_EVENT_RES_VALUES) + val eventMap5 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.LATEST_BATCH_ASSESS_EVENT) + val eventMap6 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_DUPLICATE_QUESTION_EVENT) + val eventMap7 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.INVALID_CONTENT_ID_EVENT) + val eventMap8 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_ASSESS_EVENT_WITHOUT_CACHE) + val eventMap9 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.SECOND_ATTEMPT_BATCH_ASSESS_EVENT) + ctx.collect(new Event(eventMap1)) + ctx.collect(new Event(eventMap2)) + ctx.collect(new Event(eventMap3)) + ctx.collect(new Event(eventMap4)) + ctx.collect(new Event(eventMap5)) + ctx.collect(new Event(eventMap6)) + ctx.collect(new Event(eventMap7)) + ctx.collect(new Event(eventMap8)) + ctx.collect(new Event(eventMap9)) + } + + override def cancel() = {} + +} - override def cancel() = {} + +class AssessmentAggreagatorEventSourceForceValidation extends SourceFunction[Event] { + override def run(ctx: SourceContext[Event]) { + val eventMap1 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.DUPLICATE_BATCH_ASSESS_EVENTS_1) + val eventMap2 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.DUPLICATE_BATCH_ASSESS_EVENTS_2) + val eventMap3 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.DUPLICATE_BATCH_ASSESS_EVENTS_3) + val eventMap4 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.DUPLICATE_BATCH_ASSESS_EVENTS_4) + ctx.collect(new Event(eventMap1)) + ctx.collect(new Event(eventMap2)) + ctx.collect(new Event(eventMap3)) + ctx.collect(new Event(eventMap4)) + } + + override def cancel() = {} } class FailedEventsSink extends SinkFunction[Event] { - override def invoke(value: Event): Unit = { - synchronized { - FailedEventsSink.values.add(value) - } + override def invoke(value: Event): Unit = { + synchronized { + FailedEventsSink.values.add(value) } + } } object FailedEventsSink { - val values: util.List[Event] = new util.ArrayList() + val values: util.List[Event] = new util.ArrayList() +} + +class certificateIssuedEventsSink extends SinkFunction[String] { + + override def invoke(value: String): Unit = { + synchronized { + certificateIssuedEvents.values.add(value) + } + } +} + +object certificateIssuedEvents { + val values: util.List[String] = new util.ArrayList() } \ No newline at end of file diff --git a/data-pipeline-flink/cb-preprocessor/README.md b/data-pipeline-flink/cb-preprocessor/README.md new file mode 100644 index 0000000000..49e1c6e742 --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/README.md @@ -0,0 +1,66 @@ +# Deduplication Flink + +This job is used to de-duplicate data by running the job using Flink. It can be run either on Yarn or Kubernetes. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a yarn or kubernetes. + +### Prerequisites + +1. Download flink-1.10.0-scala_2.12 from [apache-flink-downloads](https://www.apache.org/dyn/closer.lua/flink/flink-1.10.0/flink-1.10.0-bin-scala_2.12.tgz). +2. Download [hadoop dependencies](https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar) (only for running on Yarn). Copy the hadoop dependency jar under lib folder of the flink download. +3. export HADOOP_CLASSPATH=`/hadoop classpath` either in .bashrc or current execution shell. +4. Docker installed. +5. A running yarn cluster or a kubernetes cluster. + +### Build + +mvn clean install + +## Deployment + +### Yarn + +Flink requires memory to be allocated for both job-manager and task manager. -yjm parameter assigns job-manager memory and -ytm assigns task-manager memory. + +``` +./bin/flink run -m yarn-cluster -p 2 -yjm 1024m -ytm 1024m /de-duplication-flink/target/de-duplication-flink-0.0.1.jar +``` + +### Kubernetes + +``` +# Create a single node cluster +k3d create --server-arg --no-deploy --server-arg traefik --name flink-cluster --image rancher/k3s:v1.0.0 +# Export the single node cluster into KUBECONFIG in the current shell or in ~/.bashrc. +export KUBECONFIG="$(k3d get-kubeconfig --name='flink-cluster')" + +# Only for Mac OSX +# /usr/local/bin/kubectl -> /Applications/Docker.app/Contents/Resources/bin/kubectl +rm /usr/local/bin/kubectl +brew link --overwrite kubernetes-cli + +# Create a configmap using the flink-configuration-configmap.yaml +kubectl create -f sunbird-data-pipeline/kubernetes/flink-configuration-configmap.yaml + +# Create pods for jobmanager-service, job-manager and task-manager using the yaml files +kubectl create -f sunbird-data-pipeline/kubernetes/jobmanager-service.yaml +kubectl create -f sunbird-data-pipeline/kubernetes/jobmanager-deployment.yaml +kubectl create -f sunbird-data-pipeline/kubernetes/taskmanager-deployment.yaml + +# Create a port-forwarding for accessing the job-manager UI on localhost:8081 +kubectl port-forward deployment/flink-jobmanager 8081:8081 + +# Submit the de-duplication job to the Kubernetes single node cluster flink-cluster +./bin/flink run -m localhost:8081 /de-duplication-flink/target/de-duplication-flink-0.0.1.jar + +# Commands to delete the pods created in the cluster +kubectl delete deployment/flink-jobmanager +kubectl delete deployment/flink-taskmanager +kubectl delete service/flink-jobmanager +kubectl delete configmaps/flink-config + +# Command to stop the single-node cluster +k3d stop --name="flink-cluster" +``` diff --git a/data-pipeline-flink/cb-preprocessor/pom.xml b/data-pipeline-flink/cb-preprocessor/pom.xml new file mode 100644 index 0000000000..fe7bd97448 --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/pom.xml @@ -0,0 +1,238 @@ + + + + 4.0.0 + + org.sunbird.dp + pipeline-jobs + 1.0 + + + org.sunbird.dp.jobs + cb-preprocessor + 1.0.0 + jar + CBPreprocessor + + Flatten and Process CB_AUDIT Events + + + UTF-8 + 1.4.0 + + + + + org.apache.flink + flink-streaming-scala_${scala.maj.version} + ${flink.version} + provided + + + org.sunbird.dp + dp-core + 1.0.0 + + + com.github.java-json-tools + json-schema-validator + 2.2.8 + + + joda-time + joda-time + + + + + org.apache.commons + commons-lang3 + 3.9 + + + io.github.classgraph + classgraph + 4.8.90 + + + org.sunbird.dp + dp-core + 1.0.0 + test-jar + test + + + org.mockito + mockito-core + 3.3.3 + test + + + org.apache.flink + flink-test-utils_2.12 + ${flink.version} + test + + + org.apache.flink + flink-runtime_2.12 + ${flink.version} + test + tests + + + it.ozimov + embedded-redis + 0.7.1 + test + + + org.apache.flink + flink-streaming-java_2.12 + ${flink.version} + test + tests + + + org.scalatest + scalatest_2.12 + 3.0.6 + test + + + com.fiftyonred + mock-jedis + 0.4.0 + test + + + + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + + package + + shade + + + false + + + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.sunbird.dp.cbpreprocessor.task.CBPreprocessorStreamTask + + + + reference.conf + + + + + + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + maven-surefire-plugin + 2.22.2 + + true + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + dp-duplication-testsuite.txt + + + + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + + diff --git a/data-pipeline-flink/cb-preprocessor/src/main/resources/cb-preprocessor.conf b/data-pipeline-flink/cb-preprocessor/src/main/resources/cb-preprocessor.conf new file mode 100644 index 0000000000..b2a455e348 --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/main/resources/cb-preprocessor.conf @@ -0,0 +1,39 @@ +include "base-config.conf" + +kafka { + input.topic = ${job.env}".telemetry.cb.audit" + output.failed.topic = ${job.env}".telemetry.failed" + + #output.log.route.topic = ${job.env}".druid.events.log" + #output.error.route.topic = ${job.env}".telemetry.error" + #output.audit.route.topic = ${job.env}".telemetry.audit" + + output.duplicate.topic = ${job.env}".telemetry.duplicate" + output.cb.audit.topic =${job.env}".druid.cb.audit" + output.cb.work.order.row.topic =${job.env}".druid.cb.work.order.row" + output.cb.work.order.officer.topic =${job.env}".druid.cb.work.order.officer" + + groupId = ${job.env}"-cb-preprocessor-group" +} + +task { + consumer.parallelism = 1 + downstream.operators.parallelism = 1 +} + +# telemetry.schema.path="schemas/telemetry/3.0" +default.channel="b00bc992ef25f1a9a8d63291e20efc8d" +# dedup.producer.included.ids = ["dev.sunbird.portal", "dev.sunbird.desktop"] + +redis { + database { + duplicationstore.id = 2 + key.expiry.seconds = 3600 + } +} + +redis-meta { + database { + userstore.id = 12 + } +} \ No newline at end of file diff --git a/data-pipeline-flink/cb-preprocessor/src/main/resources/schemas/telemetry/3.0/druid_cb_data.json b/data-pipeline-flink/cb-preprocessor/src/main/resources/schemas/telemetry/3.0/druid_cb_data.json new file mode 100644 index 0000000000..e498ac429f --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/main/resources/schemas/telemetry/3.0/druid_cb_data.json @@ -0,0 +1,3 @@ +{ + "type": "object" +} \ No newline at end of file diff --git a/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/domain/Event.scala b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/domain/Event.scala new file mode 100644 index 0000000000..6658aec009 --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/domain/Event.scala @@ -0,0 +1,109 @@ +package org.sunbird.dp.cbpreprocessor.domain + +import java.util + +import org.apache.commons.lang3.StringUtils +import org.joda.time.format.DateTimeFormat +import org.sunbird.dp.core.domain.{Events, EventsPath} +import org.sunbird.dp.cbpreprocessor.task.CBPreprocessorConfig + +class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { + private[this] val dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZoneUTC + private val jobName = "CBPreprocessor" + + private val CB_OBJECT_TYPE_PATH = s"${EventsPath.EDATA_PATH}.cb_object.type" + + override def kafkaKey(): String = { + did() + } + + def schemaName: String = { + if (eid != null) s"${eid().toLowerCase}.json" + else "envelope.json" + } + + def updateActorId(actorId: String): Unit = { + telemetry.add(EventsPath.ACTOR_ID_PATH, actorId) + } + + def correctDialCodeKey(): Unit = { + val dialcodes = telemetry.read(s"${EventsPath.EDTA_FILTERS}.dialCodes").getOrElse("") + if (!dialcodes.isEmpty) { + telemetry.add(s"${EventsPath.EDTA_FILTERS}.dialcodes", dialcodes) + telemetry.add(s"${EventsPath.EDTA_FILTERS}.dialCodes", null) + } + } + + def correctDialCodeValue(): Unit = { + val dialcode = telemetry.read[String](EventsPath.OBJECT_ID_PATH).getOrElse(null) + if (dialcode != null) telemetry.add(EventsPath.OBJECT_ID_PATH, dialcode.toUpperCase) + } + + def markValidationFailure(errorMsg: String, flagName: String): Unit = { + telemetry.addFieldIfAbsent(EventsPath.FLAGS_PATH, new util.HashMap[String, Boolean]) + telemetry.add(s"${EventsPath.FLAGS_PATH}.$flagName", false) + telemetry.addFieldIfAbsent("metadata", new util.HashMap[String, AnyRef]) + if (null != errorMsg) { + telemetry.add("metadata.validation_error", errorMsg) + telemetry.add("metadata.src", jobName) + } + } + + def markSkipped(flagName: String): Unit = { + telemetry.addFieldIfAbsent(EventsPath.FLAGS_PATH, new util.HashMap[String, Boolean]) + telemetry.add(s"${EventsPath.FLAGS_PATH}.$flagName", true) + } + + def markSuccess(flagName: String): Unit = { + telemetry.addFieldIfAbsent(EventsPath.FLAGS_PATH, new util.HashMap[String, Boolean]) + telemetry.add(s"${EventsPath.FLAGS_PATH}.$flagName", true) + telemetry.add("type", "events") + } + + def updateDefaults(config: CBPreprocessorConfig): Unit = { + val channelString = telemetry.read[String](EventsPath.CONTEXT_CHANNEL_PATH).getOrElse(null) + val channel = StringUtils.deleteWhitespace(channelString) + if (channel == null || channel.isEmpty) { + telemetry.addFieldIfAbsent(EventsPath.CONTEXT_PATH, new util.HashMap[String, AnyRef]) + telemetry.add(EventsPath.CONTEXT_CHANNEL_PATH, config.defaultChannel) + } + telemetry.addFieldIfAbsent(EventsPath.SYNC_TS_PATH, dateFormatter.parseMillis(telemetry.getAtTimestamp)) + telemetry.addFieldIfAbsent(EventsPath.TIMESTAMP, telemetry.getSyncts) + } + + def edataDir: String = telemetry.read[String](EventsPath.EDATA_DIR_PATH).orNull + + def eventSyncTs: Long = telemetry.read[Long](EventsPath.SYNC_TS_PATH).getOrElse(System.currentTimeMillis()).asInstanceOf[Number].longValue() + + def eventTags: util.List[AnyRef] = telemetry.read[util.List[AnyRef]](EventsPath.TAGS_PATH).orNull + + def cdata: util.ArrayList[util.Map[String, AnyRef]] = telemetry.read[util.ArrayList[util.Map[String, AnyRef]]](EventsPath.CONTEXT_CDATA).orNull + + def eventPData: util.Map[String, AnyRef] = telemetry.read[util.Map[String, AnyRef]](EventsPath.CONTEXT_P_DATA_PATH).orNull + + def sessionId: String = telemetry.read[String](EventsPath.CONTEXT_SID_PATH).orNull + + def env: String = telemetry.read[String](EventsPath.CONTEXT_ENV_PATH).orNull + + def rollup: util.Map[String, AnyRef] = telemetry.read[util.Map[String, AnyRef]](EventsPath.CONTEXT_ROLLUP_PATH).orNull + + def cbObject: util.Map[String, Any] = telemetry.read[util.Map[String, Any]](s"${EventsPath.EDATA_PATH}.cb_object").orNull + + def cbData: util.Map[String, Any] = telemetry.read[util.Map[String, Any]](s"${EventsPath.EDATA_PATH}.cb_data").orNull + + def cbObjectType: String = telemetry.read[String](CB_OBJECT_TYPE_PATH).orNull + + def isWorkOrder: Boolean = telemetry.read[String](CB_OBJECT_TYPE_PATH).orNull == "WorkOrder" + + def isPublishedWorkOrder: Boolean = telemetry.read[String](s"${EventsPath.EDATA_PATH}.cb_data.data.status").orNull == "Published" + + def cbUid: String = s"CB:${mid()}" + + def updateOrgInfo(orgId: String, orgName: String): Unit = { + Map( + s"${EventsPath.EDATA_PATH}.cb_object.org" -> orgName, + s"${EventsPath.CONTEXT_CHANNEL_PATH}" -> orgId + ).foreach(i => telemetry.add(i._1, i._2)) + } + +} diff --git a/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/domain/Models.scala b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/domain/Models.scala new file mode 100644 index 0000000000..26d8968d2a --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/domain/Models.scala @@ -0,0 +1,26 @@ +package org.sunbird.dp.cbpreprocessor.domain + +import java.util + +case class ActorObject(id: String, `type`: String) + +case class Context(channel: String, + env: String, + sid: String, + did: String, + pdata: util.Map[String, AnyRef], + cdata: util.ArrayList[util.Map[String, AnyRef]], + rollup: util.Map[String, AnyRef]) + +case class EData(dir: String, `type`: String, size: Double) + +case class Rollup(l1: String) + +case class EventObject(id: String, ver: String, `type`: String, rollup: Rollup) + +case class CBObject(id: String, ver: String, `type`: String, name: String, org: String, sub_type: String, rollup: Rollup) + +case class CBData(id: String, + name: String, + status: String, + users: util.ArrayList[util.Map[String, AnyRef]]) diff --git a/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/functions/CBPreprocessorFunction.scala b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/functions/CBPreprocessorFunction.scala new file mode 100644 index 0000000000..fb0fcb0d61 --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/functions/CBPreprocessorFunction.scala @@ -0,0 +1,102 @@ +package org.sunbird.dp.cbpreprocessor.functions + +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.dp.core.cache.{DedupEngine, RedisConnect} +import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics} +import org.sunbird.dp.cbpreprocessor.domain.Event +import org.sunbird.dp.cbpreprocessor.task.CBPreprocessorConfig +import org.sunbird.dp.cbpreprocessor.util.{CBEventsFlattener, UserCacheUtil} + +class CBPreprocessorFunction(config: CBPreprocessorConfig, + @transient var cbEventsFlattener: CBEventsFlattener = null, + @transient var dedupEngine: DedupEngine = null, + @transient var userCache: UserCacheUtil = null + )(implicit val eventTypeInfo: TypeInformation[Event]) + extends BaseProcessFunction[Event, Event](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[CBPreprocessorFunction]) + + override def metricsList(): List[String] = { + List( + config.cbAuditEventMetricCount, + config.cbWorkOrderRowMetricCount, + config.cbWorkOrderOfficerMetricCount, + config.cbAuditFailedMetricCount + ) ::: deduplicationMetrics + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + if (dedupEngine == null) { + val redisConnect = new RedisConnect(config.redisHost, config.redisPort, config) + dedupEngine = new DedupEngine(redisConnect, config.dedupStore, config.cacheExpirySeconds) + } + if (userCache == null) { + val userRedisConnect = new RedisConnect(config.userRedisHost, config.userRedisPort, config) + userCache = new UserCacheUtil(config, userRedisConnect, config.userCacheStore) + } + if (cbEventsFlattener == null) { + cbEventsFlattener = new CBEventsFlattener() + } + } + + override def close(): Unit = { + super.close() + dedupEngine.closeConnectionPool() + userCache.close() + } + + def fixOrgInfo(event: Event): Unit = { + if (null != event.actorId() && event.actorId().trim().nonEmpty) { + val userId = event.actorId().trim() + val orgData = userCache.getUserOrgWithRetry(userId) + val orgId = orgData._1 + val orgName = orgData._2 + if (orgId.nonEmpty) event.updateOrgInfo(orgId, orgName) + } + } + + override def processElement(event: Event, + context: ProcessFunction[Event, Event]#Context, + metrics: Metrics): Unit = { + + val isUnique = + deDuplicate[Event, Event](event.cbUid, event, context, config.duplicateEventsOutputTag, + flagName = config.DEDUP_FLAG_NAME)(dedupEngine, metrics) + + if (isUnique) { + + val isWorkOrder = event.isWorkOrder + + // if its not a work order event, update org info from cache + if (!isWorkOrder) fixOrgInfo(event) // first thing fix the org info + + // output to druid cb audit events topic, competency/role/activity/workorder state (Draft, Approved, Published) + context.output(config.cbAuditEventsOutputTag, event) + metrics.incCounter(metric = config.cbAuditEventMetricCount) + + // flatten work order events till officer data and output to druid work order officer topic + if (isWorkOrder) { + cbEventsFlattener.flattenedOfficerEvents(event).foreach(itemEvent => { + context.output(config.cbWorkOrderOfficerOutputTag, itemEvent) + metrics.incCounter(metric = config.cbWorkOrderOfficerMetricCount) + }) + } + + val isPublishedWorkOrder = isWorkOrder && event.isPublishedWorkOrder + if (isPublishedWorkOrder) { + cbEventsFlattener.flattenedEvents(event).foreach { + case (itemEvent, childType, hasRole) => { + // here we can choose to route competencies and activities to different routes + context.output(config.cbWorkOrderRowOutputTag, itemEvent) + metrics.incCounter(metric = config.cbWorkOrderRowMetricCount) + } + } + } + + } + } +} diff --git a/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/task/CBPreprocessorConfig.scala b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/task/CBPreprocessorConfig.scala new file mode 100644 index 0000000000..3ab4c38326 --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/task/CBPreprocessorConfig.scala @@ -0,0 +1,81 @@ +package org.sunbird.dp.cbpreprocessor.task + +import com.typesafe.config.Config +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.streaming.api.scala.OutputTag +import org.sunbird.dp.core.job.BaseJobConfig +import org.sunbird.dp.cbpreprocessor.domain.Event + +import scala.collection.JavaConverters._ + +class CBPreprocessorConfig(override val config: Config) extends BaseJobConfig(config, "CBPreprocessorJob") { + + private val serialVersionUID = 2905979434303791379L // TODO: change this? + + implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + + // val schemaPath: String = config.getString("telemetry.schema.path") + + // Kafka Topic Configuration + val kafkaInputTopic: String = config.getString("kafka.input.topic") + val kafkaOutputCbAuditTopic: String = config.getString("kafka.output.cb.audit.topic") + val kafkaOutputCbWorkOrderRowTopic: String = config.getString("kafka.output.cb.work.order.row.topic") + val kafkaOutputCbWorkOrderOfficerTopic: String = config.getString("kafka.output.cb.work.order.officer.topic") + val kafkaFailedTopic: String = config.getString("kafka.output.failed.topic") + + val defaultChannel: String = config.getString("default.channel") + val fracChannel: String = config.getString("frac.channel") + + // Output tags + val cbAuditEventsOutputTag: OutputTag[Event] = OutputTag[Event]("cb-audit-events") + val cbWorkOrderRowOutputTag: OutputTag[Event] = OutputTag[Event]("cb-work-order-row") + val cbWorkOrderOfficerOutputTag: OutputTag[Event] = OutputTag[Event]("cb-work-order-officer") + val cbFailedOutputTag: OutputTag[Event] = OutputTag[Event]("cb-failed-events") + + override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") + val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.operators.parallelism") + + // Router job metrics + val cbWorkOrderRowMetricCount = "cb-work-order-row-count" + val cbWorkOrderOfficerMetricCount = "cb-work-order-officer-count" + val cbAuditEventMetricCount = "cb-audit-route-success-count" + val cbAuditFailedMetricCount = "cb-audit-route-failed-count" + + // Validation job metrics + // val validationSuccessMetricsCount = "validation-success-event-count" + // val validationFailureMetricsCount = "validation-failed-event-count" + // val validationSkipMetricsCount = "validation-skipped-event-count" + + // Consumers + val cbPreprocessorConsumer = "cb-preprocessor-consumer" + + // Producers + val cbAuditProducer = "cb-audit-sink" + val cbWorkOrderRowProducer = "cb-work-order-row-sink" + val cbWorkOrderOfficerProducer = "cb-work-order-officer-sink" + val cbFailedEventProducer = "cb-failed-events-sink" + + // val defaultSchemaFile = "envelope.json" + + val dedupStore: Int = config.getInt("redis.database.duplicationstore.id") + val cacheExpirySeconds: Int = config.getInt("redis.database.key.expiry.seconds") + + val userCacheStore: Int = config.getInt("redis-meta.database.userstore.id") + val userRedisHost: String = config.getString("redis-meta.host") + val userRedisPort: Int = config.getInt("redis-meta.port") + + val userStoreKeyPrefix = "user:" + val rootOrgId = "rootorgid" + val orgnameKey = "orgname" + + val duplicateEventsOutputTag: OutputTag[Event] = OutputTag[Event]("duplicate-events") + val DEDUP_FLAG_NAME = "cb_duplicate" + + val kafkaDuplicateTopic: String = config.getString("kafka.output.duplicate.topic") + val duplicateEventProducer = "duplicate-events-sink" + + + +} diff --git a/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/task/CBPreprocessorStreamTask.scala b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/task/CBPreprocessorStreamTask.scala new file mode 100644 index 0000000000..dc191e3e5d --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/task/CBPreprocessorStreamTask.scala @@ -0,0 +1,82 @@ +package org.sunbird.dp.cbpreprocessor.task + +import com.typesafe.config.ConfigFactory +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.api.java.utils.ParameterTool +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment +import org.sunbird.dp.core.job.FlinkKafkaConnector +import org.sunbird.dp.core.util.FlinkUtil +import org.sunbird.dp.cbpreprocessor.domain.Event +import org.sunbird.dp.cbpreprocessor.functions.CBPreprocessorFunction + +import java.io.File + +/** + * + */ + +class CBPreprocessorStreamTask(config: CBPreprocessorConfig, kafkaConnector: FlinkKafkaConnector) { + + private val serialVersionUID = 146697324640926024L + + def process(): Unit = { + implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) + implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + val kafkaConsumer = kafkaConnector.kafkaEventSource[Event](config.kafkaInputTopic) + + val eventStream: SingleOutputStreamOperator[Event] = + env.addSource(kafkaConsumer, config.cbPreprocessorConsumer) + .uid(config.cbPreprocessorConsumer).setParallelism(config.kafkaConsumerParallelism) + .rebalance() + .process(new CBPreprocessorFunction(config)).setParallelism(config.downstreamOperatorsParallelism) + + // all events to kafkaCbAuditRouteOutputTopic + eventStream.getSideOutput(config.cbAuditEventsOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaOutputCbAuditTopic)) + .name(config.cbAuditProducer).uid(config.cbAuditProducer) + .setParallelism(config.downstreamOperatorsParallelism) + + // work order officer rows generated from work order events to kafkaOutputCbWorkOrderOfficerTopic + eventStream.getSideOutput(config.cbWorkOrderOfficerOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaOutputCbWorkOrderOfficerTopic)) + .name(config.cbWorkOrderOfficerProducer).uid(config.cbWorkOrderOfficerProducer) + .setParallelism(config.downstreamOperatorsParallelism) + + // work order rows generated from published work order events to kafkaOutputCbWorkOrderRowTopic + eventStream.getSideOutput(config.cbWorkOrderRowOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaOutputCbWorkOrderRowTopic)) + .name(config.cbWorkOrderRowProducer).uid(config.cbWorkOrderRowProducer) + .setParallelism(config.downstreamOperatorsParallelism) + + // send failed events to kafkaFailedTopic + eventStream.getSideOutput(config.cbFailedOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaFailedTopic)) + .name(config.cbFailedEventProducer).uid(config.cbFailedEventProducer) + .setParallelism(config.downstreamOperatorsParallelism) + + eventStream.getSideOutput(config.duplicateEventsOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDuplicateTopic)) + .name(config.duplicateEventProducer).uid(config.duplicateEventProducer) + .setParallelism(config.downstreamOperatorsParallelism) + + + env.execute(config.jobName) + } +} + +// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster +object CBPreprocessorStreamTask { + def main(args: Array[String]): Unit = { + val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) + val config = configFilePath.map { + path => ConfigFactory.parseFile(new File(path)).resolve() + }.getOrElse(ConfigFactory.load("cb-preprocessor.conf").withFallback(ConfigFactory.systemEnvironment())) + val cbPreprocessorConfig = new CBPreprocessorConfig(config) + val kafkaUtil = new FlinkKafkaConnector(cbPreprocessorConfig) + val task = new CBPreprocessorStreamTask(cbPreprocessorConfig, kafkaUtil) + task.process() + } +} +// $COVERAGE-ON$ diff --git a/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/util/CBEventsFlattener.scala b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/util/CBEventsFlattener.scala new file mode 100644 index 0000000000..09d48d2266 --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/util/CBEventsFlattener.scala @@ -0,0 +1,228 @@ +package org.sunbird.dp.cbpreprocessor.util + +import org.sunbird.dp.cbpreprocessor.domain.Event + +import java.util +import scala.collection.mutable.ListBuffer + +// import scala.util.Try + +/** + * util for flattening cb_audit work order events + */ +class CBEventsFlattener extends java.io.Serializable { + + private val serialVersionUID = 1167435095740381669L // TODO: update? + + def emptyArrayList(): util.ArrayList[util.Map[String, Any]] = { new util.ArrayList[util.Map[String, Any]]() } + def emptyMap(): util.HashMap[String, Any] = { new util.HashMap[String, Any]() } + + /** + * merge arbitrary number of maps and return a new map + * `mergedMap(List((map, "", exclusions)))` returns a shallow copy of `map` excluding `exclusions` + * + * @param mergeParamList List[(map, prefix, exclusions)] + * @return + */ + def mergedMap(mergeParamList: List[(util.Map[String, Any], String, Set[String])]): util.Map[String, Any] = { + val newMap = new util.LinkedHashMap[String, Any]() + mergeParamList.foreach{ + case (map, prefix, exclusions) => { + // println(s"mergedMap() prefix=${prefix} exclusions=${exclusions}") + map.keySet().forEach(key => { + if (!exclusions.contains(key)) newMap.put(prefix + key, map.get(key)) + }) + } + } + newMap + } + + def getOrNone(map: util.Map[String, Any], key: String): Option[Any] = { + if (map == null || !map.containsKey(key)) return None + Option(map.get(key)) + } + + def getArrayListOrEmpty(map: util.Map[String, Any], key: String): util.ArrayList[util.Map[String, Any]] = { + getOrNone(map, key) match { + case Some(value) => value.asInstanceOf[util.ArrayList[util.Map[String, Any]]] + case None => emptyArrayList() + } + } + + def getMapOrEmpty(map: util.Map[String, Any], key: String): util.Map[String, Any] = { + getOrNone(map, key) match { + case Some(value) => value.asInstanceOf[util.Map[String, Any]] + case None => emptyMap() + } + } + + /** + * return a list of flattened (till officer) (denormalized) maps from workOrderMap + * + * @param workOrderMap + * @return List(work_order_officer_map) + */ + def flattenWorkOrderOfficerData(workOrderMap: util.Map[String, Any]): List[util.Map[String, Any]] = { + val flattenedList = ListBuffer[util.Map[String, Any]]() + // TO-MERGE: workOrderMap, prefix=None, exclude=['users'] + val users = getArrayListOrEmpty(workOrderMap, "users") + + users.forEach(user => { + // TO-MERGE: user, prefix='wa_', exclude=['roleCompetencyList', 'unmappedActivities', 'unmappedCompetencies'] + val newMap = mergedMap(List( + // Map Prefix Exclusion + (workOrderMap, "", Set("users")), + (user, "wa_", Set("roleCompetencyList", "unmappedActivities", "unmappedCompetencies")), + )) + flattenedList.append(newMap) + }) + flattenedList.toList + } + + /** + * return a list of flattened (denormalized) maps from workOrderMap + * + * @param workOrderMap + * @return List((work_order_row_map, child_type=activity|competency, has_role=true|false)) + */ + def flattenWorkOrderEventData(workOrderMap: util.Map[String, Any]): List[(util.Map[String, Any], String, Boolean)] = { + val flattenedList = ListBuffer[(util.Map[String, Any], String, Boolean)]() + // TO-MERGE: workOrderMap, prefix=None, exclude=['users'] + + val users = getArrayListOrEmpty(workOrderMap, "users") + // if users is empty, TODO: handle this for all empty lists +// if (users.isEmpty) { +// val newMap = mergedMap(List(MapMergeParams(workOrderMap, "", Set("users")))) +// flattenedList.append((newMap, "", false)) +// return flattenedList.toList +// } + + users.forEach(user => { + // TO-MERGE: user, prefix='wa_', exclude=['roleCompetencyList', 'unmappedActivities', 'unmappedCompetencies'] + + // take care of unmappedActivities + val unmappedActivities = getArrayListOrEmpty(user, "unmappedActivities") + unmappedActivities.forEach(activity => { + // TO-MERGE: activity, prefix='wa_activity_', exclude=[] + val newMap = mergedMap(List( + // Map Prefix Exclusion + (workOrderMap, "", Set("users")), + (user, "wa_", Set("roleCompetencyList", "unmappedActivities", "unmappedCompetencies")), + (activity, "wa_activity_", Set()) + )) + flattenedList.append((newMap, "activity", false)) + }) + + // take care of unmappedCompetencies + val unmappedCompetencies = getArrayListOrEmpty(user, "unmappedCompetencies") + unmappedCompetencies.forEach(competency => { + // TO-MERGE: competency, prefix='wa_competency_', exclude=[] + val newMap = mergedMap(List( + // Map Prefix Exclusion + (workOrderMap, "", Set("users")), + (user, "wa_", Set("roleCompetencyList", "unmappedActivities", "unmappedCompetencies")), + (competency, "wa_competency_", Set()) + )) + flattenedList.append((newMap, "competency", false)) + }) + + // dive into roleCompetencyList + val roleCompetencyList = getArrayListOrEmpty(user, "roleCompetencyList") + roleCompetencyList.forEach(roleCompetency => { + // TO-MERGE: roleCompetency, prefix='wa_rcl_', exclude=['roleDetails','competencyDetails'] + // exclude=['roleDetails','competencyDetails'] results in no fields from this level being merged + val roleDetails = getMapOrEmpty(roleCompetency, "roleDetails") + // TO-MERGE: roleDetails, prefix='wa_role_', exclude=['childNodes'] + + // activities associated with this role + val roleChildNodes = getArrayListOrEmpty(roleDetails, "childNodes") + roleChildNodes.forEach(childNode => { + // TO-MERGE: childNode, prefix='wa_activity_', exclude=[] + val newMap = mergedMap(List( + // Map Prefix Exclusion + (workOrderMap, "", Set("users")), + (user, "wa_", Set("roleCompetencyList", "unmappedActivities", "unmappedCompetencies")), + (roleCompetency, "wa_rcl_", Set("roleDetails", "competencyDetails")), + (roleDetails, "wa_role_", Set("childNodes")), + (childNode, "wa_activity_", Set()) + )) + flattenedList.append((newMap, "activity", true)) + }) + + // competencies associated with this role + val competencyDetails = getArrayListOrEmpty(roleCompetency, "competencyDetails") + competencyDetails.forEach(competency => { + // TO-MERGE: competency, prefix='wa_competency_', exclude=[] + val newMap = mergedMap(List( + // Map Prefix Exclusion + (workOrderMap, "", Set("users")), + (user, "wa_", Set("roleCompetencyList", "unmappedActivities", "unmappedCompetencies")), + (roleCompetency, "wa_rcl_", Set("roleDetails", "competencyDetails")), + (roleDetails, "wa_role_", Set("childNodes")), + (competency, "wa_competency_", Set()) + )) + flattenedList.append((newMap, "competency", true)) + }) + }) + }) + + flattenedList.toList + } + + /** + * generate a new Event with parent event map, and given eid and cbData + * + * @param eid + * @param parentEventMap + * @param cbData + * @return + */ + def getNewCbEvent(eid: String, parentEventMap: util.Map[String, Any], cbData: util.Map[String, Any]): Event = { + val eventMapEData = parentEventMap.get("edata").asInstanceOf[util.Map[String, Any]] + // create a shallow copy of eventMap excluding edata + val newEventMap = mergedMap(List((parentEventMap, "", Set("edata")))) + // store parent mid as parent_mid + newEventMap.put("mid_parent", newEventMap.get("mid")) + // generate new mid + newEventMap.put("mid", s"${eid}:${util.UUID.randomUUID().toString}") + // update eid + newEventMap.put("eid", eid) + // TODO: update ets and timestamp + // create a shallow copy of eventMapEData excluding cb_data + val newEventMapEData = mergedMap(List((eventMapEData, "", Set("cb_data")))) + // update new edata, put flatWorkOrderMap as cb_data + newEventMapEData.put("cb_data", cbData) + // update new event map, put newEventMapEData as edata + newEventMap.put("edata", newEventMapEData) + new Event(newEventMap) + } + + /** + * flatten work order data in `event` and return a Seq of CB_ITEM events + * + * @param event + * @return + */ + def flattenedEvents(event: Event): Seq[(Event, String, Boolean)] = { + val workOrderMap = event.cbData.get("data").asInstanceOf[util.Map[String, Any]] + val eventMap = event.getMap() + flattenWorkOrderEventData(workOrderMap).map { + case (flatWorkOrderMap, childType, hasRole) => { + (getNewCbEvent("CB_ITEM", eventMap, flatWorkOrderMap), childType, hasRole) + } + } + } + + /** + * flatten work order officer data in `event` and return a Seq of CB_ITEM_USER events + * + * @param event + * @return + */ + def flattenedOfficerEvents(event: Event): Seq[Event] = { + val workOrderMap = event.cbData.get("data").asInstanceOf[util.Map[String, Any]] + val eventMap = event.getMap() + flattenWorkOrderOfficerData(workOrderMap).map { data => getNewCbEvent("CB_ITEM_USER", eventMap, data) } + } + +} diff --git a/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/util/UserCacheUtil.scala b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/util/UserCacheUtil.scala new file mode 100644 index 0000000000..d973e915dc --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/main/scala/org/sunbird/dp/cbpreprocessor/util/UserCacheUtil.scala @@ -0,0 +1,40 @@ +package org.sunbird.dp.cbpreprocessor.util + +import org.sunbird.dp.cbpreprocessor.task.CBPreprocessorConfig +import org.sunbird.dp.core.cache.RedisConnect +import redis.clients.jedis.Jedis +import redis.clients.jedis.exceptions.JedisException + +class UserCacheUtil(config: CBPreprocessorConfig, redisConnect: RedisConnect, store: Int) extends Serializable { + + private val serialVersionUID = 6089562751616425355L + private[this] var redisConnection: Jedis = redisConnect.getConnection + redisConnection.select(store) + + @throws[JedisException] + def getUserOrg(userId: String): (String, String) = { + val cacheData = redisConnection.hgetAll(config.userStoreKeyPrefix + userId) + (cacheData.getOrDefault(config.rootOrgId, ""), cacheData.getOrDefault(config.orgnameKey, "")) + } + + @throws[JedisException] + def getUserOrgWithRetry(userId: String): (String, String) = { + var orgData = ("", "") + try { + orgData = getUserOrg(userId) + } catch { + case ex: JedisException => + ex.printStackTrace() + this.redisConnection.close() + this.redisConnection = redisConnect.getConnection(this.store, backoffTimeInMillis = 10000) + orgData = getUserOrg(userId) + } + orgData + } + + def getRedisConnection: Jedis = redisConnection + + def close(): Unit = { + redisConnection.close() + } +} diff --git a/data-pipeline-flink/cb-preprocessor/src/test/resources/test.conf b/data-pipeline-flink/cb-preprocessor/src/test/resources/test.conf new file mode 100644 index 0000000000..b094414db1 --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/test/resources/test.conf @@ -0,0 +1,39 @@ +include "base-test.conf" + +kafka { + input.topic = ${job.env}".telemetry.cb.audit" + output.failed.topic = ${job.env}".telemetry.failed" + output.duplicate.topic = ${job.env}".telemetry.duplicate" + output.cb.audit.topic =${job.env}".druid.cb.audit" + output.cb.work.order.row.topic =${job.env}".druid.cb.work.order.row" + output.cb.work.order.officer.topic =${job.env}".druid.cb.work.order.officer" + + groupId = ${job.env}"-cb-preprocessor-group" +} + +task { + consumer.parallelism = 1 + downstream.operators.parallelism = 1 +} + +# telemetry.schema.path="schemas/telemetry/3.0" +default.channel="b00bc992ef25f1a9a8d63291e20efc8d" +# dedup.producer.included.ids = ["dev.sunbird.portal", "dev.sunbird.desktop"] + +# redis { +# database { +# duplicationstore.id = 2 +# key.expiry.seconds = 3600 +# } +# } + +redis { + host = 127.0.0.1 + port = 6341 +} + +redis-meta { + database { + userstore.id = 12 + } +} \ No newline at end of file diff --git a/data-pipeline-flink/cb-preprocessor/src/test/scala/org/sunbird/dp/fixture/CBEventFixture.scala b/data-pipeline-flink/cb-preprocessor/src/test/scala/org/sunbird/dp/fixture/CBEventFixture.scala new file mode 100644 index 0000000000..c45b152aa1 --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/test/scala/org/sunbird/dp/fixture/CBEventFixture.scala @@ -0,0 +1,831 @@ +package org.sunbird.dp.fixture + +object CBEventFixture { + + val WO_EVENT: String = + """{ + | "actor": { + | "id": "59c3c2b7-b32b-4d9a-9d02-220e73004d66", + | "type": "User" + | }, + | "eid": "CB_AUDIT", + | "edata": { + | "state": "Draft", + | "props": ["WAT"], + | "cb_object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder", + | "ver": "1.0", + | "name": "Work order - Finance wing", + | "org": "New NHTest" + | }, + | "cb_data": { + | "data": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "users": [ + | { + | "id": "3f90ed64-2cba-4e14-8844-1ec53da454f8", + | "userId": "535c8d83-e5ed-4b91-82eb-89031702dcc9", + | "roleCompetencyList": [ + | { + | "roleDetails": { + | "type": "ROLE", + | "id": "id01", + | "name": "Management role", + | "description": "", + | "status": null, + | "source": null, + | "childNodes": [ + | { + | "type": "ACTIVITY", + | "id": "id01", + | "name": "", + | "description": "Manager role", + | "status": null, + | "source": null, + | "parentRole": null, + | "submittedFromId": null, + | "submittedToId": "", + | "level": null + | } + | ], + | "addedAt": 0, + | "updatedAt": 0, + | "updatedBy": null, + | "archivedAt": 0, + | "archived": false + | }, + | "competencyDetails": [ + | { + | "type": "COMPETENCY", + | "id": "id01", + | "name": "behavioural competency profiling", + | "description": "behavioural competency profiling desc", + | "source": null, + | "status": null, + | "level": "Level 1", + | "additionalProperties": { + | "competencyArea": "Area", + | "competencyType": "Behavioural" + | }, + | "children": null + | } + | ] + | } + | ], + | "unmappedActivities": [], + | "unmappedCompetencies": [], + | "userPosition": "Team management", + | "positionId": "id01", + | "positionDescription": "manage-teams", + | "workOrderId": "9a99e795-c652-4c0d-9f9f-960c737e15f3", + | "updatedAt": 1628845041770, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "errorCount": 0, + | "progress": 91, + | "createdAt": 1628845041770, + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c" + | } + | ] + | } + | } + | }, + | "ver": "3.0", + | "ets": 1629109359638, + | "context": { + | "channel": "013260789496258560586", + | "pdata": { + | "id": "dev.mdo.portal", + | "pid": "mdo", + | "ver": "1.0" + | }, + | "env": "WAT" + | }, + | "mid": "CB.b4b1a956-d8d5-48e4-8cee-0dc616823402", + | "object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder" + | } + |}""".stripMargin + + val WO_EVENT_RESULT: String = + """[ + | { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "wa_id": "3f90ed64-2cba-4e14-8844-1ec53da454f8", + | "wa_userId": "535c8d83-e5ed-4b91-82eb-89031702dcc9", + | "wa_userPosition": "Team management", + | "wa_positionId": "id01", + | "wa_positionDescription": "manage-teams", + | "wa_workOrderId": "9a99e795-c652-4c0d-9f9f-960c737e15f3", + | "wa_updatedAt": 1628845041770, + | "wa_updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "wa_errorCount": 0, + | "wa_progress": 91, + | "wa_createdAt": 1628845041770, + | "wa_createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "wa_role_type": "ROLE", + | "wa_role_id": "id01", + | "wa_role_name": "Management role", + | "wa_role_description": "", + | "wa_role_status": null, + | "wa_role_source": null, + | "wa_role_addedAt": 0, + | "wa_role_updatedAt": 0, + | "wa_role_updatedBy": null, + | "wa_role_archivedAt": 0, + | "wa_role_archived": false, + | "wa_activity_type": "ACTIVITY", + | "wa_activity_id": "id01", + | "wa_activity_name": "", + | "wa_activity_description": "Manager role", + | "wa_activity_status": null, + | "wa_activity_source": null, + | "wa_activity_parentRole": null, + | "wa_activity_submittedFromId": null, + | "wa_activity_submittedToId": "", + | "wa_activity_level": null + | }, + | { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "wa_id": "3f90ed64-2cba-4e14-8844-1ec53da454f8", + | "wa_userId": "535c8d83-e5ed-4b91-82eb-89031702dcc9", + | "wa_userPosition": "Team management", + | "wa_positionId": "id01", + | "wa_positionDescription": "manage-teams", + | "wa_workOrderId": "9a99e795-c652-4c0d-9f9f-960c737e15f3", + | "wa_updatedAt": 1628845041770, + | "wa_updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "wa_errorCount": 0, + | "wa_progress": 91, + | "wa_createdAt": 1628845041770, + | "wa_createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "wa_role_type": "ROLE", + | "wa_role_id": "id01", + | "wa_role_name": "Management role", + | "wa_role_description": "", + | "wa_role_status": null, + | "wa_role_source": null, + | "wa_role_addedAt": 0, + | "wa_role_updatedAt": 0, + | "wa_role_updatedBy": null, + | "wa_role_archivedAt": 0, + | "wa_role_archived": false, + | "wa_competency_type": "COMPETENCY", + | "wa_competency_id": "id01", + | "wa_competency_name": "behavioural competency profiling", + | "wa_competency_description": "behavioural competency profiling desc", + | "wa_competency_source": null, + | "wa_competency_status": null, + | "wa_competency_level": "Level 1", + | "wa_competency_additionalProperties": { + | "competencyArea": "Area", + | "competencyType": "Behavioural" + | }, + | "wa_competency_children": null + | } + |]""".stripMargin + + val WO_EVENT_POSITIONS_RESULT: String = + """[ + | { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "wa_id": "3f90ed64-2cba-4e14-8844-1ec53da454f8", + | "wa_userId": "535c8d83-e5ed-4b91-82eb-89031702dcc9", + | "wa_userPosition": "Team management", + | "wa_positionId": "id01", + | "wa_positionDescription": "manage-teams", + | "wa_workOrderId": "9a99e795-c652-4c0d-9f9f-960c737e15f3", + | "wa_updatedAt": 1628845041770, + | "wa_updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "wa_errorCount": 0, + | "wa_progress": 91, + | "wa_createdAt": 1628845041770, + | "wa_createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c" + | } + |]""".stripMargin + + val WO_EVENT_NO_USERS: String = + """{ + | "actor": { + | "id": "59c3c2b7-b32b-4d9a-9d02-220e73004d66", + | "type": "User" + | }, + | "eid": "CB_AUDIT", + | "edata": { + | "state": "Draft", + | "props": ["WAT"], + | "cb_object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder", + | "ver": "1.0", + | "name": "Work order - Finance wing", + | "org": "New NHTest" + | }, + | "cb_data": { + | "data": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest" + | } + | } + | }, + | "ver": "3.0", + | "ets": 1629109359638, + | "context": { + | "channel": "013260789496258560586", + | "pdata": { + | "id": "dev.mdo.portal", + | "pid": "mdo", + | "ver": "1.0" + | }, + | "env": "WAT" + | }, + | "mid": "CB.b4b1a956-d8d5-48e4-8cee-0dc616823402", + | "object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder" + | } + |}""".stripMargin + + val WO_EVENT_NO_USERS_RESULT: String = + """[]""".stripMargin + + val WO_EVENT_USERS_NULL: String = + """{ + | "actor": { + | "id": "59c3c2b7-b32b-4d9a-9d02-220e73004d66", + | "type": "User" + | }, + | "eid": "CB_AUDIT", + | "edata": { + | "state": "Draft", + | "props": ["WAT"], + | "cb_object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder", + | "ver": "1.0", + | "name": "Work order - Finance wing", + | "org": "New NHTest" + | }, + | "cb_data": { + | "data": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "users": null + | } + | } + | }, + | "ver": "3.0", + | "ets": 1629109359638, + | "context": { + | "channel": "013260789496258560586", + | "pdata": { + | "id": "dev.mdo.portal", + | "pid": "mdo", + | "ver": "1.0" + | }, + | "env": "WAT" + | }, + | "mid": "CB.b4b1a956-d8d5-48e4-8cee-0dc616823402", + | "object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder" + | } + |}""".stripMargin + + val WO_EVENT_USERS_NULL_RESULT: String = + """[]""".stripMargin + + val WO_EVENT_NO_RCL: String = + """{ + | "actor": { + | "id": "59c3c2b7-b32b-4d9a-9d02-220e73004d66", + | "type": "User" + | }, + | "eid": "CB_AUDIT", + | "edata": { + | "state": "Draft", + | "props": ["WAT"], + | "cb_object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder", + | "ver": "1.0", + | "name": "Work order - Finance wing", + | "org": "New NHTest" + | }, + | "cb_data": { + | "data": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "users": [ + | { + | "id": "3f90ed64-2cba-4e14-8844-1ec53da454f8", + | "userId": "535c8d83-e5ed-4b91-82eb-89031702dcc9", + | "unmappedActivities": [], + | "unmappedCompetencies": [], + | "userPosition": "Team management", + | "positionId": "id01", + | "positionDescription": "manage-teams", + | "workOrderId": "9a99e795-c652-4c0d-9f9f-960c737e15f3", + | "updatedAt": 1628845041770, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "errorCount": 0, + | "progress": 91, + | "createdAt": 1628845041770, + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c" + | } + | ] + | } + | } + | }, + | "ver": "3.0", + | "ets": 1629109359638, + | "context": { + | "channel": "013260789496258560586", + | "pdata": { + | "id": "dev.mdo.portal", + | "pid": "mdo", + | "ver": "1.0" + | }, + | "env": "WAT" + | }, + | "mid": "CB.b4b1a956-d8d5-48e4-8cee-0dc616823402", + | "object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder" + | } + |}""".stripMargin + + val WO_EVENT_NO_RCL_RESULT: String = + """[]""".stripMargin + + val WO_EVENT_NO_ACTIVITY: String = + """{ + | "actor": { + | "id": "59c3c2b7-b32b-4d9a-9d02-220e73004d66", + | "type": "User" + | }, + | "eid": "CB_AUDIT", + | "edata": { + | "state": "Draft", + | "props": ["WAT"], + | "cb_object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder", + | "ver": "1.0", + | "name": "Work order - Finance wing", + | "org": "New NHTest" + | }, + | "cb_data": { + | "data": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "users": [ + | { + | "id": "3f90ed64-2cba-4e14-8844-1ec53da454f8", + | "userId": "535c8d83-e5ed-4b91-82eb-89031702dcc9", + | "roleCompetencyList": [ + | { + | "roleDetails": { + | "type": "ROLE", + | "id": "id01", + | "name": "Management role", + | "description": "", + | "status": null, + | "source": null, + | "addedAt": 0, + | "updatedAt": 0, + | "updatedBy": null, + | "archivedAt": 0, + | "archived": false + | }, + | "competencyDetails": [ + | { + | "type": "COMPETENCY", + | "id": "id01", + | "name": "behavioural competency profiling", + | "description": "behavioural competency profiling desc", + | "source": null, + | "status": null, + | "level": "Level 1", + | "additionalProperties": { + | "competencyArea": "Area", + | "competencyType": "Behavioural" + | }, + | "children": null + | } + | ] + | } + | ], + | "unmappedActivities": [], + | "unmappedCompetencies": [], + | "userPosition": "Team management", + | "positionId": "id01", + | "positionDescription": "manage-teams", + | "workOrderId": "9a99e795-c652-4c0d-9f9f-960c737e15f3", + | "updatedAt": 1628845041770, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "errorCount": 0, + | "progress": 91, + | "createdAt": 1628845041770, + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c" + | } + | ] + | } + | } + | }, + | "ver": "3.0", + | "ets": 1629109359638, + | "context": { + | "channel": "013260789496258560586", + | "pdata": { + | "id": "dev.mdo.portal", + | "pid": "mdo", + | "ver": "1.0" + | }, + | "env": "WAT" + | }, + | "mid": "CB.b4b1a956-d8d5-48e4-8cee-0dc616823402", + | "object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder" + | } + |}""".stripMargin + + val WO_EVENT_NO_ACTIVITY_RESULT: String = + """[ + | { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "wa_id": "3f90ed64-2cba-4e14-8844-1ec53da454f8", + | "wa_userId": "535c8d83-e5ed-4b91-82eb-89031702dcc9", + | "wa_userPosition": "Team management", + | "wa_positionId": "id01", + | "wa_positionDescription": "manage-teams", + | "wa_workOrderId": "9a99e795-c652-4c0d-9f9f-960c737e15f3", + | "wa_updatedAt": 1628845041770, + | "wa_updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "wa_errorCount": 0, + | "wa_progress": 91, + | "wa_createdAt": 1628845041770, + | "wa_createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "wa_role_type": "ROLE", + | "wa_role_id": "id01", + | "wa_role_name": "Management role", + | "wa_role_description": "", + | "wa_role_status": null, + | "wa_role_source": null, + | "wa_role_addedAt": 0, + | "wa_role_updatedAt": 0, + | "wa_role_updatedBy": null, + | "wa_role_archivedAt": 0, + | "wa_role_archived": false, + | "wa_competency_type": "COMPETENCY", + | "wa_competency_id": "id01", + | "wa_competency_name": "behavioural competency profiling", + | "wa_competency_description": "behavioural competency profiling desc", + | "wa_competency_source": null, + | "wa_competency_status": null, + | "wa_competency_level": "Level 1", + | "wa_competency_additionalProperties": { + | "competencyArea": "Area", + | "competencyType": "Behavioural" + | }, + | "wa_competency_children": null + | } + |]""".stripMargin + + val WO_EVENT_NO_COMPETENCY: String = + """{ + | "actor": { + | "id": "59c3c2b7-b32b-4d9a-9d02-220e73004d66", + | "type": "User" + | }, + | "eid": "CB_AUDIT", + | "edata": { + | "state": "Draft", + | "props": ["WAT"], + | "cb_object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder", + | "ver": "1.0", + | "name": "Work order - Finance wing", + | "org": "New NHTest" + | }, + | "cb_data": { + | "data": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "users": [ + | { + | "id": "3f90ed64-2cba-4e14-8844-1ec53da454f8", + | "userId": "535c8d83-e5ed-4b91-82eb-89031702dcc9", + | "roleCompetencyList": [ + | { + | "roleDetails": { + | "type": "ROLE", + | "id": "id01", + | "name": "Management role", + | "description": "", + | "status": null, + | "source": null, + | "childNodes": [ + | { + | "type": "ACTIVITY", + | "id": "id01", + | "name": "", + | "description": "Manager role", + | "status": null, + | "source": null, + | "parentRole": null, + | "submittedFromId": null, + | "submittedToId": "", + | "level": null + | } + | ], + | "addedAt": 0, + | "updatedAt": 0, + | "updatedBy": null, + | "archivedAt": 0, + | "archived": false + | } + | } + | ], + | "unmappedActivities": [], + | "unmappedCompetencies": [], + | "userPosition": "Team management", + | "positionId": "id01", + | "positionDescription": "manage-teams", + | "workOrderId": "9a99e795-c652-4c0d-9f9f-960c737e15f3", + | "updatedAt": 1628845041770, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "errorCount": 0, + | "progress": 91, + | "createdAt": 1628845041770, + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c" + | } + | ] + | } + | } + | }, + | "ver": "3.0", + | "ets": 1629109359638, + | "context": { + | "channel": "013260789496258560586", + | "pdata": { + | "id": "dev.mdo.portal", + | "pid": "mdo", + | "ver": "1.0" + | }, + | "env": "WAT" + | }, + | "mid": "CB.b4b1a956-d8d5-48e4-8cee-0dc616823402", + | "object": { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "type": "WorkOrder" + | } + |}""".stripMargin + + val WO_EVENT_NO_COMPETENCY_RESULT: String = + """[ + | { + | "id": "643cb47c-3e8a-4d5e-9fd4-45302a9ae09a", + | "name": "Work order - Finance wing", + | "deptId": "013260789496258560586", + | "deptName": "New NHTest", + | "status": "Draft", + | "userIds": [ + | "3f90ed64-2cba-4e14-8844-1ec53da454f8" + | ], + | "createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "createdAt": 1628844512397, + | "updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "updatedAt": 1628845042921, + | "progress": 91, + | "errorCount": 0, + | "rolesCount": 1, + | "activitiesCount": 1, + | "competenciesCount": 1, + | "publishedPdfLink": null, + | "signedPdfLink": null, + | "mdo_name": "New NHTest", + | "wa_id": "3f90ed64-2cba-4e14-8844-1ec53da454f8", + | "wa_userId": "535c8d83-e5ed-4b91-82eb-89031702dcc9", + | "wa_userPosition": "Team management", + | "wa_positionId": "id01", + | "wa_positionDescription": "manage-teams", + | "wa_workOrderId": "9a99e795-c652-4c0d-9f9f-960c737e15f3", + | "wa_updatedAt": 1628845041770, + | "wa_updatedBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "wa_errorCount": 0, + | "wa_progress": 91, + | "wa_createdAt": 1628845041770, + | "wa_createdBy": "075e3a3f-1a56-4ea3-9042-c66e2288e60c", + | "wa_role_type": "ROLE", + | "wa_role_id": "id01", + | "wa_role_name": "Management role", + | "wa_role_description": "", + | "wa_role_status": null, + | "wa_role_source": null, + | "wa_role_addedAt": 0, + | "wa_role_updatedAt": 0, + | "wa_role_updatedBy": null, + | "wa_role_archivedAt": 0, + | "wa_role_archived": false, + | "wa_activity_type": "ACTIVITY", + | "wa_activity_id": "id01", + | "wa_activity_name": "", + | "wa_activity_description": "Manager role", + | "wa_activity_status": null, + | "wa_activity_source": null, + | "wa_activity_parentRole": null, + | "wa_activity_submittedFromId": null, + | "wa_activity_submittedToId": "", + | "wa_activity_level": null + | } + |]""".stripMargin +} diff --git a/data-pipeline-flink/cb-preprocessor/src/test/scala/org/sunbird/dp/spec/CBEventsFlattenerTestSpec.scala b/data-pipeline-flink/cb-preprocessor/src/test/scala/org/sunbird/dp/spec/CBEventsFlattenerTestSpec.scala new file mode 100644 index 0000000000..c5d88e655b --- /dev/null +++ b/data-pipeline-flink/cb-preprocessor/src/test/scala/org/sunbird/dp/spec/CBEventsFlattenerTestSpec.scala @@ -0,0 +1,78 @@ +package org.sunbird.dp.spec + +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.databind.ObjectMapper +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.sunbird.dp.cbpreprocessor.domain.Event +import org.sunbird.dp.cbpreprocessor.util.CBEventsFlattener +import org.sunbird.dp.core.util.JSONUtil +import org.sunbird.dp.fixture.CBEventFixture + +import java.util +import scala.collection.JavaConverters._ + +class CBEventsFlattenerTestSpec extends FlatSpec with Matchers with BeforeAndAfterAll { + + val cbEventsFlattener= new CBEventsFlattener() + val objectMapper = new ObjectMapper() + + override def beforeAll() { + super.beforeAll() + // important for including null values when serializing + JSONUtil.mapper.setSerializationInclusion(Include.ALWAYS) + } + + override protected def afterAll(): Unit = { + super.afterAll() + JSONUtil.mapper.setSerializationInclusion(Include.NON_NULL) + } + + def getFlattenedWorkOrderOfficerJson(eventJson: String): String = { + val cbEvent = new Event(JSONUtil.deserialize[util.LinkedHashMap[String, Any]](eventJson)) + val flattenedWorkOrderOfficerData = new util.ArrayList[util.Map[String, Any]]() + cbEventsFlattener.flattenedOfficerEvents(cbEvent).foreach { event => flattenedWorkOrderOfficerData.add(event.cbData) } + JSONUtil.serialize(flattenedWorkOrderOfficerData) + } + + def getFlattenedWorkOrderDataJson(eventJson: String): String = { + val cbEvent = new Event(JSONUtil.deserialize[util.LinkedHashMap[String, Any]](eventJson)) + val flattenedWorkOrderData = new util.ArrayList[util.Map[String, Any]]() + cbEventsFlattener.flattenedEvents(cbEvent).foreach { + case (event, childType, hasRole) => flattenedWorkOrderData.add(event.cbData) + } + JSONUtil.serialize(flattenedWorkOrderData) + } + + "CBEventsFlattener" should "flatten a valid work order event" in { + val resultJsonNode = objectMapper.readTree(getFlattenedWorkOrderDataJson(CBEventFixture.WO_EVENT)) + val expectedJsonNode = objectMapper.readTree(CBEventFixture.WO_EVENT_RESULT) + resultJsonNode.equals(expectedJsonNode) should be(true) + } + + "CBEventsFlattener" should "flatten a valid work order event to officer level" in { + val resultJsonNode = objectMapper.readTree(getFlattenedWorkOrderOfficerJson(CBEventFixture.WO_EVENT)) + val expectedJsonNode = objectMapper.readTree(CBEventFixture.WO_EVENT_POSITIONS_RESULT) + resultJsonNode.equals(expectedJsonNode) should be(true) + } + + "CBEventsFlattener" should "handle missing keys and null values in a work order event" in { + val resultJsonNodeNoUsers = objectMapper.readTree(getFlattenedWorkOrderDataJson(CBEventFixture.WO_EVENT_NO_USERS)) + val resultJsonNodeUsersNull = objectMapper.readTree(getFlattenedWorkOrderDataJson(CBEventFixture.WO_EVENT_USERS_NULL)) + val resultJsonNodeNoRCL = objectMapper.readTree(getFlattenedWorkOrderDataJson(CBEventFixture.WO_EVENT_NO_RCL)) + val resultJsonNodeNoActivity = objectMapper.readTree(getFlattenedWorkOrderDataJson(CBEventFixture.WO_EVENT_NO_ACTIVITY)) + val resultJsonNodeNoCompetency = objectMapper.readTree(getFlattenedWorkOrderDataJson(CBEventFixture.WO_EVENT_NO_COMPETENCY)) + + val expectedJsonNodeNoUsers = objectMapper.readTree(CBEventFixture.WO_EVENT_NO_USERS_RESULT) + val expectedJsonNodeUsersNull = objectMapper.readTree(CBEventFixture.WO_EVENT_USERS_NULL_RESULT) + val expectedJsonNodeNoRCL = objectMapper.readTree(CBEventFixture.WO_EVENT_NO_RCL_RESULT) + val expectedJsonNodeNoActivity = objectMapper.readTree(CBEventFixture.WO_EVENT_NO_ACTIVITY_RESULT) + val expectedJsonNodeNoCompetency = objectMapper.readTree(CBEventFixture.WO_EVENT_NO_COMPETENCY_RESULT) + + resultJsonNodeNoUsers.equals(expectedJsonNodeNoUsers) should be(true) + resultJsonNodeUsersNull.equals(expectedJsonNodeUsersNull) should be(true) + resultJsonNodeNoRCL.equals(expectedJsonNodeNoRCL) should be(true) + resultJsonNodeNoActivity.equals(expectedJsonNodeNoActivity) should be(true) + resultJsonNodeNoCompetency.equals(expectedJsonNodeNoCompetency) should be(true) + } + +} diff --git a/data-pipeline-flink/content-cache-updater/pom.xml b/data-pipeline-flink/content-cache-updater/pom.xml index d9b8cdc01d..6d4a37d338 100644 --- a/data-pipeline-flink/content-cache-updater/pom.xml +++ b/data-pipeline-flink/content-cache-updater/pom.xml @@ -31,7 +31,7 @@ org.apache.flink - flink-streaming-scala_${scala.version} + flink-streaming-scala_${scala.maj.version} ${flink.version} provided @@ -55,13 +55,13 @@ org.apache.flink flink-test-utils_2.12 - 1.10.0 + ${flink.version} test org.apache.flink flink-runtime_2.12 - 1.10.0 + ${flink.version} test tests @@ -74,7 +74,7 @@ org.apache.flink flink-streaming-java_2.12 - 1.10.0 + ${flink.version} test tests @@ -114,7 +114,14 @@ src/main/scala src/test/scala - + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + org.apache.maven.plugins maven-shade-plugin @@ -163,10 +170,11 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 4.4.0 - 1.8 - 1.8 + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} false diff --git a/data-pipeline-flink/content-cache-updater/src/main/resources/content-cache-updater.conf b/data-pipeline-flink/content-cache-updater/src/main/resources/content-cache-updater.conf index 7af48f484f..8a4f7e2bf6 100644 --- a/data-pipeline-flink/content-cache-updater/src/main/resources/content-cache-updater.conf +++ b/data-pipeline-flink/content-cache-updater/src/main/resources/content-cache-updater.conf @@ -11,6 +11,10 @@ redis-meta { contentstore.id = 5 dialcodestore.id = 6 } + content.host = "localhost" + dialcode.host = "localhost" + content.port = 6379 + dialcode.port = 6379 } diff --git a/data-pipeline-flink/content-cache-updater/src/main/resources/log4j.properties b/data-pipeline-flink/content-cache-updater/src/main/resources/log4j.properties deleted file mode 100644 index 05e98e6331..0000000000 --- a/data-pipeline-flink/content-cache-updater/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=contentcache.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/domain/Event.scala b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/domain/Event.scala index 0f2e75a737..3603bd48fe 100644 --- a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/domain/Event.scala +++ b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/domain/Event.scala @@ -2,24 +2,23 @@ package org.sunbird.dp.contentupdater.domain import java.util -import com.google.gson.internal.LinkedTreeMap import org.sunbird.dp.core.domain.Events class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { private val jobName = "ContentCacheUpdater" import scala.collection.JavaConverters._ - def extractProperties(): Map[String, Any] = { - val properties = telemetry.read[LinkedTreeMap[String, Any]]("transactionData.properties") + def extractProperties(): Map[String, AnyRef] = { + val properties = telemetry.read[util.Map[String, AnyRef]]("transactionData.properties") properties.map { propertySet => propertySet.asScala.map { - case (key, value) => key -> value.asInstanceOf[LinkedTreeMap[String, Any]].getOrDefault("nv", None) + case(key, value) => key -> value.asInstanceOf[util.Map[String, AnyRef]].getOrDefault("nv", None) }.toMap - }.getOrElse(Map.empty[String, Any]) + }.getOrElse(Map[String, AnyRef]()) } def getNodeUniqueId(): String = { - telemetry.read[String]("nodeUniqueId").get + telemetry.read[String]("nodeUniqueId").orNull } } diff --git a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/ContentUpdaterFunction.scala b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/ContentUpdaterFunction.scala index 0d610a46d4..973abd07fe 100644 --- a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/ContentUpdaterFunction.scala +++ b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/ContentUpdaterFunction.scala @@ -24,12 +24,12 @@ class ContentUpdaterFunction(config: ContentCacheUpdaterConfig)(implicit val map private lazy val gson = new Gson() override def metricsList(): List[String] = { - List(config.contentCacheHit) + List(config.contentCacheHit, config.skippedEventCount) } override def open(parameters: Configuration): Unit = { super.open(parameters) - dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.contentStore, List()) + dataCache = new DataCache(config, new RedisConnect(config.contentRedisHost, config.contentRedisPort, config), config.contentStore, List()) dataCache.init() } @@ -41,29 +41,35 @@ class ContentUpdaterFunction(config: ContentCacheUpdaterConfig)(implicit val map override def processElement(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics): Unit = { val nodeUniqueId = event.getNodeUniqueId() - val redisData = dataCache.getWithRetry(nodeUniqueId) - val finalProperties = event.extractProperties().filter(property => null != property._2) + if (null != nodeUniqueId) { + val redisData = dataCache.getWithRetry(nodeUniqueId) + val finalProperties = event.extractProperties().filter(property => null != property._2) - val newProperties = finalProperties.map { case (property, nv) => - if (config.contentDateFields.contains(property)) - (property, new SimpleDateFormat(config.contentDateFormat).parse(nv.toString).getTime) - else if (config.contentListFields.contains(property)) - (property, nv match { - case _: String => List(nv) - case _: util.ArrayList[String] => nv - }) - else - (property, nv) - }.filter(map => None != map._2) - redisData ++= newProperties.asInstanceOf[Map[String, AnyRef]] + val newProperties = finalProperties.map { case (property, nv) => + if (config.contentDateFields.contains(property) && null != nv && nv.asInstanceOf[String].nonEmpty) + (property, new SimpleDateFormat(config.contentDateFormat).parse(nv.toString).getTime) + else if (config.contentListFields.contains(property)) + (property, nv match { + case _: String => List(nv) + case _: util.ArrayList[String] => nv + }) + else + (property, nv) + }.filter(map => None != map._2) + redisData ++= newProperties.asInstanceOf[Map[String, AnyRef]] - if (redisData.nonEmpty) { - dataCache.setWithRetry(event.getNodeUniqueId(), gson.toJson(redisData.asJava)) - metrics.incCounter(config.contentCacheHit) - logger.info(nodeUniqueId + " Updated Successfully") - } + if (redisData.nonEmpty) { + dataCache.setWithRetry(event.getNodeUniqueId(), gson.toJson(redisData.asJava)) + metrics.incCounter(config.contentCacheHit) + logger.info(nodeUniqueId + " Updated Successfully") + } - if (finalProperties.exists(p => config.dialCodeProperties.contains(p._1))) - context.output(config.withContentDailCodeEventsTag, event) + if (finalProperties.exists(p => config.dialCodeProperties.contains(p._1))) + context.output(config.withContentDailCodeEventsTag, event) + } + else{ + metrics.incCounter(config.skippedEventCount) + logger.info("Skipping as nodeUniqueId retrieved is null from event. Event json might be invalid") + } } } diff --git a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/DialCodeUpdaterFunction.scala b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/DialCodeUpdaterFunction.scala index 7341748a85..9f019d4402 100644 --- a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/DialCodeUpdaterFunction.scala +++ b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/DialCodeUpdaterFunction.scala @@ -32,7 +32,7 @@ class DialCodeUpdaterFunction(config: ContentCacheUpdaterConfig) override def open(parameters: Configuration): Unit = { super.open(parameters) - dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), + dataCache = new DataCache(config, new RedisConnect(config.dialcodeRedisHost, config.dialcodeRedisPort, config), config.dialcodeStore, config.dialcodeFields) dataCache.init() } diff --git a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/task/ContentCacheUpdaterConfig.scala b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/task/ContentCacheUpdaterConfig.scala index 191160eb85..d1803889a2 100644 --- a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/task/ContentCacheUpdaterConfig.scala +++ b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/task/ContentCacheUpdaterConfig.scala @@ -17,10 +17,16 @@ class ContentCacheUpdaterConfig(override val config: Config) extends BaseJobConf // Kafka Topics Configuration val inputTopic: String = config.getString("kafka.input.topic") + val contentRedisHost: String = config.getString("redis-meta.content.host") + val dialcodeRedisHost: String = config.getString("redis-meta.dialcode.host") + + val contentRedisPort: Int = config.getInt("redis-meta.content.port") + val dialcodeRedisPort: Int = config.getInt("redis-meta.dialcode.port") + val contentStore: Int = config.getInt("redis-meta.database.contentstore.id") val dialcodeStore: Int = config.getInt("redis-meta.database.dialcodestore.id") - val contentListFields=List("gradeLevel","subject","medium","language") + val contentListFields=List("gradeLevel","subject","medium","language", "keywords") val contentDateFields=List("lastStatusChangedOn","lastUpdatedOn","createdOn") val dialcodeFields = List( "identifier", "channel", "publisher", "batchCode","status","generatedOn","publishedOn","metadata") @@ -35,6 +41,7 @@ class ContentCacheUpdaterConfig(override val config: Config) extends BaseJobConf val withDialCodeEventsTag: OutputTag[Event] = OutputTag[Event](WITH_DIALCODE_EVENTS) val withContentDailCodeEventsTag: OutputTag[Event] = OutputTag[Event](WITH_CONTENT_EVENTS) + val skippedEventCount = "skipped-event-count" val contentCacheHit = "cache-hit-count" val dialCodeCacheHit = "dial-codes-cache-hit-count" val dialCodeApiHit = "dial-codes-api-hit-count" diff --git a/data-pipeline-flink/content-cache-updater/src/test/resources/test.conf b/data-pipeline-flink/content-cache-updater/src/test/resources/test.conf index 58a01f1dad..3a58e0bed8 100644 --- a/data-pipeline-flink/content-cache-updater/src/test/resources/test.conf +++ b/data-pipeline-flink/content-cache-updater/src/test/resources/test.conf @@ -26,6 +26,10 @@ redis-meta { contentstore.id = 5 dialcodestore.id = 6 } + content.host = "localhost" + dialcode.host = "localhost" + content.port = 6340 + dialcode.port = 6340 } diff --git a/data-pipeline-flink/content-cache-updater/src/test/resources/test2.conf b/data-pipeline-flink/content-cache-updater/src/test/resources/test2.conf index c925536f4d..a473371f41 100644 --- a/data-pipeline-flink/content-cache-updater/src/test/resources/test2.conf +++ b/data-pipeline-flink/content-cache-updater/src/test/resources/test2.conf @@ -29,6 +29,10 @@ redis-meta { contentstore.id = 5 dialcodestore.id = 6 } + content.host = "localhost" + dialcode.host = "localhost" + content.port = 6340 + dialcode.port = 6340 } telemetry.ignore.period.months = 6 diff --git a/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala index 03cf5dc9b8..6524ff940e 100644 --- a/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala +++ b/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala @@ -12,9 +12,14 @@ object EventFixture { val contentData1 = """{"ets":1585729640345,"channel":"012550822176260096119","transactionData":{"properties":{"ownershipType":{"ov":null,"nv":["createdBy"]},"code":{"ov":null,"nv":"org.sunbird.kgcdDt"},"channel":{"ov":null,"nv":"012550822176260096119"},"description":{"ov":null,"nv":"Enter description for TextBook"},"organisation":{"ov":null,"nv":["diksha_ntptest_org"]},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/vnd.ekstep.content-collection"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2020-04-01T08:25:24.896+0000"},"appId":{"ov":null,"nv":"staging.diksha.portal"},"contentDisposition":{"ov":null,"nv":"inline"},"lastUpdatedOn":{"ov":null,"nv":"2020-04-01T08:25:24.896+0000"},"contentEncoding":{"ov":null,"nv":"gzip"},"contentType":{"ov":null,"nv":"TextBook"},"dialcodeRequired":{"ov":null,"nv":"No"},"creator":{"ov":null,"nv":"suborg_creator_sun 3"},"createdFor":{"ov":null,"nv":["012550822176260096119"]},"lastStatusChangedOn":{"ov":null,"nv":"2020-04-01T08:25:24.896+0000"},"audience":{"ov":null,"nv":["Learner"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"visibility":{"ov":null,"nv":"Default"},"os":{"ov":null,"nv":["All"]},"consumerId":{"ov":null,"nv":"a9cb3a83-a164-4bf0-aa49-b834cebf1c07"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"version":{"ov":null,"nv":2},"versionKey":{"ov":null,"nv":"1585729524896"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"license":{"ov":null,"nv":"CC BY 4.0"},"framework":{"ov":null,"nv":"ekstep_ncert_k-12"},"createdBy":{"ov":null,"nv":"7a6b150c-08be-4e31-8f69-7fc4b479e61d"},"compatibilityLevel":{"ov":null,"nv":1},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Content"},"name":{"ov":null,"nv":"BookA"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_2129902962679398401472"},"resourceType":{"ov":null,"nv":"Book"},"status":{"ov":null,"nv":"Draft"}}},"mid":"46e30999-1358-4c93-a03f-eebfdf83eb15","label":"BookA","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2020-04-01T08:27:20.345+0000","objectType":"Content","nodeUniqueId":"do_2129902962679398401472","requestId":null,"operationType":"CREATE","nodeGraphId":215201,"graphId":"domain"}""" val contentData2 = """{"ets":1586817395203,"channel":"01241974041332940818","transactionData":{"properties":{"ownershipType":{"ov":["createdBy"],"nv":null},"copyright":{"ov":"CBSE","nv":null},"subject":{"ov":["English"],"nv":"Hindi"},"channel":{"ov":"01241974041332940818","nv":null},"downloadUrl":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_312796455301447680117634/first-flight_1576838806540_do_312796455301447680117634_13.0_spine.ecar","nv":null},"organisation":{"ov":["CBSE"],"nv":null},"language":{"ov":["English"],"nv":null},"mimeType":{"ov":"application/vnd.ekstep.content-collection","nv":null},"variants":{"ov":"{\"online\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_312796455301447680117634/first-flight_1576838811143_do_312796455301447680117634_13.0_online.ecar\",\"size\":161521.0},\"spine\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_312796455301447680117634/first-flight_1576838806540_do_312796455301447680117634_13.0_spine.ecar\",\"size\":6522150.0}}","nv":null},"appIcon":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312796455301447680117634/artifact/book_1564719079536.thumb.png","nv":null},"gradeLevel":{"ov":["Class 10"],"nv":["Class 5"]},"me_totalRatingsCount":{"ov":2587,"nv":null},"appId":{"ov":"prod.diksha.portal","nv":null},"contentEncoding":{"ov":"gzip","nv":null},"lockKey":{"ov":"efb3aecc-efde-4078-a26f-a6c21d3ff1aa","nv":null},"mimeTypesCount":{"ov":"{\"application/vnd.ekstep.html-archive\":44,\"application/pdf\":72,\"application/vnd.ekstep.content-collection\":178,\"application/vnd.ekstep.ecml-archive\":88,\"video/mp4\":29}","nv":null},"totalCompressedSize":{"ov":336783683,"nv":null},"contentType":{"ov":"TextBook","nv":null},"lastUpdatedBy":{"ov":"edce4f4f-6c82-458a-8b23-e3521859992f","nv":null},"audience":{"ov":["Learner"],"nv":null},"toc_url":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312796455301447680117634/artifact/do_312796455301447680117634_toc.json","nv":null},"visibility":{"ov":"Default","nv":null},"contentTypesCount":{"ov":"{\"TextBookUnit\":178,\"PracticeQuestionSet\":88,\"ExplanationResource\":53,\"Resource\":1,\"LearningOutcomeDefinition\":22,\"LessonPlan\":47,\"FocusSpot\":22}","nv":null},"consumerId":{"ov":"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01","nv":null},"author":{"ov":"CBSE","nv":null},"mediaType":{"ov":"content","nv":null},"osId":{"ov":"org.ekstep.quiz.app","nv":null},"lastPublishedBy":{"ov":"99606810-7d5c-4f1f-80b0-36c4a0b4415d","nv":null},"version":{"ov":2,"nv":null},"rejectReasons":{"ov":["Others"],"nv":null},"prevState":{"ov":"Review","nv":null},"license":{"ov":"CC BY 4.0","nv":null},"lastPublishedOn":{"ov":"2019-12-20T10:46:44.043+0000","nv":null},"size":{"ov":6522150,"nv":null},"IL_FUNC_OBJECT_TYPE":{"ov":"ContentImage","nv":null},"rejectComment":{"ov":"download TOC","nv":null},"name":{"ov":"First Flight","nv":null},"status":{"ov":"Processing","nv":null},"code":{"ov":"org.sunbird.dYxVdV","nv":null},"prevStatus":{"ov":"Review","nv":null},"description":{"ov":"First Flight","nv":null},"medium":{"ov":["English"],"nv":null},"posterImage":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31281817869664256011879/artifact/book_1564719079536.png","nv":null},"idealScreenSize":{"ov":"normal","nv":null},"createdOn":{"ov":"2019-07-02T11:34:57.539+0000","nv":null},"reservedDialcodes":{"ov":"{\"D6A7V4\":10,\"F6Z8C1\":14,\"U8H8A6\":18,\"G8B5K7\":21,\"H4N9W3\":20,\"V3A2B6\":4,\"G3V4R7\":1,\"T2K1U9\":3,\"X8F3L3\":24,\"N7B3E4\":8,\"E5I2Q4\":19,\"Y2F9H5\":5,\"K9Q6I1\":22,\"S7S7T9\":17,\"S3C6Z9\":23,\"L6L2X7\":6,\"Z1N4C5\":7,\"Q4Z4Y6\":15,\"S5P5F3\":16,\"G1S2D1\":0,\"R1C9A2\":13,\"B5K7E7\":9,\"L8Y1D1\":2,\"N4A4H9\":11,\"C9X3J8\":12}","nv":null},"copyrightYear":{"ov":2019,"nv":null},"contentDisposition":{"ov":"inline","nv":null},"licenseterms":{"ov":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.","nv":null},"lastUpdatedOn":{"ov":"2020-04-13T22:34:10.853+0000","nv":null},"SYS_INTERNAL_LAST_UPDATED_ON":{"ov":"2020-04-13T21:51:42.618+0000","nv":null},"dialcodeRequired":{"ov":"Yes","nv":null},"createdFor":{"ov":["01241974041332940818"],"nv":null},"lastStatusChangedOn":{"ov":"2020-04-13T22:34:10.842+0000","nv":null},"os":{"ov":["All"],"nv":null},"IL_SYS_NODE_TYPE":{"ov":"DATA_NODE","nv":null},"pkgVersion":{"ov":13,"nv":null},"versionKey":{"ov":"1586817250853","nv":null},"idealScreenDensity":{"ov":"hdpi","nv":null},"depth":{"ov":0,"nv":null},"s3Key":{"ov":"ecar_files/do_312796455301447680117634/first-flight_1576838806540_do_312796455301447680117634_13.0_spine.ecar","nv":null},"dialcodes":{"ov":["G1S2D1"],"nv":null},"framework":{"ov":"ekstep_ncert_k-12","nv":null},"lastSubmittedOn":{"ov":"2020-04-13T21:51:56.227+0000","nv":null},"me_averageRating":{"ov":4,"nv":null},"createdBy":{"ov":"edce4f4f-6c82-458a-8b23-e3521859992f","nv":null},"leafNodesCount":{"ov":233,"nv":null},"compatibilityLevel":{"ov":1,"nv":null},"IL_UNIQUE_ID":{"ov":"do_312796455301447680117634.img","nv":null},"programId":{"ov":"97691300-7e50-11e9-865c-ad8fa09451f7","nv":null},"board":{"ov":"CBSE","nv":null},"resourceType":{"ov":"Book","nv":null}}},"mid":"070f5dd8-f896-4973-8786-592e0af08dc1","label":"First Flight","nodeType":"DATA_NODE","userId":"edce4f4f-6c82-458a-8b23-e3521859992f","createdOn":"2020-04-13T22:36:35.203+0000","objectType":"ContentImage","nodeUniqueId":"do_312796455301447680117634.img","requestId":null,"operationType":"DELETE","nodeGraphId":1058371,"graphId":"domain"}""" - val contentData3 = """{"ets":1586888770204,"channel":"0123221758376673287017","transactionData":{"properties":{"gradeLevel":{"ov":null,"nv":["Class 12"]},"appIcon":{"ov":null,"nv":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31293113605096243214/artifact/0_h2y5p6_1578507818966.png"},"copyright":{"ov":null,"nv":"EKSTEP"},"subject":{"ov":null,"nv":["Home Science"]},"copyrightYear":{"ov":null,"nv":2020},"topic":{"ov":null,"nv":[]},"licenseterms":{"ov":null,"nv":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content."},"lastUpdatedOn":{"ov":"2020-04-14T18:25:58.919+0000","nv":"2020-04-14T18:26:10.133+0000"},"medium":{"ov":null,"nv":["Sanskrit"]},"board":{"ov":null,"nv":"CBSE"},"versionKey":{"ov":"1586888758919","nv":"1586888770133"}}},"mid":"09be8943-0b8b-4c49-91f9-63fff4ebe58a","label":"ResourceACydney","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2020-04-14T18:26:10.204+0000","objectType":"Content","nodeUniqueId":"do_312999792564027392148","requestId":null,"operationType":"UPDATE","nodeGraphId":1071617,"graphId":"domain"}""" + val contentData3 = """{"ets":1586888770204,"channel":"0123221758376673287017","transactionData":{"properties":{"keywords":{"ov":null,"nv":["Story"]},"gradeLevel":{"ov":null,"nv":["Class 12"]},"appIcon":{"ov":null,"nv":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31293113605096243214/artifact/0_h2y5p6_1578507818966.png"},"copyright":{"ov":null,"nv":"EKSTEP"},"subject":{"ov":null,"nv":["Home Science"]},"copyrightYear":{"ov":null,"nv":2020},"topic":{"ov":null,"nv":[]},"licenseterms":{"ov":null,"nv":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content."},"lastUpdatedOn":{"ov":"2020-04-14T18:25:58.919+0000","nv":"2020-04-14T18:26:10.133+0000"},"medium":{"ov":null,"nv":["Sanskrit"]},"board":{"ov":null,"nv":"CBSE"},"versionKey":{"ov":"1586888758919","nv":"1586888770133"}}},"mid":"09be8943-0b8b-4c49-91f9-63fff4ebe58a","label":"ResourceACydney","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2020-04-14T18:26:10.204+0000","objectType":"Content","nodeUniqueId":"do_312999792564027392148","requestId":null,"operationType":"UPDATE","nodeGraphId":1071617,"graphId":"domain"}""" val contentUpdateData3 = """{"ets":1586888770204,"channel":"0123221758376673287017","transactionData":{"properties":{"gradeLevel":{"ov":null,"nv":["Class 8"]},"appIcon":{"ov":null,"nv":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31293113605096243214/artifact/0_h2y5p6_1578507818966.png"},"copyright":{"ov":null,"nv":"Ekstep"},"subject":{"ov":null,"nv":["Home Science"]},"copyrightYear":{"ov":null,"nv":2021},"topic":{"ov":null,"nv":[]},"licenseterms":{"ov":null,"nv":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content."},"lastUpdatedOn":{"ov":"2020-04-14T18:25:58.919+0000","nv":"2020-06-14T18:26:10.133+0000"},"medium":{"ov":null,"nv":["Sanskrit"]},"versionKey":{"ov":"1586888758919","nv":"1586888770133"}}},"mid":"09be8943-0b8b-4c49-91f9-63fff4ebe58a","label":"ResourceACydney","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2020-04-14T18:26:10.204+0000","objectType":"Content","nodeUniqueId":"do_312999792564027392148","requestId":null,"operationType":"UPDATE","nodeGraphId":1071617,"graphId":"domain"}""" + val invalidEvent1 = """Type":"DialCode"}""" + val invalidEvent2 = """reatedOn":"2021-02-11T07:41:59.691+0000","objectType":"DialCode"}""" + + // lastStatusChangedOn nv value = "" + val invalidNewValueEvent = """{"ets":1619804548117,"channel":"0123207707019919361056","transactionData":{"properties":{"lastStatusChangedOn":{"ov":"2021-04-28T22:18:18.898+0000","nv":""},"toc_url":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31269950653767680013604/artifact/do_31269950653767680013604_toc.json","nv":""},"contentTypesCount":{"ov":"{\"CourseUnit\":10,\"Resource\":8}","nv":""},"prevStatus":{"ov":"Review","nv":""},"childNodes":{"ov":["do_31269950821131059213610","do_31269950821131878413614","do_31269950821130240013605","do_31269950821131878413613","do_31269950821131878413612","do_31269950821131878413611","do_31269949561247334417064","do_31269948474744832017023","do_31269949442614886417054","do_31269948793305497617032","do_31269949714008473617067","do_31269950821131059213608","do_31269948948928102417036","do_31269950821130240013606","do_31269949803231641613567","do_31269950821130240013607","do_31269948673168179217026","do_31269950821131059213609"],"nv":[]},"downloadUrl":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1619648299781_do_31269950653767680013604_2.0_spine.ecar","nv":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1550233883224_do_31269950653767680013604_1.0_spine.ecar"},"variants":{"ov":"{\"online\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1619648299980_do_31269950653767680013604_2.0_online.ecar\",\"size\":10297.0},\"spine\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1619648299781_do_31269950653767680013604_2.0_spine.ecar\",\"size\":85954.0}}","nv":"{\"spine\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1550233883224_do_31269950653767680013604_1.0_spine.ecar\",\"size\":80030.0}}"},"lastPublishedBy":{"ov":"Diksha Implementation","nv":"08c04736-aa38-4b38-8ad9-8ea2857d38a0"},"pkgVersion":{"ov":2.0,"nv":1.0},"lastPublishedOn":{"ov":"2021-04-28T22:18:19.697+0000","nv":"2019-02-15T12:31:23.167+0000"},"size":{"ov":85954.0,"nv":80030.0},"lastUpdatedOn":{"ov":"2021-04-28T22:18:18.910+0000","nv":"2019-02-15T12:31:20.417+0000"},"SYS_INTERNAL_LAST_UPDATED_ON":{"ov":"2021-04-28T22:18:20.113+0000","nv":""},"totalCompressedSize":{"ov":3.2565897E7,"nv":0},"mimeTypesCount":{"ov":"{\"application/vnd.ekstep.h5p-archive\":8,\"application/vnd.ekstep.content-collection\":10}","nv":""}}},"mid":"3cadb9b9-1199-41e3-b069-76f9bf439b8a","label":"Coach- Role Clarity","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-04-30T17:42:28.117+0000","objectType":"Collection","nodeUniqueId":"do_31269950653767680013604","requestId":null,"operationType":"UPDATE","nodeGraphId":758267,"graphId":"domain"}""" } \ No newline at end of file diff --git a/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/spec/ContentUpdaterStreamTaskTest.scala b/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/spec/ContentUpdaterStreamTaskTest.scala index a2f9f2a558..a6970c2579 100644 --- a/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/spec/ContentUpdaterStreamTaskTest.scala +++ b/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/spec/ContentUpdaterStreamTaskTest.scala @@ -19,6 +19,7 @@ import org.sunbird.dp.contentupdater.domain.Event import org.sunbird.dp.contentupdater.task.{ContentCacheUpdaterConfig, ContentCacheUpdaterStreamTask} import org.sunbird.dp.core.cache.RedisConnect import org.sunbird.dp.core.job.FlinkKafkaConnector +import org.sunbird.dp.core.util.JSONUtil import org.sunbird.dp.fixture.EventFixture import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} import redis.embedded.RedisServer @@ -92,8 +93,9 @@ class ContentUpdaterStreamTaskTest extends BaseTestSpec { when(mockKafkaUtil.kafkaEventSource[Event](contentConfig.inputTopic)).thenReturn(new ContentDialCodeSource) val task = new ContentCacheUpdaterStreamTask(contentConfig, mockKafkaUtil) task.process() + BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.skippedEventCount}").getValue() should be(1) BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.dialCodeApiHit}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.contentCacheHit}").getValue() should be(10) + BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.contentCacheHit}").getValue() should be(11) BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.dialCodeApiMissHit}").getValue() should be(1) BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.dialCodeCacheHit}").getValue() should be(2) BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.totaldialCodeCount}").getValue() should be(3) @@ -106,6 +108,7 @@ class ContentUpdaterStreamTaskTest extends BaseTestSpec { assert(!contentJedis.get("do_312999792564027392148").contains("Class 12")) assert(contentJedis.get("do_312999792564027392148").contains("\"copyright\":\"Ekstep\"")) assert(!contentJedis.get("do_312999792564027392148").contains("\"copyright\":\"EKSTEP\"")) + assert(contentJedis.get("do_312999792564027392148").contains(""""keywords":["Story"]""")) } } @@ -114,16 +117,18 @@ class ContentDialCodeSource extends SourceFunction[Event] { override def run(ctx: SourceContext[Event]) { val gson = new Gson() - val event1 = gson.fromJson(EventFixture.contentData1, new util.LinkedHashMap[String, Any]().getClass) - val event2 = gson.fromJson(EventFixture.contentData2, new util.LinkedHashMap[String, Any]().getClass) - val event3 = gson.fromJson(EventFixture.contentData3, new util.LinkedHashMap[String, Any]().getClass) - val event4 = gson.fromJson(EventFixture.dialcodedata1, new util.LinkedHashMap[String, Any]().getClass) - val event5 = gson.fromJson(EventFixture.invalid_dialcocedata, new util.LinkedHashMap[String, Any]().getClass) - val event6 = gson.fromJson(EventFixture.reserved_dialcocedata, new util.LinkedHashMap[String, Any]().getClass) - val event7 = gson.fromJson(EventFixture.dialcodedata2, new util.LinkedHashMap[String, Any]().getClass) - val event8 = gson.fromJson(EventFixture.invalid_data, new util.LinkedHashMap[String, Any]().getClass) - val event9 = gson.fromJson(EventFixture.empty_dialcode, new util.LinkedHashMap[String, Any]().getClass) - val event10 = gson.fromJson(EventFixture.contentUpdateData3, new util.LinkedHashMap[String, Any]().getClass) + val event1 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.contentData1) + val event2 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.contentData2) + val event3 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.contentData3) + val event4 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.dialcodedata1) + val event5 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.invalid_dialcocedata) + val event6 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.reserved_dialcocedata) + val event7 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.dialcodedata2) + val event8 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.invalid_data) + val event9 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.empty_dialcode) + val event10 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.contentUpdateData3) + val event11 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.invalidNewValueEvent) + ctx.collect(new Event(event1)) ctx.collect(new Event(event2)) ctx.collect(new Event(event3)) @@ -134,7 +139,10 @@ class ContentDialCodeSource extends SourceFunction[Event] { ctx.collect(new Event(event8)) ctx.collect(new Event(event9)) ctx.collect(new Event(event10)) - + ctx.collect(new Event(event11)) + // for invalid event check - EventSerializationSchema returns empty map for invalid JSON. + // EX: """Type":"DialCode"}""" and """reatedOn":"2021-02-11T07:41:59.691+0000","objectType":"DialCode"}""" + ctx.collect(new Event(new util.HashMap[String, Any]())) } override def cancel() = { diff --git a/data-pipeline-flink/de-normalization/pom.xml b/data-pipeline-flink/de-normalization/pom.xml index 278e90d3b3..5c61f6c1df 100644 --- a/data-pipeline-flink/de-normalization/pom.xml +++ b/data-pipeline-flink/de-normalization/pom.xml @@ -31,7 +31,7 @@ org.apache.flink - flink-streaming-scala_${scala.version} + flink-streaming-scala_${scala.maj.version} ${flink.version} provided @@ -50,13 +50,13 @@ org.apache.flink flink-test-utils_2.12 - 1.10.0 + ${flink.version} test org.apache.flink flink-runtime_2.12 - 1.10.0 + ${flink.version} test tests @@ -69,7 +69,7 @@ org.apache.flink flink-streaming-java_2.12 - 1.10.0 + ${flink.version} test tests @@ -97,6 +97,14 @@ src/main/scala src/test/scala + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + org.apache.maven.plugins maven-shade-plugin @@ -142,10 +150,11 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 4.4.0 - 1.8 - 1.8 + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} false diff --git a/data-pipeline-flink/de-normalization/src/main/resources/de-normalization.conf b/data-pipeline-flink/de-normalization/src/main/resources/de-normalization.conf index e00174b253..8e962ffa24 100644 --- a/data-pipeline-flink/de-normalization/src/main/resources/de-normalization.conf +++ b/data-pipeline-flink/de-normalization/src/main/resources/de-normalization.conf @@ -3,21 +3,25 @@ include "base-config.conf" kafka { input.telemetry.topic = ${job.env}".telemetry.unique" input.summary.topic = ${job.env}".telemetry.derived" - output.success.topic = ${job.env}".telemetry.denorm" - output.summary.topic = ${job.env}".telemetry.derived.unique" + telemetry.denorm.output.topic = ${job.env}".telemetry.denorm" + summary.denorm.output.topic = ${job.env}".druid.events.summary" + summary.unique.events.topic = ${job.env}".telemetry.derived.unique" output.failed.topic = ${job.env}".telemetry.failed" output.duplicate.topic = ${job.env}".telemetry.duplicate" groupId = ${job.env}"-telemetry-denorm-group" } task { + window.count = 30 + window.shards = 1400 consumer.parallelism = 1 - denorm.parallelism = 1 - denorm.sink.parallelism = 1 - summary.sink.parallelism = 1 - denorm.summary-dedup.parallelism = 1 + telemetry.downstream.operators.parallelism = 1 + summary.downstream.operators.parallelism = 1 } +skip.events = ["INTERRUPT"] +permit.eid=["AUDIT"] + redis { database { duplicationstore.id = 9 @@ -29,8 +33,16 @@ redis { redis-meta { database { devicestore.id = 2 - userstore.id = 4 + userstore.id = 12 contentstore.id = 5 dialcodestore.id = 6 } + content.host = "localhost" + device.host = "localhost" + user.host = "localhost" + dialcode.host = "localhost" + content.port = 6380 + device.port = 6381 + user.port = 6382 + dialcode.port = 6383 } \ No newline at end of file diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Event.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Event.scala index 98ce77fcb4..8afd313ad3 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Event.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Event.scala @@ -5,9 +5,11 @@ import java.util import org.joda.time.{DateTime, DateTimeZone} import org.joda.time.format.{DateTimeFormat, DateTimeFormatter} import org.sunbird.dp.core.domain.{Events, EventsPath} +import org.sunbird.dp.denorm.task.DenormalizationConfig import scala.collection.JavaConverters._ import scala.collection.mutable.Map +import scala.collection.mutable class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { @@ -93,17 +95,22 @@ class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { telemetry.read[String](keyPath = EventsPath.OBJECT_ROLLUP_L1).orNull } - def objectRollUpl1FieldsPresent(): Boolean = { + def objectRollUpl2ID(): String = { + telemetry.read[String](keyPath = EventsPath.OBJECT_ROLLUP_L2).orNull + } + + def objectRollUpFieldsPresent(path: String): Boolean = { - val objectrollUpl1 = telemetry.read[String](keyPath = EventsPath.OBJECT_ROLLUP_L1).orNull + val objectrollUpl1 = telemetry.read[String](keyPath = path).orNull null != objectrollUpl1 && !objectrollUpl1.isEmpty } - def checkObjectIdNotEqualsRollUpl1Id(): Boolean = { - objectRollUpl1FieldsPresent() && !objectID().equals(objectRollUpl1ID()) + def checkObjectIdNotEqualsRollUpId(path: String): Boolean = { + objectRollUpFieldsPresent(path) && !objectID().equals(objectRollUpl1ID()) } - def addUserData(newData: Map[String, AnyRef]) { + // def addUserData(newData: Map[String, String]) { + def addUserData(newData: mutable.Map[String, AnyRef]) { val userdata: util.Map[String, AnyRef] = telemetry.read(EventsPath.USERDATA_PATH).getOrElse(new util.HashMap[String, AnyRef]()) userdata.putAll(newData.asJava) telemetry.add(EventsPath.USERDATA_PATH, userdata) @@ -129,6 +136,14 @@ class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { setFlag("coll_denorm", true) } + def addL2Data(newData: Map[String, AnyRef]) { + val l2Map = new util.HashMap[String, AnyRef]() + l2Map.putAll(newData.asJava) + telemetry.add(EventsPath.L2_DATA_PATH, l2Map) + setFlag("l2_denorm", true) + } + + def getEpochConvertedContentDataMap(data: Map[String, AnyRef]): Map[String, AnyRef] = { val lastSubmittedOn = data.get("lastsubmittedon") @@ -198,4 +213,8 @@ class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { } else "" } + def isValidEventForContentDenorm(config: DenormalizationConfig, objectId: String, objectType: String, eid: String): Boolean = { + (null != objectType && (config.permitEid.contains(eid) || !List("user", "qr", "dialcode").contains(objectType.toLowerCase())) && null != objectId) + } + } diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Models.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Models.scala index e6dca31e98..68958914c0 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Models.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Models.scala @@ -10,16 +10,24 @@ object Models { class DeviceProfile(val countryCode: String, val country: String, val stateCode: String, val state: String, val city: String, val districtCustom: String, val stateCodeCustom: String, val stateCustomName: String, val userDeclaredState: String, - val userDeclaredDistrict: String, val devicespec: util.Map[String, String], val firstaccess: Long) { + val userDeclaredDistrict: String, val devicespec: util.Map[String, String], val firstaccess: java.lang.Long) { } object DeviceProfile { - def apply(map: scala.collection.mutable.Map[String, String]) = new DeviceProfile( - map.getOrElse("country_code", ""), map.getOrElse("country", ""), map.getOrElse("state_code", ""), - map.getOrElse("state", ""), map.getOrElse("city", ""), map.getOrElse("district_custom", ""), - map.getOrElse("state_code_custom", ""), map.getOrElse("state_custom", ""), map.getOrElse("user_declared_state", ""), - map.getOrElse("user_declared_district", ""), Models.gson.fromJson(map.getOrElse("devicespec", "{}"), new util.HashMap[String, AnyRef]().getClass), map.getOrElse("firstaccess", "0").toLong) - + // def apply(map: scala.collection.mutable.Map[String, String]) = new DeviceProfile( + def apply(map: scala.collection.mutable.Map[String, AnyRef]) = new DeviceProfile( + map.getOrElse("country_code", "").asInstanceOf[String], + map.getOrElse("country", "").asInstanceOf[String], + map.getOrElse("state_code", "").asInstanceOf[String], + map.getOrElse("state", "").asInstanceOf[String], + map.getOrElse("city", "").asInstanceOf[String], + map.getOrElse("district_custom", "").asInstanceOf[String], + map.getOrElse("state_code_custom", "").asInstanceOf[String], + map.getOrElse("state_custom", "").asInstanceOf[String], map.getOrElse("user_declared_state", "").asInstanceOf[String], + map.getOrElse("user_declared_district", "").asInstanceOf[String], + // Models.gson.fromJson(map.getOrElse("devicespec", "{}").asInstanceOf[String], new util.HashMap[String, AnyRef]().getClass), + map.getOrElse("devicespec", new util.HashMap[String, AnyRef]()).asInstanceOf[util.Map[String, String]], + java.lang.Long.valueOf(map.getOrElse("firstaccess", "0").asInstanceOf[String])) } \ No newline at end of file diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationFunction.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationFunction.scala index dba0dbbdb2..836e3cc30b 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationFunction.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationFunction.scala @@ -4,10 +4,12 @@ import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.ProcessFunction import org.slf4j.LoggerFactory +import org.sunbird.dp.core.cache.RedisConnect import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics} import org.sunbird.dp.denorm.`type`._ import org.sunbird.dp.denorm.domain.Event import org.sunbird.dp.denorm.task.DenormalizationConfig +import org.sunbird.dp.denorm.util.DenormCache class DenormalizationFunction(config: DenormalizationConfig)(implicit val mapTypeInfo: TypeInformation[Event]) extends BaseProcessFunction[Event, Event](config) { @@ -19,17 +21,20 @@ class DenormalizationFunction(config: DenormalizationConfig)(implicit val mapTyp private[this] var dialcodeDenormalization: DialcodeDenormalization = _ private[this] var contentDenormalization: ContentDenormalization = _ private[this] var locationDenormalization: LocationDenormalization = _ + private[this] var denormCache: DenormCache = _ override def metricsList(): List[String] = { List(config.eventsExpired, config.userTotal, config.userCacheHit, config.userCacheMiss, config.contentTotal, config.contentCacheHit, config.contentCacheMiss, config.deviceTotal, - config.deviceCacheHit, config.deviceCacheMiss, config.eventsExpired, config.dialcodeTotal, - config.dialcodeCacheHit, config.dialcodeCacheMiss, config.locTotal, config.locCacheHit, config.locCacheMiss) + config.deviceCacheHit, config.deviceCacheMiss, config.dialcodeTotal, + config.dialcodeCacheHit, config.dialcodeCacheMiss, + config.locTotal, config.locCacheHit, config.locCacheMiss, config.eventsSkipped) } override def open(parameters: Configuration): Unit = { super.open(parameters) + denormCache = new DenormCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config)) deviceDenormalization = new DeviceDenormalization(config) userDenormalization = new UserDenormalization(config) dialcodeDenormalization = new DialcodeDenormalization(config) @@ -39,10 +44,7 @@ class DenormalizationFunction(config: DenormalizationConfig)(implicit val mapTyp override def close(): Unit = { super.close() - deviceDenormalization.closeDataCache() - userDenormalization.closeDataCache() - dialcodeDenormalization.closeDataCache() - contentDenormalization.closeDataCache() + denormCache.close() } override def processElement(event: Event, @@ -51,13 +53,17 @@ class DenormalizationFunction(config: DenormalizationConfig)(implicit val mapTyp if (event.isOlder(config.ignorePeriodInMonths)) { // Skip events older than configured value (default: 3 months) metrics.incCounter(config.eventsExpired) } else { - if ("ME_WORKFLOW_SUMMARY" == event.eid() || !event.eid().contains("SUMMARY")) { - val deviceDenormEvent = deviceDenormalization.denormalize(event, metrics) - val userDenormEvent = userDenormalization.denormalize(deviceDenormEvent, metrics) - val dialcodeDenormEvent = dialcodeDenormalization.denormalize(userDenormEvent, metrics) - val contentDenormEvent = contentDenormalization.denormalize(dialcodeDenormEvent, metrics) - val locationDenormEvent = locationDenormalization.denormalize(contentDenormEvent, metrics) - context.output(config.denormEventsTag, locationDenormEvent) + if ("ME_WORKFLOW_SUMMARY" == event.eid() || !(event.eid().contains("SUMMARY") || config.eventsToskip.contains(event.eid()))) { + val cacheData = denormCache.getDenormData(event) + deviceDenormalization.denormalize(event, cacheData, metrics) + userDenormalization.denormalize(event, cacheData, metrics) + dialcodeDenormalization.denormalize(event, cacheData, metrics) + contentDenormalization.denormalize(event, cacheData, metrics) + locationDenormalization.denormalize(event, metrics) + context.output(config.denormEventsTag, event) + } + else { + metrics.incCounter(config.eventsSkipped) } } } diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationWindowFunction.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationWindowFunction.scala new file mode 100644 index 0000000000..b6de1d879c --- /dev/null +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationWindowFunction.scala @@ -0,0 +1,163 @@ +package org.sunbird.dp.denorm.functions + +import java.lang + +import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction +import org.apache.flink.streaming.api.windowing.windows.{GlobalWindow, TimeWindow} +import org.slf4j.LoggerFactory +import org.sunbird.dp.core.cache.RedisConnect +import org.sunbird.dp.core.domain.EventsPath +import org.sunbird.dp.core.job.{Metrics, WindowBaseProcessFunction} +import org.sunbird.dp.denorm.`type`._ +import org.sunbird.dp.denorm.domain.Event +import org.sunbird.dp.denorm.task.DenormalizationConfig +import org.sunbird.dp.denorm.util.{CacheResponseData, DenormCache, DenormWindowCache} +import org.apache.flink.util.Collector + +import scala.collection.mutable.{Map => MMap} +import scala.collection.JavaConverters._ + +case class EventCache(event: Event, contentId: Option[String], collectionId: Option[String], l2Id: Option[String], deviceId: Option[String], userId: Option[String], dialcodeId: Option[String]) + +case class EventsMetadata(contentMap: MMap[String, AnyRef], collectionMap: MMap[String, AnyRef], l2RollupMap: MMap[String, AnyRef], + deviceMap: MMap[String, AnyRef], userMap: MMap[String, AnyRef], dialcodeMap: MMap[String, AnyRef]) + +class DenormalizationWindowFunction(config: DenormalizationConfig)(implicit val eventTypeInfo: TypeInformation[Event]) + extends WindowBaseProcessFunction[Event, Event, Int](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[DenormalizationWindowFunction]) + + private[this] var deviceDenormalization: DeviceDenormalization = _ + private[this] var userDenormalization: UserDenormalization = _ + private[this] var dialcodeDenormalization: DialcodeDenormalization = _ + private[this] var contentDenormalization: ContentDenormalization = _ + private[this] var locationDenormalization: LocationDenormalization = _ + private[this] var denormCache: DenormWindowCache = _ + + override def metricsList(): List[String] = { + List(config.eventsExpired, config.userTotal, config.userCacheHit, config.userCacheMiss, + config.contentTotal, config.contentCacheHit, config.contentCacheMiss, config.deviceTotal, + config.deviceCacheHit, config.deviceCacheMiss, config.dialcodeTotal, + config.dialcodeCacheHit, config.dialcodeCacheMiss, + config.locTotal, config.locCacheHit, config.locCacheMiss, config.eventsSkipped) + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + denormCache = new DenormWindowCache(config, + new RedisConnect(config.contentRedisHost, config.contentRedisPort, config), + new RedisConnect(config.deviceRedisHost, config.deviceRedisPort, config), + new RedisConnect(config.userRedisHost, config.userRedisPort, config), + new RedisConnect(config.dialcodeRedisHost, config.dialcodeRedisPort, config) + ) + deviceDenormalization = new DeviceDenormalization(config) + userDenormalization = new UserDenormalization(config) + dialcodeDenormalization = new DialcodeDenormalization(config) + contentDenormalization = new ContentDenormalization(config) + locationDenormalization = new LocationDenormalization(config) + } + + override def close(): Unit = { + super.close() + denormCache.close() + } + + override def process(key: Int, context: ProcessWindowFunction[Event, Event, Int, GlobalWindow]#Context, elements: lang.Iterable[Event], metrics: Metrics): Unit = { + + val summaryEventsList = List("ME_WORKFLOW_SUMMARY", "SUMMARY") + val eventsList = elements.asScala.toList + val filteredEventsList: List[Event] = eventsList.filter { event => + if (event.isOlder(config.ignorePeriodInMonths)) { // Skip events older than configured value (default: 3 months) + logger.info(s"Event Dropped: Event older than configured value (default: 3 months)") + metrics.incCounter(config.eventsExpired) + false + } else { + if (summaryEventsList.contains(event.eid()) || !(event.eid().contains("SUMMARY") || config.eventsToskip.contains(event.eid()))) { + true + } else { + metrics.incCounter(config.eventsSkipped) + false + } + } + } + denormalize(filteredEventsList, context, metrics) + } + + def denormalize(events: List[Event], context: ProcessWindowFunction[Event, Event, Int, GlobalWindow]#Context, metrics: Metrics) = { + + val eventCacheData = parseLookupIds(events) + val eventsDenormMeta = denormCache.getDenormData(eventCacheData._2) + + eventCacheData._1.foreach { + eventData => + val event = eventData.event + val cacheData = CacheResponseData( + content = eventsDenormMeta.contentMap.getOrElse(eventData.contentId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]], + collection = eventsDenormMeta.collectionMap.getOrElse(eventData.collectionId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]], + l2data = eventsDenormMeta.l2RollupMap.getOrElse(eventData.l2Id.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]], + device = eventsDenormMeta.deviceMap.getOrElse(eventData.deviceId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]], + user = eventsDenormMeta.userMap.getOrElse(eventData.userId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]], + dialCode = eventsDenormMeta.dialcodeMap.getOrElse(eventData.dialcodeId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]] + ) + deviceDenormalization.denormalize(event, cacheData, metrics) + userDenormalization.denormalize(event, cacheData, metrics) + dialcodeDenormalization.denormalize(event, cacheData, metrics) + contentDenormalization.denormalize(event, cacheData, metrics) + locationDenormalization.denormalize(event, metrics) + context.output(config.denormEventsTag, event) + } + } + + def parseLookupIds(events: List[Event]): (List[EventCache], EventsMetadata) = { + val contentMap = MMap[String, AnyRef]() + val collectionMap = MMap[String, AnyRef]() + val l2RollupMap = MMap[String, AnyRef]() + val dialcodeMap = MMap[String, AnyRef]() + val deviceMap = MMap[String, AnyRef]() + val userMap = MMap[String, AnyRef]() + + val eventsCache = events.map { event => + val objectType = event.objectType() + val objectId = event.objectID() + val deviceId = if (null != event.did()) Some(event.did()) else None + val actorId = event.actorId() + val actorType = event.actorType() + + val contentIds = if (event.isValidEventForContentDenorm(config, objectId, objectType, event.eid())) { + contentMap.put(objectId, null) + val collectionId = if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L1)) { + collectionMap.put(event.objectRollUpl1ID(), null) + Some(event.objectRollUpl1ID()) + } else None + + val l2RollupId = if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L2)) { + l2RollupMap.put(event.objectRollUpl2ID(), null) + Some(event.objectRollUpl2ID()) + } else None + + Some((objectId, collectionId, l2RollupId)) + } else { + None + } + + val dialcodeId = if (null != objectType && List("dialcode", "qr").contains(objectType.toLowerCase())) { + dialcodeMap.put(objectId.toUpperCase(), null) + Some(objectId.toUpperCase) + } else None + + deviceId.map(did => deviceMap.put(did, null)) + + val userId = if (null != actorId && actorId.nonEmpty && !"anonymous".equalsIgnoreCase(actorId) && ("user".equalsIgnoreCase(Option(actorType).getOrElse("")) || "ME_WORKFLOW_SUMMARY".equals(event.eid()))) { + userMap.put(config.userStoreKeyPrefix + actorId, null) + Some(config.userStoreKeyPrefix + actorId) + } else None + + EventCache(event, contentId = contentIds.map(_._1), collectionId = contentIds.flatMap(_._2), l2Id = contentIds.flatMap(_._3), deviceId = deviceId, userId = userId, dialcodeId = dialcodeId) + } + val eventsMetaCache = EventsMetadata(contentMap, collectionMap, l2RollupMap, deviceMap, userMap, dialcodeMap) + (eventsCache, eventsMetaCache) + } +} \ No newline at end of file diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationConfig.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationConfig.scala index 3c1b521e3c..bd3c275968 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationConfig.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationConfig.scala @@ -6,8 +6,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.streaming.api.scala.OutputTag import org.sunbird.dp.core.job.BaseJobConfig import org.sunbird.dp.denorm.domain.Event - -import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ class DenormalizationConfig(override val config: Config, jobName: String) extends BaseJobConfig(config, jobName ) { @@ -19,29 +18,44 @@ class DenormalizationConfig(override val config: Config, jobName: String) extend // Kafka Topics Configuration val telemetryInputTopic: String = config.getString("kafka.input.telemetry.topic") val summaryInputTopic: String = config.getString("kafka.input.summary.topic") - val denormSuccessTopic: String = config.getString("kafka.output.success.topic") + val telemetryDenormOutputTopic: String = config.getString("kafka.telemetry.denorm.output.topic") + val summaryDenormOutputTopic: String = config.getString("kafka.summary.denorm.output.topic") val failedTopic: String = config.getString("kafka.output.failed.topic") - val summaryOutputEventsTopic: String = config.getString("kafka.output.summary.topic") + val summaryUniqueEventsTopic: String = config.getString("kafka.summary.unique.events.topic") override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - val denormParallelism: Int = config.getInt("task.denorm.parallelism") - val denormSinkParallelism: Int = config.getInt("task.denorm.sink.parallelism") - val summarySinkParallelism: Int = config.getInt("task.summary.sink.parallelism") + val telemetryDownstreamOperatorsParallelism: Int = config.getInt("task.telemetry.downstream.operators.parallelism") + val summaryDownstreamOperatorsParallelism: Int = config.getInt("task.summary.downstream.operators.parallelism") + + // Windows + val windowCount: Int = config.getInt("task.window.count") + val windowShards: Int = config.getInt("task.window.shards") val userStore: Int = config.getInt("redis-meta.database.userstore.id") val contentStore: Int = config.getInt("redis-meta.database.contentstore.id") val deviceStore: Int = config.getInt("redis-meta.database.devicestore.id") val dialcodeStore: Int = config.getInt("redis-meta.database.dialcodestore.id") + val userRedisHost: String = config.getString("redis-meta.user.host") + val deviceRedisHost: String = config.getString("redis-meta.device.host") + val contentRedisHost: String = config.getString("redis-meta.content.host") + val dialcodeRedisHost: String = config.getString("redis-meta.dialcode.host") + + val userRedisPort: Int = config.getInt("redis-meta.user.port") + val deviceRedisPort: Int = config.getInt("redis-meta.device.port") + val contentRedisPort: Int = config.getInt("redis-meta.content.port") + val dialcodeRedisPort: Int = config.getInt("redis-meta.dialcode.port") + val deviceFields = List("country_code", "country", "state_code", "state", "city", "district_custom", "state_code_custom", "state_custom", "user_declared_state", "user_declared_district", "devicespec", "firstaccess") val contentFields = List("name", "objectType", "contentType", "mediaType", "language", "medium", "mimeType", "createdBy", - "createdFor", "framework", "board", "subject", "status", "pkgVersion", "lastSubmittedOn", "lastUpdatedOn", "lastPublishedOn", "channel") - val userFields = List("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype") + "createdFor", "framework", "board", "subject", "status", "pkgVersion", "lastSubmittedOn", "lastUpdatedOn", "lastPublishedOn", "channel", "gradeLevel", "keywords") + val l2DataFields = List("name", "contentType", "mimeType", "framework", "subject", "medium", "board", "channel", "createdFor", "gradeLevel") + val userFields = List("usertype", "usersubtype","grade", "language", "subject", "state", "district", "usersignintype", "userlogintype", "block", "cluster", "schoolname") val dialcodeFields = List("identifier", "channel", "batchcode", "publisher", "generated_on", "published_on", "status") val ignorePeriodInMonths:Int = if(config.hasPath("telemetry.ignore.period.months")) config.getInt("telemetry.ignore.period.months") else 3 - val summaryFilterEvents: List[String] = if(config.hasPath("summary.filter.events")) config.getStringList("summary.filter.events").toList else List("ME_WORKFLOW_SUMMARY") + val summaryFilterEvents: List[String] = if(config.hasPath("summary.filter.events")) config.getStringList("summary.filter.events").asScala.toList else List("ME_WORKFLOW_SUMMARY") val userSignInTypeDefault: String = if (config.hasPath("user.signin.type.default")) config.getString("user.signin.type.default") else "Anonymous" val userLoginInTypeDefault: String = if (config.hasPath("user.login.type.default")) config.getString("user.login.type.default") else "NA" @@ -62,6 +76,10 @@ class DenormalizationConfig(override val config: Config, jobName: String) extend val withDialCodeEventsTag: OutputTag[Event] = OutputTag[Event](WITH_DIALCODE_EVENTS) val denormEventsTag: OutputTag[Event] = OutputTag[Event](DENORM_EVENTS) + val eventsToskip: List[String] = config.getStringList("skip.events").asScala.toList + val permitEid: List[String] = config.getStringList("permit.eid").asScala.toList + val eventsSkipped = "events-skipped" + // Device Denorm Metrics val deviceTotal = "device-total" val deviceCacheHit = "device-cache-hit" @@ -118,11 +136,10 @@ class DenormalizationConfig(override val config: Config, jobName: String) extend val summaryDedupFunction = "SummaryDeduplicationFunction" val summaryDenormalizationFunction = "SummaryDenormalizationFunction" - val summaryDedupParallelism: Int = config.getInt("task.denorm.summary-dedup.parallelism") - val summarydenormParallelism: Int = config.getInt("task.denorm.parallelism") - val summaryDenormSinkParallelism: Int = config.getInt("task.summary.sink.parallelism") - // Metrics val summaryEventsCount = "summary-events-count" + //user store key prefix + val userStoreKeyPrefix = "user:" + } diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationStreamTask.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationStreamTask.scala index 4b434d6cac..8531e8396c 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationStreamTask.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationStreamTask.scala @@ -4,6 +4,7 @@ import java.io.File import com.typesafe.config.ConfigFactory import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.functions.KeySelector import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.api.java.utils.ParameterTool import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment @@ -61,12 +62,13 @@ class DenormalizationStreamTask(config: DenormalizationConfig, kafkaConnector: F val denormStream = env.addSource(source, config.denormalizationConsumer).uid(config.denormalizationConsumer) .setParallelism(config.kafkaConsumerParallelism).rebalance() - .process(new DenormalizationFunction(config)).name(config.denormalizationFunction).uid(config.denormalizationFunction) - .setParallelism(config.denormParallelism) + .keyBy(new DenormKeySelector(config)).countWindow(config.windowCount) + .process(new DenormalizationWindowFunction(config)).name(config.denormalizationFunction).uid(config.denormalizationFunction) + .setParallelism(config.telemetryDownstreamOperatorsParallelism) - denormStream.getSideOutput(config.denormEventsTag).addSink(kafkaConnector.kafkaEventSink(config.denormSuccessTopic)) + denormStream.getSideOutput(config.denormEventsTag).addSink(kafkaConnector.kafkaEventSink(config.telemetryDenormOutputTopic)) .name(config.DENORM_EVENTS_PRODUCER).uid(config.DENORM_EVENTS_PRODUCER) - .setParallelism(config.denormSinkParallelism) + .setParallelism(config.telemetryDownstreamOperatorsParallelism) env.execute(config.jobName) } @@ -88,3 +90,10 @@ object DenormalizationStreamTask { } } // $COVERAGE-ON$ + +class DenormKeySelector(config: DenormalizationConfig) extends KeySelector[Event, Int] { + val shards = config.windowShards + override def getKey(in: Event): Int = { + if (Option(in.did()) == None) "".hashCode % shards else in.did().hashCode % shards + } +} \ No newline at end of file diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/SummaryDenormalizationStreamTask.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/SummaryDenormalizationStreamTask.scala index fdf2e77e11..84aaa41682 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/SummaryDenormalizationStreamTask.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/SummaryDenormalizationStreamTask.scala @@ -8,10 +8,9 @@ import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.api.java.utils.ParameterTool import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.sunbird.dp.denorm.domain.Event -import org.sunbird.dp.denorm.functions.DenormalizationFunction +import org.sunbird.dp.denorm.functions.{DenormalizationFunction, DenormalizationWindowFunction, SummaryDeduplicationFunction} import org.sunbird.dp.core.job.FlinkKafkaConnector import org.sunbird.dp.core.util.FlinkUtil -import org.sunbird.dp.denorm.functions.SummaryDeduplicationFunction /** * Denormalization stream task does the following pipeline processing in a sequence: @@ -61,25 +60,27 @@ class SummaryDenormalizationStreamTask(config: DenormalizationConfig, kafkaConne env.addSource(source, config.summaryDenormalizationConsumer).uid(config.summaryDenormalizationConsumer) .setParallelism(config.kafkaConsumerParallelism).rebalance() .process(new SummaryDeduplicationFunction(config)).name(config.summaryDedupFunction).uid(config.summaryDedupFunction) - .setParallelism(config.summaryDedupParallelism) + .setParallelism(config.summaryDownstreamOperatorsParallelism) val summaryDenormStream = summaryEventStream.getSideOutput(config.uniqueSummaryEventsOutputTag) - .process(new DenormalizationFunction(config)) + .keyBy(new DenormKeySelector(config)).countWindow(config.windowCount) + .process(new DenormalizationWindowFunction(config)) .name(config.summaryDenormalizationFunction).uid(config.summaryDenormalizationFunction) - .setParallelism(config.denormParallelism) + .setParallelism(config.summaryDownstreamOperatorsParallelism) summaryEventStream.getSideOutput(config.duplicateEventsOutputTag) .addSink(kafkaConnector.kafkaEventSink[Event](config.duplicateTopic)) .name(config.summaryDuplicateEventProducer).uid(config.summaryDuplicateEventProducer) + .setParallelism(config.summaryDownstreamOperatorsParallelism) - summaryDenormStream.getSideOutput(config.denormEventsTag).addSink(kafkaConnector.kafkaEventSink(config.denormSuccessTopic)) + summaryDenormStream.getSideOutput(config.denormEventsTag).addSink(kafkaConnector.kafkaEventSink(config.summaryDenormOutputTopic)) .name(config.summaryDenormEventsProducer).uid(config.summaryDenormEventsProducer) - .setParallelism(config.denormSinkParallelism) + .setParallelism(config.summaryDownstreamOperatorsParallelism) summaryEventStream.getSideOutput(config.uniqueSummaryEventsOutputTag) - .addSink(kafkaConnector.kafkaEventSink(config.summaryOutputEventsTopic)) + .addSink(kafkaConnector.kafkaEventSink(config.summaryUniqueEventsTopic)) .name(config.summaryEventsProducer).uid(config.summaryEventsProducer) - .setParallelism(config.summarySinkParallelism) + .setParallelism(config.summaryDownstreamOperatorsParallelism) env.execute(config.jobName) } diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/ContentDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/ContentDenormalization.scala index 5adfea1602..83cc453961 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/ContentDenormalization.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/ContentDenormalization.scala @@ -1,23 +1,21 @@ package org.sunbird.dp.denorm.`type` -import org.sunbird.dp.core.cache.{DataCache, RedisConnect} +import org.sunbird.dp.core.domain.EventsPath import org.sunbird.dp.core.job.Metrics import org.sunbird.dp.denorm.domain.Event import org.sunbird.dp.denorm.task.DenormalizationConfig +import org.sunbird.dp.denorm.util.CacheResponseData class ContentDenormalization(config: DenormalizationConfig) { - private val contentDataCache = - new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), - config.contentStore, config.contentFields) - contentDataCache.init() - - def denormalize(event: Event, metrics: Metrics): Event = { + def denormalize(event: Event, cacheData: CacheResponseData, metrics: Metrics) = { val objectType = event.objectType() val objectId = event.objectID() - if (!List("user", "qr", "dialcode").contains(objectType) && null != objectId) { + if (event.isValidEventForContentDenorm(config, objectId, objectType, event.eid())) { metrics.incCounter(config.contentTotal) - val contentData = contentDataCache.getWithRetry(objectId).map(f => {(f._1.toLowerCase().replace("_", ""), f._2)}) + val contentData = cacheData.content.map(f => { + (f._1.toLowerCase().replace("_", ""), f._2) + }) if (contentData.nonEmpty) { metrics.incCounter(config.contentCacheHit) @@ -27,14 +25,17 @@ class ContentDenormalization(config: DenormalizationConfig) { event.setFlag("content_denorm", value = false) } - if (event.checkObjectIdNotEqualsRollUpl1Id()) { - event.addCollectionData(contentDataCache.getWithRetry(event.objectRollUpl1ID())) + if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L1)) { + event.addCollectionData(cacheData.collection.map(f => { + (f._1.toLowerCase().replace("_", ""), f._2) + })) + } + if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L2)) { + event.addL2Data(cacheData.l2data.filter(x => config.l2DataFields.contains(x._1)).map(f => { + (f._1.toLowerCase().replace("_", ""), f._2) + })) } } - event } - def closeDataCache() = { - contentDataCache.close() - } } diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DeviceDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DeviceDenormalization.scala index 99c5f2f41a..8f35ac4fd1 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DeviceDenormalization.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DeviceDenormalization.scala @@ -1,23 +1,18 @@ package org.sunbird.dp.denorm.`type` -import org.sunbird.dp.core.cache.{DataCache, RedisConnect} import org.sunbird.dp.core.job.Metrics import org.sunbird.dp.denorm.domain.{DeviceProfile, Event} import org.sunbird.dp.denorm.task.DenormalizationConfig +import org.sunbird.dp.denorm.util.CacheResponseData class DeviceDenormalization(config: DenormalizationConfig) { - private val deviceDataCache: DataCache = - new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), - config.deviceStore, config.deviceFields) - deviceDataCache.init() - - def denormalize(event: Event, metrics: Metrics): Event = { + def denormalize(event: Event, cacheData: CacheResponseData, metrics: Metrics) = { event.compareAndAlterEts() // Reset ets to today's date if we get future value val did = event.did() if (null != did && did.nonEmpty) { metrics.incCounter(config.deviceTotal) - val deviceDetails = deviceDataCache.hgetAllWithRetry(did) + val deviceDetails = cacheData.device if (deviceDetails.nonEmpty) { metrics.incCounter(config.deviceCacheHit) event.addDeviceProfile(DeviceProfile.apply(deviceDetails)) @@ -26,10 +21,6 @@ class DeviceDenormalization(config: DenormalizationConfig) { event.setFlag("device_denorm", value = false) } } - event } - def closeDataCache() = { - deviceDataCache.close() - } } diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DialcodeDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DialcodeDenormalization.scala index 229d603f96..d5bc16215c 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DialcodeDenormalization.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DialcodeDenormalization.scala @@ -1,21 +1,16 @@ package org.sunbird.dp.denorm.`type` -import org.sunbird.dp.core.cache.{DataCache, RedisConnect} import org.sunbird.dp.core.job.Metrics import org.sunbird.dp.denorm.domain.Event import org.sunbird.dp.denorm.task.DenormalizationConfig +import org.sunbird.dp.denorm.util.CacheResponseData class DialcodeDenormalization(config: DenormalizationConfig) { - private val dialcodeDataCache = - new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), - config.dialcodeStore, config.dialcodeFields) - dialcodeDataCache.init() - - def denormalize(event: Event, metrics: Metrics): Event = { + def denormalize(event: Event, cacheData: CacheResponseData, metrics: Metrics) = { if (null != event.objectType() && List("dialcode", "qr").contains(event.objectType().toLowerCase())) { metrics.incCounter(config.dialcodeTotal) - val dialcodeData = dialcodeDataCache.getWithRetry(event.objectID().toUpperCase()).map(f => {(f._1.toLowerCase().replace("_", ""), f._2)}) + val dialcodeData = cacheData.dialCode.map(f => {(f._1.toLowerCase().replace("_", ""), f._2)}) if (dialcodeData.nonEmpty) { metrics.incCounter(config.dialcodeCacheHit) @@ -25,11 +20,6 @@ class DialcodeDenormalization(config: DenormalizationConfig) { event.setFlag("dialcode_denorm", value = false) } } - event - } - - def closeDataCache() = { - dialcodeDataCache.close() } } diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/LocationDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/LocationDenormalization.scala index a47e0e53fb..74bef1e9cf 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/LocationDenormalization.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/LocationDenormalization.scala @@ -6,14 +6,16 @@ import org.sunbird.dp.denorm.task.DenormalizationConfig class LocationDenormalization(config: DenormalizationConfig) { - def denormalize(event: Event, metrics: Metrics): Event = { + def denormalize(event: Event, metrics: Metrics) = { metrics.incCounter(config.locTotal) val userProfileLocation = event.getUserProfileLocation() val userDeclaredLocation = event.getUserDeclaredLocation() val ipLocation = event.getIpLocation() val declaredLocation = if (nonEmpty(userProfileLocation)) userProfileLocation - else if (nonEmpty(userDeclaredLocation)) userDeclaredLocation else ipLocation + else if (nonEmpty(userDeclaredLocation)) { + if(userDeclaredLocation.get._1.nonEmpty) userDeclaredLocation else ipLocation + } else ipLocation if (nonEmpty(declaredLocation)) { event.addDerivedLocation(declaredLocation.get) @@ -21,7 +23,6 @@ class LocationDenormalization(config: DenormalizationConfig) { } else { metrics.incCounter(config.locCacheMiss) } - event } private def nonEmpty(loc: Option[(String, String, String)]): Boolean = { diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/UserDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/UserDenormalization.scala index 9a2ee31788..0e202dd058 100644 --- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/UserDenormalization.scala +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/UserDenormalization.scala @@ -1,23 +1,25 @@ package org.sunbird.dp.denorm.`type` +import scala.collection.mutable.Map import org.sunbird.dp.core.cache.{DataCache, RedisConnect} import org.sunbird.dp.core.job.Metrics import org.sunbird.dp.denorm.domain.Event import org.sunbird.dp.denorm.task.DenormalizationConfig +import org.sunbird.dp.denorm.util.CacheResponseData -class UserDenormalization(config: DenormalizationConfig) { +import scala.collection.mutable - private val userDataCache = - new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), - config.userStore, config.userFields) - userDataCache.init() +class UserDenormalization(config: DenormalizationConfig) { - def denormalize(event: Event, metrics: Metrics): Event = { + def denormalize(event: Event, cacheData: CacheResponseData, metrics: Metrics) = { val actorId = event.actorId() val actorType = event.actorType() - if (null != actorId && actorId.nonEmpty && !"anonymous".equalsIgnoreCase(actorId) && "user".equalsIgnoreCase(actorType)) { + if (null != actorId && actorId.nonEmpty && !"anonymous".equalsIgnoreCase(actorId) && + ("user".equalsIgnoreCase(Option(actorType).getOrElse("")) || "ME_WORKFLOW_SUMMARY".equals(event.eid()))) { + metrics.incCounter(config.userTotal) - val userData = userDataCache.getWithRetry(actorId).map(f => {(f._1.toLowerCase().replace("_", ""), f._2)}) + val userData: mutable.Map[String, AnyRef] = + cacheData.user.map(f => {(f._1.toLowerCase().replace("_", ""), f._2)}) if (userData.isEmpty) { metrics.incCounter(config.userCacheMiss) @@ -25,16 +27,11 @@ class UserDenormalization(config: DenormalizationConfig) { metrics.incCounter(config.userCacheHit) } if (!userData.contains("usersignintype")) - userData.put("usersignintype", config.userSignInTypeDefault) + userData += "usersignintype" -> config.userSignInTypeDefault if (!userData.contains("userlogintype")) - userData.put("userlogintype", config.userLoginInTypeDefault) + userData += "userlogintype" -> config.userLoginInTypeDefault event.addUserData(userData) } - event - } - - def closeDataCache() = { - userDataCache.close() } } diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormBaseCache.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormBaseCache.scala new file mode 100644 index 0000000000..0e0261bb07 --- /dev/null +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormBaseCache.scala @@ -0,0 +1,74 @@ +package org.sunbird.dp.denorm.util + +import java.util +import com.google.gson.Gson +import org.slf4j.LoggerFactory +import redis.clients.jedis.Response + +import scala.collection.mutable.{Map => MMap} +import scala.collection.JavaConverters._ +import scala.collection.mutable + +trait DenormBaseCache { + + val gson = new Gson() + private[this] val logger = LoggerFactory.getLogger(classOf[DenormBaseCache]) + + def getData(data: Response[java.util.Map[String, String]], fields: List[String]): MMap[String, String] = { + val dataMap = data.get() + if (dataMap.size() > 0) { + if (fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava) + dataMap.values().removeAll(util.Collections.singleton("")) + dataMap.asScala + } else { + MMap[String, String]() + } + } + + def getDataMap(dataStr: Response[String], fields: List[String]): MMap[String, AnyRef] = { + val data = dataStr.get + if (data != null && !data.isEmpty) { + val dataMap = gson.fromJson(data, new util.HashMap[String, AnyRef]().getClass) + if (fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava) + dataMap.values().removeAll(util.Collections.singleton("")) + dataMap.asScala + } else { + MMap[String, AnyRef]() + } + } + + def isArray(value: String): Boolean = { + val redisValue = value.trim + redisValue.length > 0 && redisValue.startsWith("[") + } + + def isObject(value: String) = { + val redisValue = value.trim + redisValue.length > 0 && redisValue.startsWith("{") + } + + def convertToComplexDataTypes(data: mutable.Map[String, String]): MMap[String, AnyRef] = { + val result = mutable.Map[String, AnyRef]() + data.keys.map { + redisKey => + val redisValue = data(redisKey) + try { + if (isArray(redisValue)) { + result += redisKey -> gson.fromJson(redisValue, new util.ArrayList[AnyRef]().getClass) + } else if (isObject(redisValue)) { + result += redisKey -> gson.fromJson(redisValue, new util.HashMap[String, AnyRef]().getClass) + } else { + result += redisKey -> redisValue.replaceAll("\\\\", "") + } + } + catch { + case ex: Exception => + logger.error("Denorm convertToComplexDataTypes failure for redis key : " + redisKey + " and value : " + redisValue) + logger.error("Denorm convertToComplexDataTypes failure :", ex) + throw ex + } + } + result + } + +} diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormCache.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormCache.scala new file mode 100644 index 0000000000..faac7feec4 --- /dev/null +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormCache.scala @@ -0,0 +1,160 @@ +package org.sunbird.dp.denorm.util + +import java.util +import com.google.gson.Gson +import org.slf4j.LoggerFactory + +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.collection.mutable.{Map => MMap} +import org.sunbird.dp.core.cache.RedisConnect +import org.sunbird.dp.denorm.domain.Event +import redis.clients.jedis.Pipeline +import org.sunbird.dp.denorm.task.DenormalizationConfig +import org.sunbird.dp.core.domain.EventsPath +import redis.clients.jedis.Response + +case class CacheResponseData(content: MMap[String, AnyRef], collection: MMap[String, AnyRef], l2data: MMap[String, AnyRef], device: MMap[String, AnyRef], + dialCode: MMap[String, AnyRef], user: MMap[String, AnyRef]) + +class DenormCache(val config: DenormalizationConfig, val redisConnect: RedisConnect) { + + private[this] val logger = LoggerFactory.getLogger(classOf[DenormCache]) + private val pipeline: Pipeline = redisConnect.getConnection(0).pipelined() + val gson = new Gson() + + def close() { + this.pipeline.close() + } + + def getDenormData(event: Event): CacheResponseData = { + this.pipeline.clear() + val responses = MMap[String, AnyRef]() + getContentCache(event, responses) + getDeviceCache(event, responses) + getDialcodeCache(event, responses) + getUserCache(event, responses) + this.pipeline.sync() + parseResponses(responses) + } + + private def getContentCache(event: Event, responses: MMap[String, AnyRef]) { + this.pipeline.select(config.contentStore) + val objectType = event.objectType() + val objectId = event.objectID() + if (!List("user", "qr", "dialcode").contains(objectType) && null != objectId) { + responses.put("content", this.pipeline.get(objectId).asInstanceOf[AnyRef]) + + if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L1)) { + responses.put("collection", this.pipeline.get(event.objectRollUpl1ID()).asInstanceOf[AnyRef]) + } + if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L2)) { + responses.put("l2data", this.pipeline.get(event.objectRollUpl2ID()).asInstanceOf[AnyRef]) + } + } + } + + private def getDialcodeCache(event: Event, responses: MMap[String, AnyRef]) { + this.pipeline.select(config.dialcodeStore) + if (null != event.objectType() && List("dialcode", "qr").contains(event.objectType().toLowerCase())) { + responses.put("dialcode", this.pipeline.get(event.objectID().toUpperCase()).asInstanceOf[AnyRef]) + } + } + + private def getDeviceCache(event: Event, responses: MMap[String, AnyRef]) { + this.pipeline.select(config.deviceStore) + if (null != event.did() && event.did().nonEmpty) { + responses.put("device", this.pipeline.hgetAll(event.did()).asInstanceOf[AnyRef]) + } + } + + private def getUserCache(event: Event, responses: scala.collection.mutable.Map[String, AnyRef]) { + this.pipeline.select(config.userStore) + val actorId = event.actorId() + val actorType = event.actorType() + if (null != actorId && actorId.nonEmpty && !"anonymous".equalsIgnoreCase(actorId) && ("user".equalsIgnoreCase(Option(actorType).getOrElse("")) || "ME_WORKFLOW_SUMMARY".equals(event.eid()))) { + responses.put("user", this.pipeline.hgetAll(config.userStoreKeyPrefix + actorId).asInstanceOf[AnyRef]) + } + } + + private def parseResponses(responses: MMap[String, AnyRef]) : CacheResponseData = { + + val userData = responses.get("user").map(data => { + convertToComplexDataTypes(getData(data.asInstanceOf[Response[java.util.Map[String, String]]], config.userFields)) + }).getOrElse(MMap[String, AnyRef]()) + + val deviceData = responses.get("device").map(data => { + convertToComplexDataTypes(getData(data.asInstanceOf[Response[java.util.Map[String, String]]], config.deviceFields)) + }).getOrElse(MMap[String, AnyRef]()) + + val contentData = responses.get("content").map(data => { + getDataMap(data.asInstanceOf[Response[String]], config.contentFields) + }).getOrElse(MMap[String, AnyRef]()) + + val collectionData = responses.get("collection").map(data => { + getDataMap(data.asInstanceOf[Response[String]], config.contentFields) + }).getOrElse(MMap[String, AnyRef]()) + + val l2Data = responses.get("l2data").map(data => { + getDataMap(data.asInstanceOf[Response[String]], config.contentFields) + }).getOrElse(MMap[String, AnyRef]()) + + val dialData = responses.get("dialcode").map(data => { + getDataMap(data.asInstanceOf[Response[String]], config.dialcodeFields) + }).getOrElse(MMap[String, AnyRef]()) + + CacheResponseData(contentData, collectionData, l2Data, deviceData, dialData, userData) + } + + private def getData(data: Response[java.util.Map[String, String]], fields: List[String]): MMap[String, String] = { + val dataMap = data.get() + if (dataMap.size() > 0) { + if (fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava) + dataMap.values().removeAll(util.Collections.singleton("")) + dataMap.asScala + } else { + MMap[String, String]() + } + } + + private def getDataMap(dataStr: Response[String], fields: List[String]): MMap[String, AnyRef] = { + val data = dataStr.get + if (data != null && !data.isEmpty) { + val dataMap = gson.fromJson(data, new util.HashMap[String, AnyRef]().getClass) + if (fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava) + dataMap.values().removeAll(util.Collections.singleton("")) + dataMap.asScala + } else { + MMap[String, AnyRef]() + } + } + + def isArray(value: String): Boolean = { + val redisValue = value.trim + redisValue.length > 0 && redisValue.startsWith("[") + } + + def isObject(value: String) = { + val redisValue = value.trim + redisValue.length > 0 && redisValue.startsWith("{") + } + + def convertToComplexDataTypes(data: mutable.Map[String, String]): MMap[String, AnyRef] = { + val result = mutable.Map[String, AnyRef]() + data.keys.map { + redisKey => + val redisValue = data(redisKey) + if (isArray(redisValue)) { + result += redisKey -> gson.fromJson(redisValue, new util.ArrayList[AnyRef]().getClass) + } else if (isObject(redisValue)) { + result += redisKey -> gson.fromJson(redisValue, new util.HashMap[String, AnyRef]().getClass) + } else { + result += redisKey -> redisValue + } + } + result + } + +} + +// $COVERAGE-ON$ diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormWindowCache.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormWindowCache.scala new file mode 100644 index 0000000000..4c6ecc5ce2 --- /dev/null +++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormWindowCache.scala @@ -0,0 +1,106 @@ +package org.sunbird.dp.denorm.util + +import org.sunbird.dp.core.cache.RedisConnect +import org.sunbird.dp.denorm.functions.EventsMetadata +import org.sunbird.dp.denorm.task.DenormalizationConfig +import redis.clients.jedis.Response + +import scala.collection.mutable.{Map => MMap} + +class DenormWindowCache(config: DenormalizationConfig, contentRedis: RedisConnect, deviceRedis: RedisConnect, + userRedis: RedisConnect, dialcodeRedis: RedisConnect) extends DenormBaseCache { + + private val contentPipeline = contentRedis.getConnection(config.contentStore).pipelined() + private val devicePipeline = deviceRedis.getConnection(config.deviceStore).pipelined() + private val userPipeline = userRedis.getConnection(config.userStore).pipelined() + private val dialcodePipeline = dialcodeRedis.getConnection(config.dialcodeStore).pipelined() + + def close(): Unit = { + contentPipeline.close() + devicePipeline.close() + userPipeline.close() + dialcodePipeline.close() + } + + def getDenormData(eventsMeta: EventsMetadata): EventsMetadata = { + contentPipeline.clear() + devicePipeline.clear() + userPipeline.clear() + dialcodePipeline.clear() + getContentCacheData(eventsMeta.contentMap, eventsMeta.collectionMap, eventsMeta.l2RollupMap) + getDeviceCacheData(eventsMeta.deviceMap) + getUserCacheData(eventsMeta.userMap) + getDialcodeCacheData(eventsMeta.dialcodeMap) + contentPipeline.sync() + devicePipeline.sync() + userPipeline.sync() + dialcodePipeline.sync() + parseResponses(eventsMeta) + eventsMeta + } + + private def getContentCacheData(contentMap: MMap[String, AnyRef], collectionMap: MMap[String, AnyRef], l2rollupMap: MMap[String, AnyRef]) = { + contentMap.keySet.foreach { + contentId => contentMap.put(contentId, contentPipeline.get(contentId)) + } + + collectionMap.keySet.foreach { + collectionId => collectionMap.put(collectionId, contentPipeline.get(collectionId)) + } + + l2rollupMap.keySet.foreach { + l2RollupId => l2rollupMap.put(l2RollupId, contentPipeline.get(l2RollupId)) + } + } + + private def getDeviceCacheData(deviceMap: MMap[String, AnyRef]) = { + deviceMap.keySet.foreach { + deviceId => deviceMap.put(deviceId, devicePipeline.hgetAll(deviceId)) + } + } + + private def getUserCacheData(userMap: MMap[String, AnyRef]) = { + userMap.keySet.foreach { + userId => userMap.put(userId, userPipeline.hgetAll(userId)) + } + } + + private def getDialcodeCacheData(dialcodeMap: MMap[String, AnyRef]) = { + dialcodeMap.keySet.foreach { + dialcodeId => dialcodeMap.put(dialcodeId, dialcodePipeline.get(dialcodeId)) + } + } + + def parseResponses(eventsMeta: EventsMetadata) = { + eventsMeta.contentMap.foreach { + case (contentId, contentResponse) => + eventsMeta.contentMap.put(contentId, getDataMap(contentResponse.asInstanceOf[Response[String]], config.contentFields)) + } + + eventsMeta.collectionMap.foreach { + case (collectionId, contentResponse) => + eventsMeta.collectionMap.put(collectionId, getDataMap(contentResponse.asInstanceOf[Response[String]], config.contentFields)) + } + + eventsMeta.l2RollupMap.foreach { + case (l2RollupId, contentResponse) => + eventsMeta.l2RollupMap.put(l2RollupId, getDataMap(contentResponse.asInstanceOf[Response[String]], config.contentFields)) + } + + eventsMeta.deviceMap.foreach { + case (deviceId, deviceResponse) => + eventsMeta.deviceMap.put(deviceId, convertToComplexDataTypes(getData(deviceResponse.asInstanceOf[Response[java.util.Map[String, String]]], config.deviceFields))) + } + + eventsMeta.userMap.foreach { + case (userId, userResponse) => + eventsMeta.userMap.put(userId, convertToComplexDataTypes(getData(userResponse.asInstanceOf[Response[java.util.Map[String, String]]], config.userFields))) + } + + eventsMeta.dialcodeMap.foreach { + case (dialcodeId, dialcodeResponse) => + eventsMeta.dialcodeMap.put(dialcodeId, getDataMap(dialcodeResponse.asInstanceOf[Response[String]], config.dialcodeFields)) + } + + } +} diff --git a/data-pipeline-flink/de-normalization/src/test/resources/test.conf b/data-pipeline-flink/de-normalization/src/test/resources/test.conf index 397072f343..d8b87ad70b 100644 --- a/data-pipeline-flink/de-normalization/src/test/resources/test.conf +++ b/data-pipeline-flink/de-normalization/src/test/resources/test.conf @@ -3,8 +3,9 @@ include "base-test.conf" kafka { input.telemetry.topic = "flink.telemetry.unique" input.summary.topic = "flink.telemetry.derived" - output.success.topic = "flink.telemetry.denorm" - output.summary.topic = "flink.telemetry.derived.unique" + telemetry.denorm.output.topic = "flink.telemetry.denorm" + summary.denorm.output.topic = "flink.druid.events.summary" + summary.unique.events.topic = "flink.telemetry.derived.unique" output.failed.topic = "flink.telemetry.failed" output.duplicate.topic = "flink.telemetry.duplicate" groupId = "flink-telemetry-denorm-group" @@ -12,12 +13,15 @@ kafka { telemetry.ignore.period.months = 4 task { - denorm.parallelism = 1 - denorm.sink.parallelism = 1 - summary.sink.parallelism = 1 - denorm.summary-dedup.parallelism = 1 + window.count = 1 + window.shards = 10 + telemetry.downstream.operators.parallelism = 1 + summary.downstream.operators.parallelism = 1 } +skip.events = ["INTERRUPT"] +permit.eid=["AUDIT"] + redis { host = 127.0.0.1 port = 6341 @@ -32,8 +36,16 @@ redis-meta { port = 6341 database { devicestore.id = 2 - userstore.id = 4 + userstore.id = 12 contentstore.id = 5 dialcodestore.id = 6 } + content.host = "localhost" + device.host = "localhost" + user.host = "localhost" + dialcode.host = "localhost" + content.port = 6341 + device.port = 6341 + user.port = 6341 + dialcode.port = 6341 } \ No newline at end of file diff --git a/data-pipeline-flink/de-normalization/src/test/resources/test2.conf b/data-pipeline-flink/de-normalization/src/test/resources/test2.conf index b06010376f..0ee9d378e2 100644 --- a/data-pipeline-flink/de-normalization/src/test/resources/test2.conf +++ b/data-pipeline-flink/de-normalization/src/test/resources/test2.conf @@ -3,20 +3,24 @@ include "base-test.conf" kafka { input.telemetry.topic = "flink.telemetry.unique" input.summary.topic = "flink.telemetry.derived" - output.success.topic = "flink.telemetry.denorm" - output.summary.topic = "flink.telemetry.derived.unique" + telemetry.denorm.output.topic = "flink.telemetry.denorm" + summary.denorm.output.topic = "flink.druid.events.summary" + summary.unique.events.topic = "flink.telemetry.derived.unique" output.failed.topic = "flink.telemetry.failed" output.duplicate.topic = "flink.telemetry.duplicate" groupId = "flink-telemetry-denorm-group" } task { - denorm.parallelism = 1 - denorm.sink.parallelism = 1 - summary.sink.parallelism = 1 - denorm.summary-dedup.parallelism = 1 + window.count = 1 + window.shards = 10 + telemetry.downstream.operators.parallelism = 1 + summary.downstream.operators.parallelism = 1 } +skip.events = ["INTERRUPT"] +permit.eid=["AUDIT"] + redis { host = 127.0.0.1 port = 6340 @@ -31,10 +35,18 @@ redis-meta { port = 6340 database { devicestore.id = 2 - userstore.id = 4 + userstore.id = 12 contentstore.id = 5 dialcodestore.id = 6 } + content.host = "localhost" + device.host = "localhost" + user.host = "localhost" + dialcode.host = "localhost" + content.port = 6340 + device.port = 6340 + user.port = 6340 + dialcode.port = 6340 } telemetry.ignore.period.months = 6 diff --git a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala index 72f0932fe7..56e2b35597 100644 --- a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala +++ b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala @@ -1,20 +1,32 @@ package org.sunbird.dp.fixture +import java.util + +import com.google.gson.Gson import org.joda.time.DateTime object EventFixture { - - + + val gson = new Gson() val deviceCacheData1 = """{"country":"India","state_custom":"Karnataka","devicespec":"{}","city":"Bengaluru","uaspec":"{}","district_custom":"BENGALURU URBAN SOUTH","country_code":"IN","firstaccess":"1571999041881","state_code_custom":"29"}""" val deviceCacheData2 = """{"user_declared_state":"Maharashtra","state_custom":"Maharashtra","devicespec":"{\"scrn\":\"5.46\",\"camera\":\"\",\"idisk\":\"25.44\",\"os\":\"Android 9\",\"id\":\"45f32f48592cb9bcf26bef9178b7bd20abe24932\",\"sims\":\"-1\",\"cpu\":\"abi: armeabi-v7a processor\t: 0 \",\"webview\":\"79.0.3945.116\",\"edisk\":\"25.42\",\"make\":\"Samsung SM-J400F\"}","uaspec":"{\"agent\":\"UNKNOWN\",\"ver\":\"UNKNOWN\",\"system\":\"Android\",\"raw\":\"Dalvik/2.1.0 (Linux U Android 9 SM-J400F Build/PPR1.180610.011)\"}","city":"Mumbai","country_code":"IN","firstaccess":"1578972432419","country":"India","country_name":"India","state":"Maharashtra","continent_name":"Asia","state_code":"MH","fcm_token":"d3ddT88xXLI:APA91bF9lJ4eH8tshAPgKiiZ3hL3sbib0pUN2I388T58oFDxUBQ2WKKuvtBga6iKiOPrgssNKLs4QBjZxE_BbtdGdO0gPdFPataEeXshgYxMKC0VT-oyjrNIZKdKkybQoyichBCiokTD","producer":"sunbirddev.diksha.app","district_custom":"Mumbai","user_declared_district":"Raigad","state_code_custom":"27"}""" + // device data without user declared location fields + val deviceCacheData3 = """{"state_custom":"Maharashtra","devicespec":"{\"scrn\":\"5.46\",\"camera\":\"\",\"idisk\":\"25.44\",\"os\":\"Android 9\",\"id\":\"45f32f48592cb9bcf26bef9178b7bd20abe24932\",\"sims\":\"-1\",\"cpu\":\"abi: armeabi-v7a processor\t: 0 \",\"webview\":\"79.0.3945.116\",\"edisk\":\"25.42\",\"make\":\"Samsung SM-J400F\"}","uaspec":"{\"agent\":\"UNKNOWN\",\"ver\":\"UNKNOWN\",\"system\":\"Android\",\"raw\":\"Dalvik/2.1.0 (Linux U Android 9 SM-J400F Build/PPR1.180610.011)\"}","city":"Mumbai","country_code":"IN","firstaccess":"1578972432419","country":"India","country_name":"India","state":"Maharashtra","continent_name":"Asia","state_code":"MH","fcm_token":"d3ddT88xXLI:APA91bF9lJ4eH8tshAPgKiiZ3hL3sbib0pUN2I388T58oFDxUBQ2WKKuvtBga6iKiOPrgssNKLs4QBjZxE_BbtdGdO0gPdFPataEeXshgYxMKC0VT-oyjrNIZKdKkybQoyichBCiokTD","producer":"sunbirddev.diksha.app","district_custom":"Mumbai","state_code_custom":"27"}""" val userCacheData1 = """{"usersignintype":"Anonymous","usertype":"TEACHER"}""" val userCacheData2 = """{"channel":"KV123","phoneverified":false,"createdby":"c8e51123-61a3-454d-beb0-2202450b0096","subject":["English"],"email":"BJAguqy3GaJECrYqDUPjeducVxa5J9ZsW9A8qc7YHelkV7KbgkCKW10quCbhpgxbh2t4toXC8uXW\\ngiguS+8ucwzbmgPm7q7YSYz26SfpHnzBo/0Vh3TWqr2MOq9LlX6gT6a+wzaAmCWueMEdPmZuRg==","username":"I+CyiN6Bx0GCRm9lkA3xn5uNBm0AODhxeDwJebxxBfuGJ5V2v1R8v1PEQsP+V+y9sAFcM2WtaMLj\\n91hpzBq0PFcQTq6OSPQOm0sySPXTDzyLvm1cKaLwzvJ6fzLLs9nKT6a+wzaAmCWueMEdPmZuRg==","firstname":"A512","framework":{},"userid":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","usertype":"TEACHER","rootorgid":"0126978705345576967","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","language":[],"grade":[],"roles":["BOOK_REVIEWER"],"status":1,"webpages":[],"createddate":"2019-04-11 08:58:16:512+0000","emailverified":true,"isdeleted":false,"locationids":[],"maskedemail":"a5**@yopmail.com","profilevisibility":{},"loginid":"I+CyiN6Bx0GCRm9lkA3xnx2W8+QgN39Y0We3KjR98O8hD6YjyoCirIBDsWHGwRf65PY/Cx+pFFK1\\nIz1VinIaKgDnSQwkl7ajzQjjRTzQbKOyHsAXkJgo9I5l7ulEYVXRT6a+wzaAmCWueMEdPmZuRg==","usersignintype":"Self-Signed-In","userlogintype":"Student","state":"Telangana","district":"Hyderabad"}""" - + val userCacheData3 = """{"channel":"KV123","phoneverified":"false","createdby":"c8e51123-61a3-454d-beb0-2202450b0096","subject":"[\"English\"]","email":"BJAguqy3GaJECrYqDUPjeducVxa5J9ZsW9A8qc7YHelkV7KbgkCKW10quCbhpgxbh2t4toXC8uXW\\ngiguS+8ucwzbmgPm7q7YSYz26SfpHnzBo/0Vh3TWqr2MOq9LlX6gT6a+wzaAmCWueMEdPmZuRg==","username":"I+CyiN6Bx0GCRm9lkA3xn5uNBm0AODhxeDwJebxxBfuGJ5V2v1R8v1PEQsP+V+y9sAFcM2WtaMLj\\n91hpzBq0PFcQTq6OSPQOm0sySPXTDzyLvm1cKaLwzvJ6fzLLs9nKT6a+wzaAmCWueMEdPmZuRg==","firstname":"A512","framework":"{}","userid":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","usertype":"administrator", "usersubtype":"deo,hm","rootorgid":"0126978705345576967","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","language":"","grade":"","roles":"[\"BOOK_REVIEWER\"]","status":"1","webpages":"[]","createddate":"2019-04-11 08:58:16:512+0000","emailverified":"true","isdeleted":"false","locationids":"[\"location-1\",\"location-2\",\"location-3\"]","maskedemail":"a5**@yopmail.com","profilevisibility":"{}","loginid":"I+CyiN6Bx0GCRm9lkA3xnx2W8+QgN39Y0We3KjR98O8hD6YjyoCirIBDsWHGwRf65PY/Cx+pFFK1\\nIz1VinIaKgDnSQwkl7ajzQjjRTzQbKOyHsAXkJgo9I5l7ulEYVXRT6a+wzaAmCWueMEdPmZuRg==","usersignintype":"Self-Signed-In","userlogintype":"Student","state":"Telangana","district":"Hyderabad","cluster":"Cluster001","block":"Sri Sai ACC Block","schoolname":"\\[RPMMAT M.S UDHADIH"}""" + val userCacheDataMap1: util.HashMap[String, String] = gson.fromJson(userCacheData1, new util.HashMap[String, String]().getClass) + val userCacheDataMap2: util.HashMap[String, String] = gson.fromJson(userCacheData3, new util.HashMap[String, String]().getClass) + val contentCacheData1 = """{"code":"org.ekstep.literacy.story.21797","keywords":["Story"],"subject":["English"],"channel":"in.ekstep","description":"Write a short description of your lesson","language":["English"],"mimeType":"application/vnd.ekstep.ecml-archive","idealScreenSize":"normal","createdOn":"2018-04-26T09:59:27.336+0000","objectType":"Content","gradeLevel":["Class 9"],"appId":"ekstep_portal","contentDisposition":"inline","contentEncoding":"gzip","lastUpdatedOn":1571999041881,"lastSubmittedOn":"2018-06-15T13:06:56.090+0000","lastPublishedOn":1571999041881,"contentType":"Resource","lastUpdatedBy":"7224","identifier":"do_31249064359802470412856","audience":["Learner"],"creator":"Pravin Panpatil","os":["All"],"visibility":"Default","consumerId":"62e15662-bb09-439f-86e2-d65bd84f3c23","mediaType":"content","osId":"org.ekstep.quiz.app","graph_id":"domain","nodeType":"DATA_NODE","versionKey":"1524736767336","idealScreenDensity":"hdpi","framework":"NCF","createdBy":"7224","compatibilityLevel":1.0,"domain":["literacy"],"name":"Walk a little slower","board":"State (Maharashtra)","resourceType":"Read","status":"Draft","node_id":310297.0,"license":"CC BY 4.0","copyright":"Ekstep","author":"Ekstep","copyrightYear":2019.0}""" val contentCacheData2 = """{"identifier":"do_312526125187809280139353","code":"c361b157-408c-4347-9be3-38d9d4ea2b90","visibility":"Parent","description":"Anction Words","mimeType":"application/vnd.ekstep.content-collection","graph_id":"domain","nodeType":"DATA_NODE","createdOn":"2018-06-15T13:06:56.090+0000","versionKey":"1529068016090","objectType":"Content","dialcodes":["TNPF7T"],"collections":["do_312526125186424832139255"],"name":"Anction Words","lastUpdatedOn":"2018-06-15T13:06:56.090+0000","lastsubmittedon":1571999041881,"lastPublishedOn":"2018-06-15T13:06:56.090+0000","contentType":"TextBookUnit","status":"Draft","node_id":438622.0,"license":"CC BY 4.0"}""" val contentCacheData3 = """{"code":"do_312526125187809280139355","channel":"in.ekstep","downloadUrl":"https://ekstep-public-prod.s3-ap-south-1.amazonaws.com/content/14681653089315c3173a3e1.mp3","language":["English"],"mimeType":"application/octet-stream","idealScreenSize":"normal","createdOn":"2016-07-10T15:41:49.111+0000","objectType":"Content","gradeLevel":["Class 1"],"contentDisposition":"inline","contentEncoding":"identity","lastUpdatedOn":"2017-03-10T18:10:00.448+0000","contentType":"Asset","identifier":"do_312526125187809280139355","os":["All"],"visibility":"Default","mediaType":"audio","osId":"org.ekstep.launcher","ageGroup":["5-6"],"graph_id":"domain","nodeType":"DATA_NODE","pkgVersion":1.0,"versionKey":"1489169400448","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCF","compatibilityLevel":1.0,"name":"do_312526125187809280139355","status":"Live","node_id":65545.0,"size":677510.0}""" - + val contentCacheData4 = """{"code":"org.sunbird.XiKRjo","channel":"0123221617357783046602","description":"गणित तीसरी कक्षा","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","medium":["Hindi"],"idealScreenSize":"normal","createdOn":"2018-06-12T08:26:22.841+0000","objectType":"Content","gradeLevel":["Class 3"],"children":["do_312523863926120448117914","do_312523863926120448117918","do_312523863926120448117917","do_312523863926120448117916","do_312523863926120448117915","do_312523863926128640117922","do_312523863926128640117921","do_312523863926128640117920","do_312523863926128640117919","do_312523863926136832117926","do_312523863926136832117925","do_312523863926136832117924","do_312523863926128640117923","do_312523863926153216117930","do_312523863926145024117929","do_312523863926145024117928","do_312523863926145024117927","do_312523863926161408117934","do_312523863926161408117933","do_312523863926153216117932","do_312523863926153216117931","do_312523863926169600117938","do_312523863926161408117937","do_312523863926161408117936","do_312523863926161408117935","do_312523863926177792117942","do_312523863926169600117941","do_312523863926169600117940","do_312523863926169600117939","do_312523863926177792117946","do_312523863926177792117945","do_312523863926177792117944","do_312523863926177792117943","do_312523863926185984117950","do_312523863926185984117949","do_312523863926185984117948","do_312523863926185984117947","do_312523863926194176117954","do_312523863926194176117953","do_312523863926194176117952","do_312523863926185984117951","do_312523863926202368117958","do_312523863926202368117957","do_312523863926202368117956","do_312523863926194176117955","do_312523863926210560117962","do_312523863926210560117961","do_312523863926210560117960","do_312523863926202368117959","do_312523863926218752117966","do_312523863926218752117965","do_312523863926218752117964","do_312523863926210560117963","do_312523863926226944117970","do_312523863926226944117969","do_312523863926218752117968","do_312523863926218752117967","do_312523863926235136117974","do_312523863926226944117973","do_312523863926226944117972","do_312523863926226944117971"],"appId":"prod.diksha.portal","contentDisposition":"inline","contentEncoding":"gzip","lastUpdatedOn":"2018-06-12T08:31:38.614+0000","contentType":"TextBook","identifier":"do_312523863923441664117896","createdFor":["0123221617357783046602"],"audience":["Learner"],"os":["All"],"visibility":"Default","consumerId":"0aa13c48-dda0-4259-9007-c795dacd7b9c","mediaType":"content","osId":"org.ekstep.quiz.app","graph_id":"domain","nodeType":"DATA_NODE","versionKey":"1528792298614","idealScreenDensity":"hdpi","framework":"mh_k-12_1","createdBy":"80ae760d-f9dd-43a9-91a7-09dfabba2293","compatibilityLevel":1.0,"name":"test","board":"State (Maharashtra)","resourceType":"Book","status":"Retired","node_id":398184.0,"license":"CC BY 4.0","copyright":"MITRA","author":"MITRA","copyrightYear":2019.0}""" + val contentCacheData5 = """{"identifier":"7426472e-8b1a-4387-8b7a-962cb6cda006","code":"c361b157-408c-4347-9be3-38d9d4ea2b90","visibility":"Parent","description":"Anction Words","mimeType":"application/vnd.ekstep.content-collection","graph_id":"domain","nodeType":"DATA_NODE","createdOn":"2018-06-15T13:06:56.090+0000","versionKey":"1529068016090","objectType":"Content","dialcodes":["TNPF7T"],"collections":["do_312526125186424832139255"],"name":"Anction Words","lastUpdatedOn":"2018-06-15T13:06:56.090+0000","lastsubmittedon":1571999041881,"lastPublishedOn":"2018-06-15T13:06:56.090+0000","contentType":"TextBookUnit","status":"Draft","node_id":438622.0,"license":"CC BY 4.0"}""" + + val collectionCache1 = """{"identifier":"do_31331086175718604812701","code":"c361b157-408c-4347-9be3-38d9d4ea2b90","visibility":"Parent","description":"Anction Words","mimeType":"application/vnd.ekstep.content-collection","graph_id":"domain","nodeType":"DATA_NODE","createdOn":"2018-06-15T13:06:56.090+0000","versionKey":"1529068016090","objectType":"Content","dialcodes":["TNPF7T"],"collections":["do_312526125186424832139255"],"name":"Anction Words","lastUpdatedOn":"2018-06-15T13:06:56.090+0000","lastsubmittedon":1571999041881,"lastPublishedOn":"2018-06-15T13:06:56.090+0000","contentType":"TextBookUnit","status":"Draft","node_id":438622.0,"license":"CC BY 4.0"}""" + val dialcodeCacheData1 = """{"identifier":"GWNI38","batchcode":"jkpublisher.20180801T134105","channel":"01254592085869363222","dialcode_index":2328993,"generated_on":"2018-08-01T13:41:53.695","published_on":1571999041881,"publisher":"jkpublisher","status":"Draft"}""" val dialcodeCacheData2 = """{"identifier":"PCZKA3","batchcode":"jkpublisher.20180801T122031","channel":"01254592085869363222","dialcode_index":1623464,"generated_on":1571999041881,"published_on":"2018-08-01T13:41:53.695Z","publisher":"jkpublisher","status":"Draft"}""" @@ -22,6 +34,15 @@ object EventFixture { val currentDate: Long = DateTime.now().getMillis val olderDate: Long = DateTime.now().minusMonths(5).getMillis val futureDate: Long = DateTime.now().plusMonths(1).getMillis + + val telemetryEvent: String = s"""{"actor":{"type":"User","id":"b7470841-7451-43db-b5c7-2dcf4f8d3b23"},"eid":"INTERACT", + |"edata":{"type":"OTHER","subtype":"sheen-animation-ended","id":"library","pageid":"library","extra":{"pos":[]}}, + |"ver":"3.0","syncts":1.579564974098E12,"@timestamp":"2020-01-21T00:02:54.098Z","ets":$currentDate, + |"context":{"cdata":[],"env":"home","channel":"505c7c48ac6dc1edc9b08f21db5a571d", + |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272", + |"did":"45f32f48592cb9bcf26bef9178b7bd20abe24932"},"flags":{"dd_processed":true}, + |"mid":"mid1","type":"events","object":{"id":"","type":"", + |"version":"","rollup":{}}}""".stripMargin val telemetrEvents: List[String] = List( @@ -96,7 +117,7 @@ object EventFixture { |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272", |"did":""},"flags":{"dd_processed":true}, |"mid":"mid8","type":"events","object":{"id":"do_312526125187809280139352","type":"Content", - |"version":"","rollup":{"l1":"do_312526125187809280139353"}}}""".stripMargin, + |"version":"","rollup":{"l1":"do_312526125187809280139353","l2":"do_312523863923441664117896"}}}""".stripMargin, // Future date s"""{"actor":{"type":"System","id":"anonymous"},"eid":"INTERACT", @@ -126,8 +147,68 @@ object EventFixture { |"cdata":[],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_31249064359802470412856","type":"content","ver":"","rollup":{}}, |"tags":["505c7c48ac6dc1edc9b08f21db5a571d"],"edata":{"state":"COMPLETED","prevstate":"INPROGRESS", |"props":["stats.downloadedSize","status","updatedOn"],"duration":0.8},"syncts":1.58156611767E12,"@timestamp":"2020-02-13T03:55:17.670Z", - |"contentdata":{"objectType":"Content"},"userdata":{"firstname":"A512"},"type":"events"}""".stripMargin - + |"contentdata":{"objectType":"Content"},"userdata":{"firstname":"A512"},"type":"events"}""".stripMargin, + + s"""{"actor":{"type":"User","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc82"},"eid":"INTERRUPT", + |"edata":{"type":"OTHER","subtype":"sheen-animation-ended","id":"library","pageid":"library","extra":{"pos":[]}}, + |"ver":"3.0","syncts":1.579564974098E12,"@timestamp":"2020-01-21T00:02:54.098Z","ets":$currentDate, + |"context":{"cdata":[],"env":"home","channel":"505c7c48ac6dc1edc9b08f21db5a571d", + |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272", + |"did":"45f32f48592cb9bcf26bef9178b7bd20abe24932"},"flags":{"dd_processed":true}, + |"mid":"mid6","type":"events","object":{"id":"PCZKA4","type":"qr", + |"version":"","rollup":{}}}""".stripMargin, + + // event without did - INTERRUPT skipped + s"""{"actor":{"type":"User","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc82"},"eid":"INTERRUPT", + |"edata":{"type":"OTHER","subtype":"sheen-animation-ended","id":"library","pageid":"library","extra":{"pos":[]}}, + |"ver":"3.0","syncts":1.579564974098E12,"@timestamp":"2020-01-21T00:02:54.098Z","ets":$currentDate, + |"context":{"cdata":[],"env":"home","channel":"505c7c48ac6dc1edc9b08f21db5a571d", + |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272"}, + |"flags":{"dd_processed":true}, + |"mid":"mid11","type":"events","object":{"id":"PCZKA4","type":"qr", + |"version":"","rollup":{}}}""".stripMargin, + + // event without did + s"""{"actor":{"type":"User","id":"b7470841-7451-43db-b5c7-2dcf4f8d3b23"},"eid":"INTERACT", + |"edata":{"type":"OTHER","subtype":"sheen-animation-ended","id":"library","pageid":"library","extra":{"pos":[]}}, + |"ver":"3.0","syncts":1.579564974098E12,"@timestamp":"2020-01-21T00:02:54.098Z","ets":$currentDate, + |"context":{"cdata":[],"env":"home","channel":"505c7c48ac6dc1edc9b08f21db5a571d", + |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272"}, + |"flags":{"dd_processed":true}, + |"mid":"mid12","type":"events","object":{"id":"","type":"", + |"version":"","rollup":{}}}""".stripMargin, + + // SUMMARY event - user_denorm = true, device_denorm=true, content_denorm=true + s"""{"eid":"SUMMARY","ets":$currentDate,"ver":"3.0","mid":"SUMMARY:a3e517153c4ba392297e70521aa5e17a", + |"actor":{"id":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","type":"User"}, + |"context":{"channel":"01268904781886259221","pdata":{"id":"staging.sunbird.portal","ver":"4.1.0", + |"pid":"sunbird-portal"},"env":"contentplayer","sid":"73d82044-8ea5-dffc-1af5-6cdf2a1fa1da", + |"did":"264d679186d4b0734d858d4e18d4d31e","cdata":[{"id":"kubXMwcsJK2JANa0PeYc00GK5CSXoS1q","type":"ContentSession"}, + |{"id":"xcFG0rntKUGltu2m8zJh7ZqattT9u3Ix","type":"PlaySession"},{"id":"2.0","type":"PlayerVersion"}], + |"rollup":{"l1":"01268904781886259221"},"uid":"anonymous"},"object":{"id":"do_31249064359802470412856","ver":"1","type":"Content","rollup":{}}, + |"tags":["01268904781886259221"],"edata":{"type":"content","mode":"play","starttime":1625043385301, + |"endtime":1625043401236,"timespent":16,"pageviews":2,"interactions":2,"extra":[{"id":"progress","value":"100"}, + |{"id":"endpageseen","value":"true"},{"id":"score","value":"2"},{"id":"correct","value":"2"}, + |{"id":"incorrect","value":"0"},{"id":"partial","value":"0"},{"id":"skipped","value":"0"}]}}""".stripMargin, + + // ME_DEVICE_SUMMARY event - should skip + s""" + |{"eid":"ME_DEVICE_SUMMARY","ets":$currentDate,"syncts":$currentDate,"ver":"1.0","mid": + |"CFBA22543AA3EAD4C2737931D34F2E8D","context":{"pdata":{"id":"AnalyticsDataPipeline","ver":"1.0", + |"model":"DeviceSummary"},"granularity":"DAY","date_range":{"from":1572786370125,"to":1572786403121}}, + |"dimensions":{"did":"3eb8d5dc49b063650ca18920956ea04e","channel":"ROOT_ORG"},"edata":{"eks":{"firstAccess": + |1572786370121,"dial_stats":{"total_count":3,"success_count":3,"failure_count":0},"content_downloads":0, + |"contents_played":0,"total_ts":0.0,"total_launches":0,"unique_contents_played":0}}} + """.stripMargin, + //SB-25755: if enrol-complete , denorm the event + s""" + |{"actor":{"id":"7426472e-8b1a-4387-8b7a-962cb6cda006","type":"User"},"eid":"AUDIT","edata":{"props":["status","completedon"], + |"type":"enrol-complete"},"ver":"3.0","syncts":1626346241621,"ets":$currentDate,"context":{"channel":"in.sunbird", + |"env":"Course","sid":"2ba8a10a-9722-42c6-a27f-f9fd86ff6bb5","did":"0a65cbaf-2d0e-4cc8-bc70-6ee1e71fb605","pdata": + |{"ver":"3.0","id":"org.sunbird.learning.platform","pid":"course-progress-updater"},"cdata":[{"type":"CourseBatch","id":"01331092647782809655"}, + |{"type":"Course","id":"do_31331086175718604812701"}]},"mid":"LP.AUDIT.d14d8be6-da4e-4ee9-b833-fd86d57b8808", + |"object":{"id":"7426472e-8b1a-4387-8b7a-962cb6cda006","type":"User","rollup":{"l1":"do_31331086175718604812701"}},"tags":[]} + """.stripMargin ) val summaryEvents: List[String] = List( diff --git a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/CheckDerivedLocationStreamTaskTestSpec.scala b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/CheckDerivedLocationStreamTaskTestSpec.scala new file mode 100644 index 0000000000..45cbab0c47 --- /dev/null +++ b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/CheckDerivedLocationStreamTaskTestSpec.scala @@ -0,0 +1,118 @@ +package org.sunbird.dp.spec + +import com.google.gson.Gson +import com.typesafe.config.{Config, ConfigFactory} +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration +import org.apache.flink.streaming.api.functions.sink.SinkFunction +import org.apache.flink.streaming.api.functions.source.SourceFunction +import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext +import org.apache.flink.test.util.MiniClusterWithClientResource +import org.mockito.Mockito +import org.mockito.Mockito.when +import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} +import org.sunbird.dp.core.cache.RedisConnect +import org.sunbird.dp.core.job.FlinkKafkaConnector +import org.sunbird.dp.denorm.domain.Event +import org.sunbird.dp.denorm.task.{DenormalizationConfig, DenormalizationStreamTask} +import org.sunbird.dp.fixture.EventFixture +import redis.embedded.RedisServer + +import java.util + +class CheckDerivedLocationStreamTaskTestSpec extends BaseTestSpec { + implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + + val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() + .setConfiguration(testConfiguration()) + .setNumberSlotsPerTaskManager(1) + .setNumberTaskManagers(1) + .build) + + var redisServer: RedisServer = _ + val config: Config = ConfigFactory.load("test.conf") + val denormConfig: DenormalizationConfig = new DenormalizationConfig(config, "DenormTest") + val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) + val gson = new Gson() + + override protected def beforeAll(): Unit = { + super.beforeAll() + redisServer = new RedisServer(6341) + redisServer.start() + + BaseMetricsReporter.gaugeMetrics.clear() + + setupRedisTestData() + flinkCluster.before() + } + + override protected def afterAll(): Unit = { + super.afterAll() + redisServer.stop() + flinkCluster.after() + } + + def setupRedisTestData() { + + val redisConnect = new RedisConnect(denormConfig.metaRedisHost, denormConfig.metaRedisPort, denormConfig) + + // Insert device test data + var jedis = redisConnect.getConnection(denormConfig.deviceStore) + jedis.hmset("45f32f48592cb9bcf26bef9178b7bd20abe24932", gson.fromJson(EventFixture.deviceCacheData3, new util.HashMap[String, String]().getClass)) + jedis.close() + + } + + "De-normalization pipeline" should "denormalize location metadata properly for blank user declared scenario" in { + + when(mockKafkaUtil.kafkaEventSource[Event](denormConfig.telemetryInputTopic)).thenReturn(new InputSource1) + when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.telemetryDenormOutputTopic)).thenReturn(new DenormEventsSink1) + + val task = new DenormalizationStreamTask(denormConfig, mockKafkaUtil) + task.process() + + val event = DenormEventsSink1.values("mid1") + event.flags().get("loc_denorm").asInstanceOf[Boolean] should be (true) + // derived location should be from ip-resolved + event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("district") should be("Mumbai") + event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("state") should be("Maharashtra") + event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("from") should be("ip-resolved") + + val device3Data = event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]] + device3Data.get("firstaccess") should be (1578972432419L) + val deviceSpecData = device3Data.get("devicespec").asInstanceOf[util.Map[String, Any]] + deviceSpecData.get("edisk") should be ("25.42") + deviceSpecData.get("make") should be ("Samsung SM-J400F") + // user declared is empty + val userDeclaredLocationData = device3Data.get("userdeclared").asInstanceOf[util.Map[String, Any]] + userDeclaredLocationData.get("district") should be ("") + userDeclaredLocationData.get("state") should be ("") + + } + +} + +class InputSource1 extends SourceFunction[Event] { + + override def run(ctx: SourceContext[Event]) { + val gson = new Gson() + val eventMap = gson.fromJson(EventFixture.telemetryEvent, new util.HashMap[String, Any]().getClass) + ctx.collect(new Event(eventMap)) + } + + override def cancel() = {} +} + +class DenormEventsSink1 extends SinkFunction[Event] { + + override def invoke(event: Event): Unit = { + synchronized { + DenormEventsSink1.values.put(event.mid(), event) + } + } +} + +object DenormEventsSink1 { + val values: scala.collection.mutable.Map[String, Event] = scala.collection.mutable.Map[String, Event]() +} \ No newline at end of file diff --git a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/DenormalizationStreamTaskTestSpec.scala b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/DenormalizationStreamTaskTestSpec.scala index 3bf3477734..7fd2533d0f 100644 --- a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/DenormalizationStreamTaskTestSpec.scala +++ b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/DenormalizationStreamTaskTestSpec.scala @@ -21,6 +21,8 @@ import org.sunbird.dp.fixture.EventFixture import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} import redis.embedded.RedisServer +import scala.collection.JavaConverters._ + class DenormalizationStreamTaskTestSpec extends BaseTestSpec { implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) @@ -66,8 +68,8 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec { // Insert user test data jedis = redisConnect.getConnection(denormConfig.userStore) - jedis.set("b7470841-7451-43db-b5c7-2dcf4f8d3b23", EventFixture.userCacheData1) - jedis.set("610bab7d-1450-4e54-bf78-c7c9b14dbc81", EventFixture.userCacheData2) + jedis.hmset(denormConfig.userStoreKeyPrefix + "b7470841-7451-43db-b5c7-2dcf4f8d3b23", EventFixture.userCacheDataMap1) + jedis.hmset(denormConfig.userStoreKeyPrefix + "610bab7d-1450-4e54-bf78-c7c9b14dbc81", EventFixture.userCacheDataMap2) jedis.close() // Insert dialcode test data @@ -81,6 +83,9 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec { jedis.set("do_31249064359802470412856", EventFixture.contentCacheData1) jedis.set("do_312526125187809280139353", EventFixture.contentCacheData2) jedis.set("do_312526125187809280139355", EventFixture.contentCacheData3) + jedis.set("do_312523863923441664117896", EventFixture.contentCacheData4) + jedis.set("7426472e-8b1a-4387-8b7a-962cb6cda006", EventFixture.contentCacheData5) + jedis.set("do_31331086175718604812701", EventFixture.collectionCache1) jedis.close() } @@ -88,21 +93,26 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec { "De-normalization pipeline" should "denormalize content, user, device and location metadata" in { when(mockKafkaUtil.kafkaEventSource[Event](denormConfig.telemetryInputTopic)).thenReturn(new InputSource) - when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.denormSuccessTopic)).thenReturn(new DenormEventsSink) + when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.telemetryDenormOutputTopic)).thenReturn(new DenormEventsSink) val task = new DenormalizationStreamTask(denormConfig, mockKafkaUtil) task.process() - DenormEventsSink.values.size should be (10) + + DenormEventsSink.values.size should be (13) DenormEventsSink.values.get("mid10") should be (None) var event = DenormEventsSink.values("mid1") event.kafkaKey() should be ("758e054a400f20f7677f2def76427dc13ad1f837") - event.flags().get("device_denorm").asInstanceOf[Boolean] should be (false) event.flags().get("user_denorm").asInstanceOf[Boolean] should be (true) Option(event.flags().get("dialcode_denorm")) should be (None) Option(event.flags().get("content_denorm")) should be (None) Option(event.flags().get("location_denorm")) should be (None) + + val user1Data = event.getMap().get("userdata").asInstanceOf[util.Map[String, Any]] + user1Data.get("usersignintype") should be("Anonymous") + user1Data.get("usertype") should be("TEACHER") + user1Data.get("userlogintype") should be("NA") event = DenormEventsSink.values("mid2") event.flags().get("device_denorm").asInstanceOf[Boolean] should be (true) @@ -113,14 +123,54 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec { Option(event.flags().get("coll_denorm")) should be (None) event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("lastsubmittedon") should be(1529068016090L) - + event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("channel") should be("in.ekstep") + event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("lastpublishedon") should be(1.571999041881E12) + event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("contenttype") should be("Resource") + event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("keywords").asInstanceOf[util.ArrayList[String]].get(0) should be ("Story") + + event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]].get("statecustomcode") should be("29") + event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]].get("countrycode") should be("IN") + event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]].get("firstaccess") should be(1571999041881L) + event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]].get("districtcustom") should be("BENGALURU URBAN SOUTH") + + val user2Data = event.getMap().get("userdata").asInstanceOf[util.Map[String, Any]] + user2Data.get("userlogintype") should be("Student") + user2Data.get("usersignintype") should be("Self-Signed-In") + user2Data.get("usertype") should be("administrator") + user2Data.get("usersubtype") should be("deo,hm") + user2Data.get("subject").asInstanceOf[util.List[String]].asScala should be(List("English")) + user2Data.get("state") should be("Telangana") + user2Data.get("cluster") should be("Cluster001") + user2Data.get("schoolname") should be("[RPMMAT M.S UDHADIH") + user2Data.get("block") should be ("Sri Sai ACC Block") + event = DenormEventsSink.values("mid3") event.flags().get("device_denorm").asInstanceOf[Boolean] should be (true) event.flags().get("user_denorm").asInstanceOf[Boolean] should be (false) event.flags().get("content_denorm").asInstanceOf[Boolean] should be (true) event.flags().get("coll_denorm").asInstanceOf[Boolean] should be (true) event.flags().get("loc_denorm").asInstanceOf[Boolean] should be (true) - + + event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("contenttype") should be("Asset") + Option(event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("contentType")) should be (None) + event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("contenttype") should be("Asset") + event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("framework") should be("NCF") + event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("name") should be("do_312526125187809280139355") + event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("lastupdatedon") should be(1489169400448L) + + event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("district") should be("Raigad") + event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("state") should be("Maharashtra") + event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("from") should be("user-declared") + + val device3Data = event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]] + device3Data.get("firstaccess") should be (1578972432419L) + val deviceSpecData = device3Data.get("devicespec").asInstanceOf[util.Map[String, Any]] + deviceSpecData.get("edisk") should be ("25.42") + deviceSpecData.get("make") should be ("Samsung SM-J400F") + val userDeclaredLocationData = device3Data.get("userdeclared").asInstanceOf[util.Map[String, Any]] + userDeclaredLocationData.get("district") should be ("Raigad") + userDeclaredLocationData.get("state") should be ("Maharashtra") + event = DenormEventsSink.values("mid4") event.flags().get("device_denorm").asInstanceOf[Boolean] should be (true) event.flags().get("user_denorm").asInstanceOf[Boolean] should be (true) @@ -137,26 +187,63 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec { Option(event.flags().get("content_denorm")) should be (None) Option(event.flags().get("location_denorm")) should be (None) + event.getMap().get("dialcodedata").asInstanceOf[util.Map[String, Any]].get("batchcode") should be("jkpublisher.20180801T122031") + event.getMap().get("dialcodedata").asInstanceOf[util.Map[String, Any]].get("channel") should be("01254592085869363222") + event.getMap().get("dialcodedata").asInstanceOf[util.Map[String, Any]].get("generatedon") should be(1.571999041881E12) + event.getMap().get("dialcodedata").asInstanceOf[util.Map[String, Any]].get("publishedon") should be(1533130913695L) + // TODO: Complete the assertions event = DenormEventsSink.values("mid6") event = DenormEventsSink.values("mid7") event = DenormEventsSink.values("mid8") + event.flags().get("coll_denorm").asInstanceOf[Boolean] should be (true) + event.flags().get("l2_denorm").asInstanceOf[Boolean] should be (true) + val l2Data = event.getMap().get("l2data").asInstanceOf[util.Map[String, Any]] + l2Data should not be null + + l2Data.get("contenttype") should be("TextBook") + l2Data.get("mimetype") should be("application/vnd.ekstep.content-collection") + l2Data.get("contenttype") should be("TextBook") + l2Data.get("channel") should be("0123221617357783046602") + l2Data.get("board") should be("State (Maharashtra)") + l2Data.get("name") should be("test") + l2Data.get("framework") should be("mh_k-12_1") event = DenormEventsSink.values("mid9") + event = DenormEventsSink.values("LP.AUDIT.d14d8be6-da4e-4ee9-b833-fd86d57b8808") + event.flags().get("device_denorm").asInstanceOf[Boolean] should be (false) + event.flags().get("user_denorm").asInstanceOf[Boolean] should be (false) + event.flags().get("coll_denorm").asInstanceOf[Boolean] should be (true) + event.flags().get("content_denorm").asInstanceOf[Boolean] should be (true) + + val contentData = event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]] + contentData.get("name") should be ("Anction Words") + contentData.get("mimetype") should be ("application/vnd.ekstep.content-collection") + contentData.get("objecttype") should be ("Content") + contentData.get("contenttype") should be ("TextBookUnit") + contentData.get("status") should be ("Draft") + + val collData = event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]] + collData.get("name") should be ("Anction Words") + collData.get("mimetype") should be ("application/vnd.ekstep.content-collection") + collData.get("objecttype") should be ("Content") + collData.get("contenttype") should be ("TextBookUnit") + collData.get("status") should be ("Draft") + // Location Denorm Metrics Assertion - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheHit}").getValue() should be (7) - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheMiss}").getValue() should be (3) - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locTotal}").getValue() should be (10) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheHit}").getValue() should be (8) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheMiss}").getValue() should be (5) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locTotal}").getValue() should be (13) // Content Denorm Metrics Assertion - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentCacheHit}").getValue() should be (4) - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentCacheMiss}").getValue() should be (3) - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentTotal}").getValue() should be (7) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentCacheHit}").getValue() should be (6) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentCacheMiss}").getValue() should be (2) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentTotal}").getValue() should be (8) // User Denorm Metrics Assertion - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheHit}").getValue() should be (3) - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheMiss}").getValue() should be (4) - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userTotal}").getValue() should be (7) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheHit}").getValue() should be (5) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheMiss}").getValue() should be (5) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userTotal}").getValue() should be (10) // Dialcode Denorm Metrics Assertion BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.dialcodeCacheHit}").getValue() should be (2) @@ -164,11 +251,12 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec { BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.dialcodeTotal}").getValue() should be (3) // Device Denorm Metrics Assertion - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceCacheHit}").getValue() should be (7) - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceCacheMiss}").getValue() should be (2) - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceTotal}").getValue() should be (9) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceCacheHit}").getValue() should be (8) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceCacheMiss}").getValue() should be (3) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceTotal}").getValue() should be (11) BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.eventsExpired}").getValue() should be (1) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.eventsSkipped}").getValue() should be (3) // Skipped INTERRUPT event } diff --git a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/SummaryDenormalizationStreamTaskTestSpec.scala b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/SummaryDenormalizationStreamTaskTestSpec.scala index 99db935d88..8f55181288 100644 --- a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/SummaryDenormalizationStreamTaskTestSpec.scala +++ b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/SummaryDenormalizationStreamTaskTestSpec.scala @@ -21,6 +21,8 @@ import org.sunbird.dp.fixture.EventFixture import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} import redis.embedded.RedisServer +import scala.collection.JavaConverters._ + class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec { implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) @@ -66,8 +68,8 @@ class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec { // Insert user test data jedis = redisConnect.getConnection(denormConfig.userStore) - jedis.set("b7470841-7451-43db-b5c7-2dcf4f8d3b23", EventFixture.userCacheData1) - jedis.set("610bab7d-1450-4e54-bf78-c7c9b14dbc81", EventFixture.userCacheData2) + jedis.hmset(denormConfig.userStoreKeyPrefix + "b7470841-7451-43db-b5c7-2dcf4f8d3b23", EventFixture.userCacheDataMap1) + jedis.hmset(denormConfig.userStoreKeyPrefix + "610bab7d-1450-4e54-bf78-c7c9b14dbc81", EventFixture.userCacheDataMap2) jedis.close() // Insert dialcode test data @@ -88,8 +90,8 @@ class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec { "Summary Denormalization pipeline" should "denormalize content, user, device and location metadata for summary events" in { when(mockKafkaUtil.kafkaEventSource[Event](denormConfig.summaryInputTopic)).thenReturn(new SummaryInputSource) - when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.denormSuccessTopic)).thenReturn(new SummaryDenormEventsSink) - when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.summaryOutputEventsTopic)).thenReturn(new SummaryEventsSink) + when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.summaryDenormOutputTopic)).thenReturn(new SummaryDenormEventsSink) + when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.summaryUniqueEventsTopic)).thenReturn(new SummaryEventsSink) when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.duplicateTopic)).thenReturn(new DuplicateEventsSink) val task = new SummaryDenormalizationStreamTask(denormConfig, mockKafkaUtil) @@ -102,19 +104,35 @@ class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec { event1.kafkaKey() should be ("45f32f48592cb9bcf26bef9178b7bd20abe24932") event1.flags().get("device_denorm").asInstanceOf[Boolean] should be (true) - event1.flags().get("user_denorm").asInstanceOf[Boolean] should be (false) + event1.flags().get("user_denorm").asInstanceOf[Boolean] should be (true) Option(event1.flags().get("dialcode_denorm")) should be (None) Option(event1.flags().get("content_denorm")) should be (None) Option(event1.flags().get("location_denorm")) should be (None) + val event1UserData = event1.getTelemetry.read[util.HashMap[String, AnyRef]]("userdata").getOrElse(new util.HashMap()).asScala + event1UserData("usersignintype").asInstanceOf[String] should be ("Anonymous") + event1UserData("userlogintype").asInstanceOf[String] should be ("NA") + event1UserData("usertype").asInstanceOf[String] should be ("TEACHER") val event2 = SummaryDenormEventsSink.values("mid2") event2.flags().get("device_denorm").asInstanceOf[Boolean] should be (true) - event2.flags().get("user_denorm").asInstanceOf[Boolean] should be (false) + event2.flags().get("user_denorm").asInstanceOf[Boolean] should be (true) Option(event2.flags().get("dialcode_denorm")) should be (None) event2.flags().get("content_denorm").asInstanceOf[Boolean] should be (true) event2.flags().get("loc_denorm").asInstanceOf[Boolean] should be (true) Option(event2.flags().get("coll_denorm")) should be (Some(true)) + val event2UserData = event2.getTelemetry.read[util.HashMap[String, AnyRef]]("userdata").getOrElse(new util.HashMap()).asScala + event2UserData("usersignintype") should be ("Self-Signed-In") + event2UserData("userlogintype") should be ("Student") + event2UserData("usertype") should be ("administrator") + event2UserData("usersubtype") should be ("deo,hm") + event2UserData("subject").asInstanceOf[util.List[String]].asScala should be(List("English")) + event2UserData("cluster") should be ("Cluster001") + event2UserData("schoolname") should be ("[RPMMAT M.S UDHADIH") + event2UserData("block") should be ("Sri Sai ACC Block") + + val event2ContentData = event2.getTelemetry.read[util.HashMap[String, AnyRef]]("contentdata").getOrElse(new util.HashMap()).asScala + event2ContentData.get("keywords").get.asInstanceOf[util.ArrayList[String]].get(0) should be ("Story") BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheHit}").getValue() should be (2) BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheMiss}").getValue() should be (0) @@ -126,9 +144,9 @@ class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec { BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentTotal}").getValue() should be (1) // User Denorm Metrics Assertion - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheHit}").getValue() should be (0) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheHit}").getValue() should be (2) BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheMiss}").getValue() should be (0) - BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userTotal}").getValue() should be (0) + BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userTotal}").getValue() should be (2) // Dialcode Denorm Metrics Assertion BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.dialcodeCacheHit}").getValue() should be (0) diff --git a/data-pipeline-flink/device-profile-updater/pom.xml b/data-pipeline-flink/device-profile-updater/pom.xml index d047f265cf..a10f9b54c0 100644 --- a/data-pipeline-flink/device-profile-updater/pom.xml +++ b/data-pipeline-flink/device-profile-updater/pom.xml @@ -30,7 +30,7 @@ org.apache.flink - flink-streaming-scala_${scala.version} + flink-streaming-scala_${scala.maj.version} ${flink.version} provided @@ -54,13 +54,13 @@ org.apache.flink flink-test-utils_2.12 - 1.10.0 + ${flink.version} test org.apache.flink flink-runtime_2.12 - 1.10.0 + ${flink.version} test tests @@ -73,7 +73,7 @@ org.apache.flink flink-streaming-java_2.12 - 1.10.0 + ${flink.version} test tests @@ -107,6 +107,14 @@ src/main/scala src/test/scala + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + org.apache.maven.plugins maven-shade-plugin @@ -157,10 +165,11 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 4.4.0 - 1.8 - 1.8 + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} false diff --git a/data-pipeline-flink/device-profile-updater/src/main/resources/log4j.properties b/data-pipeline-flink/device-profile-updater/src/main/resources/log4j.properties deleted file mode 100644 index dda4b21056..0000000000 --- a/data-pipeline-flink/device-profile-updater/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=telemetry-extractor.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/data-pipeline-flink/dp-core/pom.xml b/data-pipeline-flink/dp-core/pom.xml index b1eb6d8442..cfe8164c5f 100644 --- a/data-pipeline-flink/dp-core/pom.xml +++ b/data-pipeline-flink/dp-core/pom.xml @@ -20,13 +20,13 @@ org.apache.flink - flink-streaming-scala_${scala.version} + flink-streaming-scala_${scala.maj.version} ${flink.version} provided org.apache.flink - flink-connector-kafka_${scala.version} + flink-connector-kafka_${scala.maj.version} ${flink.version} @@ -49,6 +49,11 @@ gson 2.4 + + com.fasterxml.jackson.core + jackson-databind + 2.10.0 + redis.clients jedis @@ -78,20 +83,20 @@ net.manub - scalatest-embedded-kafka_${scala.version} + scalatest-embedded-kafka_${scala.maj.version} 2.0.0 test org.apache.kafka - kafka_${scala.version} + kafka_${scala.maj.version} 2.4.1 test org.apache.flink flink-test-utils_2.12 - 1.10.0 + ${flink.version} test @@ -122,7 +127,7 @@ org.cassandraunit cassandra-unit - 3.1.1.0 + 3.11.2.0 test @@ -132,6 +137,14 @@ src/test/scala + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + maven-surefire-plugin 2.20 @@ -186,10 +199,11 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 4.4.0 - 1.8 - 1.8 + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} false diff --git a/data-pipeline-flink/dp-core/src/main/resources/base-config.conf b/data-pipeline-flink/dp-core/src/main/resources/base-config.conf index eed90dcebf..a7c74e02b3 100644 --- a/data-pipeline-flink/dp-core/src/main/resources/base-config.conf +++ b/data-pipeline-flink/dp-core/src/main/resources/base-config.conf @@ -1,8 +1,11 @@ kafka { - broker-servers = "localhost:9092" - zookeeper = "localhost:2181" + consumer.broker-servers = "localhost:9092" producer { + broker-servers = "localhost:9092" max-request-size = 1572864 + batch.size = 98304 + linger.ms = 10 + compression = "snappy" } } @@ -22,11 +25,13 @@ job { } task { - parallelism = 1 - consumer.parallelism = 1 + checkpointing.compressed = true checkpointing.interval = 60000 + checkpointing.pause.between.seconds = 30000 restart-strategy.attempts = 3 restart-strategy.delay = 30000 # in milli-seconds + parallelism = 1 + consumer.parallelism = 1 } redisdb.connection.timeout = 30000 diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/cache/DataCache.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/cache/DataCache.scala index f2e61fa512..6420074a14 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/cache/DataCache.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/cache/DataCache.scala @@ -1,15 +1,16 @@ package org.sunbird.dp.core.cache import java.util - import com.google.gson.Gson import org.slf4j.LoggerFactory import org.sunbird.dp.core.job.BaseJobConfig import redis.clients.jedis.Jedis -import redis.clients.jedis.exceptions.{JedisConnectionException, JedisException} +import redis.clients.jedis.exceptions.JedisException import scala.collection.JavaConverters._ +import scala.collection.mutable import scala.collection.mutable.Map +import scala.collection.immutable class DataCache(val config: BaseJobConfig, val redisConnect: RedisConnect, val dbIndex: Int, val fields: List[String]) { @@ -25,31 +26,56 @@ class DataCache(val config: BaseJobConfig, val redisConnect: RedisConnect, val d this.redisConnection.close() } - def hgetAllWithRetry(key: String): Map[String, String] = { + def hgetAllWithRetry(key: String): mutable.Map[String, AnyRef] = { try { - hgetAll(key) + convertToComplexDataTypes(hgetAll(key)) } catch { case ex: JedisException => logger.error("Exception when retrieving data from redis cache", ex) this.redisConnection.close() this.redisConnection = redisConnect.getConnection(dbIndex) - hgetAll(key) + convertToComplexDataTypes(hgetAll(key)) } + } + def isArray(value: String): Boolean = { + val redisValue = value.trim + redisValue.length > 0 && redisValue.startsWith("[") + } + + def isObject(value: String) = { + val redisValue = value.trim + redisValue.length > 0 && redisValue.startsWith("{") + } + + def convertToComplexDataTypes(data: mutable.Map[String, String]): mutable.Map[String, AnyRef] = { + val result = mutable.Map[String, AnyRef]() + data.keys.map { + redisKey => + val redisValue = data(redisKey) + if(isArray(redisValue)) { + result += redisKey -> gson.fromJson(redisValue, new util.ArrayList[AnyRef]().getClass) + } else if (isObject(redisValue)) { + result += redisKey -> gson.fromJson(redisValue, new util.HashMap[String, AnyRef]().getClass) + } else { + result += redisKey -> redisValue + } + } + result } - private def hgetAll(key: String): Map[String, String] = { + private def hgetAll(key: String): mutable.Map[String, String] = { val dataMap = redisConnection.hgetAll(key) if (dataMap.size() > 0) { if(fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava) dataMap.values().removeAll(util.Collections.singleton("")) dataMap.asScala } else { - Map[String, String]() + mutable.Map[String, String]() } } - def getWithRetry(key: String): Map[String, AnyRef] = { + def getWithRetry(key: String): mutable.Map[String, AnyRef] = { try { get(key) } catch { @@ -62,15 +88,15 @@ class DataCache(val config: BaseJobConfig, val redisConnect: RedisConnect, val d } - private def get(key: String): Map[String, AnyRef] = { + private def get(key: String): mutable.Map[String, AnyRef] = { val data = redisConnection.get(key) - if (data != null && !data.isEmpty()) { + if (data != null && !data.isEmpty) { val dataMap = gson.fromJson(data, new util.HashMap[String, AnyRef]().getClass) if(fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava) dataMap.values().removeAll(util.Collections.singleton("")) dataMap.asScala } else { - Map[String, AnyRef]() + mutable.Map[String, AnyRef]() } } @@ -104,7 +130,7 @@ class DataCache(val config: BaseJobConfig, val redisConnect: RedisConnect, val d try { set(key, value); } catch { - case ex@(_: JedisConnectionException | _: JedisException) => + case ex: JedisException => logger.error("Exception when update data to redis cache", ex) this.redisConnection.close() this.redisConnection = redisConnect.getConnection(dbIndex); @@ -116,6 +142,54 @@ class DataCache(val config: BaseJobConfig, val redisConnect: RedisConnect, val d redisConnection.set(key, value) } + def sMembers(key: String): util.Set[String] = { + redisConnection.smembers(key) + } + def getKeyMembers(key: String): util.Set[String] = { + try { + sMembers(key) + } catch { + case ex: JedisException => + logger.error("Exception when retrieving data from redis cache", ex) + this.redisConnection.close() + this.redisConnection = redisConnect.getConnection(dbIndex) + sMembers(key) + } + } + + def hdel(key: String, fieldSeq: Seq[String]): Unit = { + this.redisConnection.hdel(key, fieldSeq: _*) + } + + def hdelWithRetry(key: String, fieldSeq: Seq[String]): Unit = { + try { + hdel(key, fieldSeq) + } catch { + case ex: JedisException => + logger.error("Exception when deleting fields in hash", ex) + this.redisConnection.close() + this.redisConnection = redisConnect.getConnection(dbIndex) + hdel(key, fieldSeq) + } + } + + def hIncBy(key: String, field: String, value: Long): Unit = { + this.redisConnection.hincrBy(key, field, value) + } + + def hIncByWithRetry(key: String, field: String, value: Long): Unit = { + try { + hIncBy(key, field, value) + } catch { + case ex: JedisException => { + logger.error(s"Exception while incrementing count key=${key}, field=${field}, value=${value}", ex) + this.redisConnection.close() + this.redisConnection = redisConnect.getConnection(dbIndex) + hIncBy(key, field, value) + } + } + } + } // $COVERAGE-ON$ \ No newline at end of file diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/Events.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/Events.scala index cbd9586bfa..357ebb0206 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/Events.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/Events.scala @@ -4,6 +4,7 @@ import java.util import com.google.gson.Gson import org.sunbird.dp.core.reader.Telemetry +import org.sunbird.dp.core.util.JSONUtil abstract class Events(val map: util.Map[String, Any]) { @@ -23,7 +24,8 @@ abstract class Events(val map: util.Map[String, Any]) { def getMap(): util.Map[String, Any] = telemetry.getMap - def getJson(): String = new Gson().toJson(getMap) + // def getJson(): String = new Gson().toJson(getMap()) + def getJson(): String = JSONUtil.serialize(getMap()) def mid(): String = telemetry.read[String](keyPath = EventsPath.MID_PATH).orNull diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/EventsPath.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/EventsPath.scala index a019234170..b6b5da695b 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/EventsPath.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/EventsPath.scala @@ -48,4 +48,6 @@ object EventsPath { val OBJECT_ROLLUP_L1 = s"$OBJECT_PATH.rollup.l1" val SYNC_TS_PATH ="syncts" val TAGS_PATH ="tags" + val OBJECT_ROLLUP_L2 = s"$OBJECT_PATH.rollup.l2" + val L2_DATA_PATH = "l2data" } diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseJobConfig.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseJobConfig.scala index a7705de962..eec9f3a76b 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseJobConfig.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseJobConfig.scala @@ -2,12 +2,13 @@ package org.sunbird.dp.core.job import java.util.Properties import java.io.Serializable - import com.typesafe.config.Config import org.apache.kafka.clients.producer.ProducerConfig import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.streaming.api.scala.{OutputTag, createTypeInformation} import org.apache.kafka.clients.consumer.ConsumerConfig +import org.sunbird.dp.core.domain.Events class BaseJobConfig(val config: Config, val jobName: String) extends Serializable { @@ -15,9 +16,13 @@ class BaseJobConfig(val config: Config, val jobName: String) extends Serializabl implicit val metricTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - val kafkaBrokerServers: String = config.getString("kafka.broker-servers") - val zookeeper: String = config.getString("kafka.zookeeper") + val kafkaProducerBrokerServers: String = config.getString("kafka.producer.broker-servers") + val kafkaConsumerBrokerServers: String = config.getString("kafka.consumer.broker-servers") + // Producer Properties val kafkaProducerMaxRequestSize: Int = config.getInt("kafka.producer.max-request-size") + val kafkaProducerBatchSize: Int = config.getInt("kafka.producer.batch.size") + val kafkaProducerLingerMs: Int = config.getInt("kafka.producer.linger.ms") + val kafkaProducerCompression: String = if (config.hasPath("kafka.producer.compression")) config.getString("kafka.producer.compression") else "snappy" val groupId: String = config.getString("kafka.groupId") val restartAttempts: Int = config.getInt("task.restart-strategy.attempts") val delayBetweenAttempts: Long = config.getLong("task.restart-strategy.delay") @@ -35,14 +40,15 @@ class BaseJobConfig(val config: Config, val jobName: String) extends Serializabl val metaRedisPort: Int = Option(config.getInt("redis-meta.port")).getOrElse(6379) // Checkpointing config + val enableCompressedCheckpointing: Boolean = config.getBoolean("task.checkpointing.compressed") val checkpointingInterval: Int = config.getInt("task.checkpointing.interval") + val checkpointingPauseSeconds: Int = config.getInt("task.checkpointing.pause.between.seconds") val enableDistributedCheckpointing: Option[Boolean] = if (config.hasPath("job")) Option(config.getBoolean("job.enable.distributed.checkpointing")) else None val checkpointingBaseUrl: Option[String] = if (config.hasPath("job")) Option(config.getString("job.statebackend.base.url")) else None - def kafkaConsumerProperties: Properties = { val properties = new Properties() - properties.setProperty("bootstrap.servers", kafkaBrokerServers) + properties.setProperty("bootstrap.servers", kafkaConsumerBrokerServers) properties.setProperty("group.id", groupId) properties.setProperty(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed") kafkaAutoOffsetReset.map { properties.setProperty("auto.offset.reset", _) } @@ -51,10 +57,10 @@ class BaseJobConfig(val config: Config, val jobName: String) extends Serializabl def kafkaProducerProperties: Properties = { val properties = new Properties() - properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBrokerServers) - properties.put(ProducerConfig.LINGER_MS_CONFIG, new Integer(10)) - properties.put(ProducerConfig.BATCH_SIZE_CONFIG, new Integer(16384 * 4)) - properties.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "snappy") + properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProducerBrokerServers) + properties.put(ProducerConfig.LINGER_MS_CONFIG, new Integer(kafkaProducerLingerMs)) + properties.put(ProducerConfig.BATCH_SIZE_CONFIG, new Integer(kafkaProducerBatchSize)) + properties.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, kafkaProducerCompression) properties.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, new Integer(kafkaProducerMaxRequestSize)) properties } diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseProcessFunction.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseProcessFunction.scala index a05b5b585b..cf074ab4ec 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseProcessFunction.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseProcessFunction.scala @@ -1,12 +1,16 @@ package org.sunbird.dp.core.job +import java.lang import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong - import org.apache.flink.api.scala.metrics.ScalaGauge import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.ProcessFunction +import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction +import org.apache.flink.streaming.api.scala.{OutputTag, createTypeInformation} +import org.apache.flink.streaming.api.windowing.windows.GlobalWindow import org.apache.flink.util.Collector +import org.slf4j.LoggerFactory case class Metrics(metrics: ConcurrentHashMap[String, AtomicLong]) { def incCounter(metric: String): Unit = { @@ -29,6 +33,7 @@ trait JobMetrics { abstract class BaseProcessFunction[T, R](config: BaseJobConfig) extends ProcessFunction[T, R] with BaseDeduplication with JobMetrics { private val metrics: Metrics = registerMetrics(metricsList()) + private[this] val logger = LoggerFactory.getLogger(classOf[BaseProcessFunction[T, R]]) override def open(parameters: Configuration): Unit = { metricsList().map { metric => @@ -44,3 +49,27 @@ abstract class BaseProcessFunction[T, R](config: BaseJobConfig) extends ProcessF processElement(event, context, metrics) } } + +abstract class WindowBaseProcessFunction[I, O, K](config: BaseJobConfig) extends ProcessWindowFunction[I, O, K, GlobalWindow] with BaseDeduplication with JobMetrics { + + private val metrics: Metrics = registerMetrics(metricsList()) + private[this] val logger = LoggerFactory.getLogger(classOf[WindowBaseProcessFunction[I, O, K]]) + + override def open(parameters: Configuration): Unit = { + metricsList().map { metric => + getRuntimeContext.getMetricGroup.addGroup(config.jobName) + .gauge[Long, ScalaGauge[Long]](metric, ScalaGauge[Long](() => metrics.getAndReset(metric))) + } + } + + def metricsList(): List[String] + + def process(key: K, + context: ProcessWindowFunction[I, O, K, GlobalWindow]#Context, + elements: lang.Iterable[I], + metrics: Metrics): Unit + + override def process(key: K, context: ProcessWindowFunction[I, O, K, GlobalWindow]#Context, elements: lang.Iterable[I], out: Collector[O]): Unit = { + process(key, context, elements, metrics) + } +} \ No newline at end of file diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/FlinkKafkaConnector.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/FlinkKafkaConnector.scala index 5845611dc7..87dc28fd8f 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/FlinkKafkaConnector.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/FlinkKafkaConnector.scala @@ -4,7 +4,7 @@ import java.util import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaProducer} +import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaProducer, KafkaDeserializationSchema} import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer.Semantic import org.sunbird.dp.core.domain.Events import org.sunbird.dp.core.serde._ @@ -23,6 +23,10 @@ class FlinkKafkaConnector(config: BaseJobConfig) extends Serializable { new FlinkKafkaConsumer[String](kafkaTopic, new StringDeserializationSchema, config.kafkaConsumerProperties) } + def kafkaBytesSource(kafkaTopic: String): SourceFunction[Array[Byte]] = { + new FlinkKafkaConsumer[Array[Byte]](kafkaTopic, new ByteDeserializationSchema, config.kafkaConsumerProperties) + } + def kafkaStringSink(kafkaTopic: String): SinkFunction[String] = { new FlinkKafkaProducer[String](kafkaTopic, new StringSerializationSchema(kafkaTopic), config.kafkaProducerProperties, Semantic.AT_LEAST_ONCE) } @@ -36,4 +40,7 @@ class FlinkKafkaConnector(config: BaseJobConfig) extends Serializable { new EventSerializationSchema[T](kafkaTopic), config.kafkaProducerProperties, Semantic.AT_LEAST_ONCE) } + def kafkaBytesSink(kafkaTopic: String): SinkFunction[Array[Byte]] = { + new FlinkKafkaProducer[Array[Byte]](kafkaTopic, new ByteSerializationSchema(kafkaTopic), config.kafkaProducerProperties, Semantic.AT_LEAST_ONCE) + } } \ No newline at end of file diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/ByteSerde.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/ByteSerde.scala new file mode 100644 index 0000000000..8ad7aa838d --- /dev/null +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/ByteSerde.scala @@ -0,0 +1,31 @@ +package org.sunbird.dp.core.serde + +import java.nio.charset.StandardCharsets + +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.streaming.connectors.kafka.{KafkaDeserializationSchema, KafkaSerializationSchema} +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.apache.kafka.clients.producer.ProducerRecord + +class ByteDeserializationSchema extends KafkaDeserializationSchema[Array[Byte]] { + + override def isEndOfStream(nextElement: Array[Byte]): Boolean = false + + override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): Array[Byte] = { + record.value() + } + + override def getProducedType: TypeInformation[Array[Byte]] = TypeExtractor.getForClass(classOf[Array[Byte]]) +} + +class ByteSerializationSchema(topic: String, key: Option[String] = None) extends KafkaSerializationSchema[Array[Byte]] { + + private val serialVersionUID = -4284080856874185929L + + override def serialize(element: Array[Byte], timestamp: java.lang.Long): ProducerRecord[Array[Byte], Array[Byte]] = { + key.map { kafkaKey => + new ProducerRecord[Array[Byte], Array[Byte]](topic, kafkaKey.getBytes(StandardCharsets.UTF_8), element) + }.getOrElse(new ProducerRecord[Array[Byte], Array[Byte]](topic, element)) + } +} \ No newline at end of file diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/EventSerde.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/EventSerde.scala index 06e840faf0..ebe37c12a3 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/EventSerde.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/EventSerde.scala @@ -9,19 +9,29 @@ import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.streaming.connectors.kafka.{KafkaDeserializationSchema, KafkaSerializationSchema} import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.producer.ProducerRecord +import org.slf4j.LoggerFactory import org.sunbird.dp.core.domain.Events +import org.sunbird.dp.core.util.JSONUtil import scala.reflect.{ClassTag, classTag} class EventDeserializationSchema[T <: Events](implicit ct: ClassTag[T]) extends KafkaDeserializationSchema[T] { private val serialVersionUID = - 7339003654529835367L - override def isEndOfStream(nextElement: T): Boolean = false + private[this] val logger = LoggerFactory.getLogger(classOf[EventDeserializationSchema[Events]]) override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): T = { - val parsedString = new String(record.value(), StandardCharsets.UTF_8) - val result = new Gson().fromJson(parsedString, new util.HashMap[String, AnyRef]().getClass) - ct.runtimeClass.getConstructor(classOf[util.Map[String, AnyRef]]).newInstance(result).asInstanceOf[T] + // val parsedString = new String(record.value(), StandardCharsets.UTF_8) + // val result = new Gson().fromJson(parsedString, new util.HashMap[String, AnyRef]().getClass) + try { + val result = JSONUtil.deserialize[util.HashMap[String, AnyRef]](record.value()) + ct.runtimeClass.getConstructor(classOf[util.Map[String, AnyRef]]).newInstance(result).asInstanceOf[T] + } + catch { + case ex: Exception => + logger.error("Exception when parsing event from kafka: " + record, ex) + ct.runtimeClass.getConstructor(classOf[util.Map[String, AnyRef]]).newInstance(new util.HashMap[String, AnyRef]()).asInstanceOf[T] + } } override def getProducedType: TypeInformation[T] = TypeExtractor.getForClass(classTag[T].runtimeClass).asInstanceOf[TypeInformation[T]] diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/MapSerde.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/MapSerde.scala index 7d5fa57386..bfb7a1fa41 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/MapSerde.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/MapSerde.scala @@ -9,6 +9,8 @@ import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.streaming.connectors.kafka.{KafkaDeserializationSchema, KafkaSerializationSchema} import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.producer.ProducerRecord +import org.sunbird.dp.core.util.JSONUtil + import collection.JavaConverters._ class MapDeserializationSchema extends KafkaDeserializationSchema[util.Map[String, AnyRef]] { @@ -18,9 +20,11 @@ class MapDeserializationSchema extends KafkaDeserializationSchema[util.Map[Strin override def isEndOfStream(nextElement: util.Map[String, AnyRef]): Boolean = false override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): util.Map[String, AnyRef] = { - val parsedString = new String(record.value(), StandardCharsets.UTF_8) - val recordMap = new Gson().fromJson(parsedString, new util.HashMap[String, AnyRef]().getClass).asScala - recordMap.asJava + // val parsedString = new String(record.value(), StandardCharsets.UTF_8) + // val recordMap = new Gson().fromJson(parsedString, new util.HashMap[String, AnyRef]().getClass).asScala + val recordMap = JSONUtil.deserialize[util.HashMap[String, AnyRef]](record.value()) + // recordMap.asJava + recordMap } override def getProducedType: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/CassandraUtil.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/CassandraUtil.scala index 3b046c4d09..87a23ca284 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/CassandraUtil.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/CassandraUtil.scala @@ -4,15 +4,17 @@ import java.util import com.datastax.driver.core._ import com.datastax.driver.core.exceptions.DriverException -import com.datastax.driver.core.querybuilder.Insert +import org.slf4j.LoggerFactory class CassandraUtil(host: String, port: Int) { - + val logger = LoggerFactory.getLogger("CassandraUtil") + val options : QueryOptions = new QueryOptions() val cluster = { Cluster.builder() .addContactPoint(host) .withPort(port) + .withQueryOptions(options.setConsistencyLevel(ConsistencyLevel.QUORUM)) .withoutJMXReporting() .build() } @@ -29,8 +31,11 @@ class CassandraUtil(host: String, port: Int) { rs.all } catch { case ex: DriverException => - this.reconnect() - this.find(query) + logger.info(s"Failed cassandra query is ${query}") + ex.printStackTrace() + throw ex + // this.reconnect() + // this.find(query) } } diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/FlinkUtil.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/FlinkUtil.scala index becfb01166..6bbd184446 100644 --- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/FlinkUtil.scala +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/FlinkUtil.scala @@ -13,18 +13,19 @@ object FlinkUtil { def getExecutionContext(config: BaseJobConfig): StreamExecutionEnvironment = { val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment + env.getConfig.setUseSnapshotCompression(config.enableCompressedCheckpointing) env.enableCheckpointing(config.checkpointingInterval) /** * Use Blob storage as distributed state backend if enabled */ - config.enableDistributedCheckpointing match { case Some(true) => { val stateBackend: StateBackend = new FsStateBackend(s"${config.checkpointingBaseUrl.getOrElse("")}/${config.jobName}", true) env.setStateBackend(stateBackend) val checkpointConfig: CheckpointConfig = env.getCheckpointConfig checkpointConfig.enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION) + checkpointConfig.setMinPauseBetweenCheckpoints(config.checkpointingPauseSeconds ) } case _ => // Do nothing } diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/JSONUtil.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/JSONUtil.scala new file mode 100644 index 0000000000..6453191e80 --- /dev/null +++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/JSONUtil.scala @@ -0,0 +1,46 @@ +package org.sunbird.dp.core.util + +import java.lang.reflect.{ParameterizedType, Type} + +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.core.JsonGenerator.Feature +import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature} + +import com.fasterxml.jackson.core.`type`.TypeReference + +object JSONUtil { + + @transient val mapper = new ObjectMapper() + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) + mapper.configure(Feature.WRITE_BIGDECIMAL_AS_PLAIN, true) + mapper.setSerializationInclusion(Include.NON_NULL) + + @throws(classOf[Exception]) + def serialize(obj: AnyRef): String = { + mapper.writeValueAsString(obj) + } + + def deserialize[T: Manifest](json: String): T = { + mapper.readValue(json, typeReference[T]) + } + + def deserialize[T: Manifest](json: Array[Byte]): T = { + mapper.readValue(json, typeReference[T]) + } + + private[this] def typeReference[T: Manifest] = new TypeReference[T] { + override def getType = typeFromManifest(manifest[T]) + } + + + private[this] def typeFromManifest(m: Manifest[_]): Type = { + if (m.typeArguments.isEmpty) { m.runtimeClass } + else new ParameterizedType { + def getRawType = m.runtimeClass + def getActualTypeArguments = m.typeArguments.map(typeFromManifest).toArray + def getOwnerType = null + } + } + +} diff --git a/data-pipeline-flink/dp-core/src/test/resources/base-test.conf b/data-pipeline-flink/dp-core/src/test/resources/base-test.conf index a0c6d2fe87..2c678d1993 100644 --- a/data-pipeline-flink/dp-core/src/test/resources/base-test.conf +++ b/data-pipeline-flink/dp-core/src/test/resources/base-test.conf @@ -1,6 +1,6 @@ kafka { - broker-servers = "localhost:9093" - zookeeper = "localhost:2183" + producer.broker-servers = "localhost:9093" + consumer.broker-servers = "localhost:9093" map.input.topic = "local.telemetry.map.input" map.output.topic = "local.telemetry.map.output" event.input.topic = "local.telemetry.event.input" @@ -12,11 +12,16 @@ kafka { auto.offset.reset = "earliest" producer { max-request-size = 102400 + batch.size = 8192 + linger.ms = 1 + compression = "snappy" } } task { + checkpointing.compressed = true checkpointing.interval = 60000 + checkpointing.pause.between.seconds = 30000 restart-strategy.attempts = 1 restart-strategy.delay = 10000 parallelism = 1 diff --git a/data-pipeline-flink/dp-core/src/test/resources/test.conf b/data-pipeline-flink/dp-core/src/test/resources/test.conf index f922afa615..22665e323e 100644 --- a/data-pipeline-flink/dp-core/src/test/resources/test.conf +++ b/data-pipeline-flink/dp-core/src/test/resources/test.conf @@ -5,20 +5,25 @@ kafka { event.output.topic = "local.telemetry.event.output" string.input.topic = "local.telemetry.string.input" string.output.topic = "local.telemetry.string.output" - broker-servers = "localhost:9093" - zookeeper = "localhost:2183" + producer.broker-servers = "localhost:9093" + consumer.broker-servers = "localhost:9093" groupId = "pipeline-preprocessor-group" auto.offset.reset = "earliest" producer { max-request-size = 102400 + batch.size = 8192 + linger.ms = 1 + compression = "snappy" } } kafka.output.metrics.topic = "pipeline_metrics" task { + checkpointing.compressed = true parallelism = 1 consumer.parallelism = 1 checkpointing.interval = 60000 + checkpointing.pause.between.seconds = 30000 metrics.window.size = 100 # 3 min restart-strategy.attempts = 1 # retry once restart-strategy.delay = 1000 # in milli-seconds diff --git a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/fixture/EventFixture.scala b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/fixture/EventFixture.scala index fedcfffea7..32ac8e0006 100644 --- a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/fixture/EventFixture.scala +++ b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/fixture/EventFixture.scala @@ -32,19 +32,24 @@ object EventFixture { | event.output.topic = "local.telemetry.event.output" | string.input.topic = "local.telemetry.string.input" | string.output.topic = "local.telemetry.string.output" - | broker-servers = "localhost:9093" - | zookeeper = "localhost:2183" + | producer.broker-servers = "localhost:9093" + | consumer.broker-servers = "localhost:9093" | groupId = "pipeline-preprocessor-group" | auto.offset.reset = "earliest" | producer { | max-request-size = 102400 + | batch.size = 8192 + | linger.ms = 1 + | compression = "snappy" | } |} | |task { | parallelism = 2 | consumer.parallelism = 1 + | checkpointing.compressed = true | checkpointing.interval = 60000 + | checkpointing.pause.between.seconds = 30000 | metrics.window.size = 100 # 3 min | restart-strategy.attempts = 1 # retry once | restart-strategy.delay = 1000 # in milli-seconds diff --git a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/BaseProcessFunctionTestSpec.scala b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/BaseProcessFunctionTestSpec.scala index d21d177d29..7907b5195e 100644 --- a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/BaseProcessFunctionTestSpec.scala +++ b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/BaseProcessFunctionTestSpec.scala @@ -14,6 +14,7 @@ import org.apache.kafka.common.serialization.StringDeserializer import org.scalatest.Matchers import org.sunbird.dp.core.job.FlinkKafkaConnector import org.sunbird.dp.core.util.FlinkUtil +import scala.concurrent.duration._ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future @@ -68,6 +69,12 @@ class BaseProcessFunctionTestSpec extends BaseSpec with Matchers { |"mid":"LP.1586994119534.4bfe9b31-216d-46ea-8e60-d7ea1b1a103c","type":"events"} """.stripMargin + val EVENT_INVALID = + """ + |"{\"eid\":\"AUDIT\",\"ets\":1605674597050,\"ver\":\"3.0\",\"mid\":\"LP.1605674597050.e920319b-b6e4-4be8-877d-f4c588ee7a3c\",\"actor\":{\"id\":\"9bb884fc-8a56-4727-9522-25a7d5b8ea06\",\"type\":\"User\"},\"context\":{\"channel\":\"ORG_001\",\"pdata\":{\"pid\":\"lms-service\",\"ver\":\"1.0\"},\"env\":\"CourseBatch\",\"cdata\":[{\"id\":\"do_213153022369169408120\",\"type\":\"Course\"},{\"id\":\"01315302480262758472\",\"type\":\"CourseBatch\"}]},\"object\":{\"id\":\"9bb884fc-8a56-4727-9522-25a7d5b8ea06\",\"type\":\"User\",\"rollup\":{\"l1\":\"do_213153022369169408120\"}},\"edata\":{\"state\":\"Create\",\"type\":\"enrol\",\"props\":[\"courseId\",\"enrolledDate\",\"userId\",\"batchId\",\"active\"]}}" + | + |""".stripMargin + val customKafkaConsumerProperties: Map[String, String] = Map[String, String]("auto.offset.reset" -> "earliest", "group.id" -> "test-event-schema-group") implicit val embeddedKafkaConfig: EmbeddedKafkaConfig = @@ -89,6 +96,7 @@ class BaseProcessFunctionTestSpec extends BaseSpec with Matchers { publishStringMessageToKafka(bsConfig.kafkaEventInputTopic, SHARE_EVENT) publishStringMessageToKafka(bsConfig.kafkaMapInputTopic, EVENT_WITH_MESSAGE_ID) publishStringMessageToKafka(bsConfig.kafkaStringInputTopic, SHARE_EVENT) + publishStringMessageToKafka(bsConfig.kafkaEventInputTopic, EVENT_INVALID) flinkCluster.before() } @@ -139,21 +147,26 @@ class BaseProcessFunctionTestSpec extends BaseSpec with Matchers { env.execute("TestSerDeFunctionality") } - val eventSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaEventOutputTopic, 2) - val eventSchemaDuplicates = consumeNumberMessagesFrom[String](bsConfig.kafkaEventDuplicateTopic, 1) - val mapSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaMapOutputTopic, 1) - val stringSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaStringOutputTopic, 1) - - eventSchemaMessages.size should be (2) - eventSchemaDuplicates.size should be (1) - mapSchemaMessages.size should be (1) - stringSchemaMessages.size should be (1) - - retrieveMid(mapSchemaMessages.head) should be ("56c0c430-748b-11e8-ae77-cd19397ca6b0") - retrieveMid(eventSchemaDuplicates.head) should be ("02ba33e5-15fe-4ec5-b32") - retrieveMid(stringSchemaMessages.head) should be ("02ba33e5-15fe-4ec5-b32") - retrieveMid(eventSchemaMessages.head) should be ("02ba33e5-15fe-4ec5-b32") - retrieveMid(eventSchemaMessages.last) should be ("LP.1586994119534.4bfe9b31-216d-46ea-8e60-d7ea1b1a103c") + // val test = consumeNumberMessagesFromTopics[String](topics = Set(bsConfig.kafkaEventOutputTopic), number = 2, timeout = 30.seconds).values.toList + try { + val eventSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaEventOutputTopic, 2) + val eventSchemaDuplicates = consumeNumberMessagesFrom[String](bsConfig.kafkaEventDuplicateTopic, 1) + val mapSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaMapOutputTopic, 1) + val stringSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaStringOutputTopic, 1) + + eventSchemaMessages.size should be(2) + eventSchemaDuplicates.size should be(1) + mapSchemaMessages.size should be(1) + stringSchemaMessages.size should be(1) + + retrieveMid(mapSchemaMessages.head) should be("56c0c430-748b-11e8-ae77-cd19397ca6b0") + retrieveMid(eventSchemaDuplicates.head) should be("02ba33e5-15fe-4ec5-b32") + retrieveMid(stringSchemaMessages.head) should be("02ba33e5-15fe-4ec5-b32") + retrieveMid(eventSchemaMessages.head) should be("02ba33e5-15fe-4ec5-b32") + retrieveMid(eventSchemaMessages.last) should be("LP.1586994119534.4bfe9b31-216d-46ea-8e60-d7ea1b1a103c") + } catch { + case ex: Exception => println("Error occurred when consuming events from Embedded Kafka...") + } } diff --git a/data-pipeline-flink/druid-events-validator/pom.xml b/data-pipeline-flink/druid-events-validator/pom.xml index 7bba6120d8..9090cf6588 100644 --- a/data-pipeline-flink/druid-events-validator/pom.xml +++ b/data-pipeline-flink/druid-events-validator/pom.xml @@ -31,7 +31,7 @@ org.apache.flink - flink-streaming-scala_${scala.version} + flink-streaming-scala_${scala.maj.version} ${flink.version} provided @@ -61,13 +61,13 @@ org.apache.flink flink-test-utils_2.12 - 1.10.0 + ${flink.version} test org.apache.flink flink-runtime_2.12 - 1.10.0 + ${flink.version} test tests @@ -80,7 +80,7 @@ org.apache.flink flink-streaming-java_2.12 - 1.10.0 + ${flink.version} test tests @@ -108,7 +108,14 @@ src/main/scala src/test/scala - + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + org.apache.maven.plugins maven-shade-plugin @@ -156,10 +163,11 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 4.4.0 - 1.8 - 1.8 + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} false diff --git a/data-pipeline-flink/druid-events-validator/src/main/resources/druid-events-validator.conf b/data-pipeline-flink/druid-events-validator/src/main/resources/druid-events-validator.conf index 66cf82cd5e..71ccc4b25e 100644 --- a/data-pipeline-flink/druid-events-validator/src/main/resources/druid-events-validator.conf +++ b/data-pipeline-flink/druid-events-validator/src/main/resources/druid-events-validator.conf @@ -11,8 +11,7 @@ kafka { task { consumer.parallelism = 1 - validator.parallelism = 1 - router.parallelism = 1 + downstream.operators.parallelism = 1 druid.validation.enabled = true druid.deduplication.enabled = true } diff --git a/data-pipeline-flink/druid-events-validator/src/main/resources/log4j.properties b/data-pipeline-flink/druid-events-validator/src/main/resources/log4j.properties deleted file mode 100644 index a15d8c34cb..0000000000 --- a/data-pipeline-flink/druid-events-validator/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=druid-events-validator.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/filter.json b/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/filter.json index c6f61d886d..19f440abab 100644 --- a/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/filter.json +++ b/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/filter.json @@ -2,14 +2,5 @@ "type": "object", "properties": { }, - "anyOf": [ - { - "not": { - "required": [ - "dialCodes" - ] - } - } - ], "additionalProperties": true } \ No newline at end of file diff --git a/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/search.json b/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/search.json index 73ddce9313..3e27c731b3 100644 --- a/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/search.json +++ b/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/search.json @@ -53,7 +53,8 @@ "type": "number" }, "filters": { - "$ref": "resource:/schemas/telemetry/filter.json" + "id": "http://api.sunbird.org/telemetry/edata/filters", + "type": "object" }, "topn": { "id": "http://api.sunbird.org/telemetry/edata/topn", diff --git a/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorConfig.scala b/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorConfig.scala index afbeea61b8..97ba43782d 100644 --- a/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorConfig.scala +++ b/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorConfig.scala @@ -16,8 +16,7 @@ class DruidValidatorConfig(override val config: Config) extends BaseJobConfig(co val cacheExpirySeconds: Int = config.getInt("redis.database.key.expiry.seconds") override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - val validatorParallelism: Int = config.getInt("task.validator.parallelism") - val routerParallelism: Int = config.getInt("task.router.parallelism") + val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.operators.parallelism") val druidValidationEnabled: Boolean = config.getBoolean("task.druid.validation.enabled") val druidDeduplicationEnabled: Boolean = config.getBoolean("task.druid.deduplication.enabled") diff --git a/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorStreamTask.scala b/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorStreamTask.scala index 01c273e099..77fc43c8fe 100644 --- a/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorStreamTask.scala +++ b/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorStreamTask.scala @@ -49,22 +49,28 @@ class DruidValidatorStreamTask(config: DruidValidatorConfig, kafkaConnector: Fli .uid(config.druidValidatorConsumer).setParallelism(config.kafkaConsumerParallelism) .rebalance() .process(new DruidValidatorFunction(config)).name(config.druidValidatorFunction).uid(config.druidValidatorFunction) - .setParallelism(config.validatorParallelism) + .setParallelism(config.downstreamOperatorsParallelism) + + validationDataStream.getSideOutput(config.telemetryRouterOutputTag) /** * Separate sinks for valid telemetry events, valid summary events, valid error events, valid log events and invalid events */ validationDataStream.getSideOutput(config.telemetryRouterOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaTelemetryRouteTopic)) .name(config.telemetryEventsProducer).uid(config.telemetryEventsProducer) + .setParallelism(config.downstreamOperatorsParallelism) validationDataStream.getSideOutput(config.summaryRouterOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaSummaryRouteTopic)) .name(config.summaryEventsProducer).uid(config.summaryEventsProducer) + .setParallelism(config.downstreamOperatorsParallelism) validationDataStream.getSideOutput(config.duplicateEventOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDuplicateTopic)) .name(config.druidDuplicateEventsProducer).uid(config.druidDuplicateEventsProducer) + .setParallelism(config.downstreamOperatorsParallelism) validationDataStream.getSideOutput(config.invalidEventOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaFailedTopic)) .name(config.druidInvalidEventsProducer).uid(config.druidInvalidEventsProducer) + .setParallelism(config.downstreamOperatorsParallelism) env.execute(config.jobName) diff --git a/data-pipeline-flink/druid-events-validator/src/test/resources/test.conf b/data-pipeline-flink/druid-events-validator/src/test/resources/test.conf index 4ecbc2c0b6..f1baad4e8c 100644 --- a/data-pipeline-flink/druid-events-validator/src/test/resources/test.conf +++ b/data-pipeline-flink/druid-events-validator/src/test/resources/test.conf @@ -10,8 +10,7 @@ kafka { } task { - validator.parallelism = 1 - router.parallelism = 1 + downstream.operators.parallelism = 1 druid.validation.enabled = true druid.deduplication.enabled = true } diff --git a/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala index ab2177a0d5..791fccd848 100644 --- a/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala +++ b/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala @@ -81,7 +81,7 @@ object EventFixture { |"objecttype":"Content","mediatype":"content","status":"Live"}} """.stripMargin - val VALID_SERACH_EVENT: String = + val VALID_SEARCH_EVENT: String = """ |{"eid":"SEARCH","ver":"3.0","syncts":1.59518415538E12,"ets":1.59518415538E12,"flags": |{"pp_validation_processed":true,"pp_duplicate":false,"device_denorm":false,"dialcode_denorm":true, @@ -97,4 +97,21 @@ object EventFixture { |"pid":"search-service"},"did":"79838ccb0ff2c7d0a9dd05f5b337fbca","env":"search","channel":"ROOT_ORG"}, |"@version":"1","object":{"id":"KLQ2G7","type":"DialCode"}} """.stripMargin + + val SEARCH_EVENT_WITH_INCORRECT_DIALCODES_KEY: String = + """ + |{"eid":"SEARCH","ver":"3.0","syncts":1.59518415538E12,"ets":1.59518415538E12,"flags": + |{"pp_validation_processed":true,"pp_duplicate":false,"device_denorm":false,"dialcode_denorm":true, + |"content_denorm":false},"dialcodedata":{"identifier":"KLQ2G7","channel":"0123221617357783046602", + |"publisher":"MHPUBLISHER","status":2.0},"mid":"invalid_dialcode_key", + |"type":"events","tags":["kp-events"],"actor":{"id":"org.sunbird.learning.platform","type":"System"}, + |"edata":{"topn":[{"identifier":"do_312528046917705728246886"}],"query":"","size":7.0,"type":"content", + |"filters":{"contentType":["TextBookUnit","Resource","TextBook","Collection","Course"],"mimeType":{}, + |"resourceType":{},"status":["Live"],"objectType":["Content"],"dialCodes":"KLQ2G7","framework":{}, + |"compatibilityLevel":{"max":4.0,"min":1.0},"channel":{"ne":["0124433024890224640","0124446042259128320", + |"0124487522476933120","0125840271570288640","0124453662635048969"]}},"sort":{}}, + |"@timestamp":"2020-07-19T18:42:41.524Z","context":{"pdata":{"ver":"1.0","id":"prod.diksha.portal", + |"pid":"search-service"},"did":"79838ccb0ff2c7d0a9dd05f5b337fbca","env":"search","channel":"ROOT_ORG"}, + |"@version":"1","object":{"id":"KLQ2G7","type":"DialCode"}} + """.stripMargin } diff --git a/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/spec/DruidValidatorStreamTaskTestSpec.scala b/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/spec/DruidValidatorStreamTaskTestSpec.scala index 670971566e..e19508da2e 100644 --- a/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/spec/DruidValidatorStreamTaskTestSpec.scala +++ b/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/spec/DruidValidatorStreamTaskTestSpec.scala @@ -13,6 +13,7 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceCont import org.apache.flink.test.util.MiniClusterWithClientResource import org.mockito.Mockito import org.mockito.Mockito.when +import org.sunbird.dp.core.domain.EventsPath import org.sunbird.dp.core.job.FlinkKafkaConnector import org.sunbird.dp.fixture.EventFixture import org.sunbird.dp.validator.domain.Event @@ -20,6 +21,8 @@ import org.sunbird.dp.validator.task.{DruidValidatorConfig, DruidValidatorStream import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} import redis.embedded.RedisServer +import scala.collection.JavaConverters._ + class DruidValidatorStreamTaskTestSpec extends BaseTestSpec { @@ -32,8 +35,6 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec { .build) var redisServer: RedisServer = _ - // val config: Config = ConfigFactory.load("test.conf") - // val druidValidatorConfig: DruidValidatorConfig = new DruidValidatorConfig(config) var config: Config = _ var druidValidatorConfig: DruidValidatorConfig = _ val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) @@ -45,16 +46,6 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec { redisServer.start() BaseMetricsReporter.gaugeMetrics.clear() - - /* - when(mockKafkaUtil.kafkaEventSource[Event](druidValidatorConfig.kafkaInputTopic)).thenReturn(new DruidValidatorEventSource) - - when(mockKafkaUtil.kafkaEventSink[Event](druidValidatorConfig.kafkaDuplicateTopic)).thenReturn(new DupEventsSink) - when(mockKafkaUtil.kafkaEventSink[Event](druidValidatorConfig.kafkaTelemetryRouteTopic)).thenReturn(new TelemetryEventsSink) - when(mockKafkaUtil.kafkaEventSink[Event](druidValidatorConfig.kafkaSummaryRouteTopic)).thenReturn(new SummaryEventsSink) - when(mockKafkaUtil.kafkaEventSink[Event](druidValidatorConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink) - */ - flinkCluster.before() } @@ -81,24 +72,27 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec { val task = new DruidValidatorStreamTask(druidValidatorConfig, mockKafkaUtil) task.process() - TelemetryEventsSink.values.size() should be (2) + TelemetryEventsSink.values.size() should be (3) SummaryEventsSink.values.size() should be (1) FailedEventsSink.values.size() should be (1) DupEventsSink.values.size() should be (1) + val invalidDialCodeKeyEvent = TelemetryEventsSink.values.asScala.filter(event => event.mid() === "invalid_dialcode_key").head + invalidDialCodeKeyEvent.getTelemetry.read(s"${EventsPath.EDTA_FILTERS}.dialCodes") should not be None + DupEventsSink.values.get(0).getFlags.get("dv_duplicate").booleanValue() should be(true) FailedEventsSink.values.get(0).getFlags.get("dv_processed").booleanValue() should be(false) FailedEventsSink.values.get(0).getFlags.get("dv_validation_failed").booleanValue() should be(true) - BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationSuccessMetricsCount}").getValue() should be (3) + BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationSuccessMetricsCount}").getValue() should be (4) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationFailureMetricsCount}").getValue() should be (1) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.duplicate-event-count").getValue() should be (1) - BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (4) + BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (5) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.summaryRouterMetricCount}").getValue() should be (1) - BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (2) + BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (3) } @@ -122,7 +116,7 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec { val task = new DruidValidatorStreamTask(druidValidatorConfig, mockKafkaUtil) task.process() - TelemetryEventsSink.values.size() should be (4) + TelemetryEventsSink.values.size() should be (5) SummaryEventsSink.values.size() should be (1) FailedEventsSink.values.size() should be (0) DupEventsSink.values.size() should be (0) @@ -134,7 +128,7 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec { BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (0) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.summaryRouterMetricCount}").getValue() should be (1) - BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (4) + BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (5) } @@ -158,19 +152,22 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec { val task = new DruidValidatorStreamTask(druidValidatorConfig, mockKafkaUtil) task.process() - TelemetryEventsSink.values.size() should be (3) + TelemetryEventsSink.values.size() should be (4) SummaryEventsSink.values.size() should be (1) FailedEventsSink.values.size() should be (1) DupEventsSink.values.size() should be (0) - BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationSuccessMetricsCount}").getValue() should be (4) + val invalidDialCodeKeyEvent = TelemetryEventsSink.values.asScala.filter(event => event.mid() === "invalid_dialcode_key").head + invalidDialCodeKeyEvent.getTelemetry.read(s"${EventsPath.EDTA_FILTERS}.dialCodes") should not be None + + BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationSuccessMetricsCount}").getValue() should be (5) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationFailureMetricsCount}").getValue() should be (1) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.duplicate-event-count").getValue() should be (0) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (0) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.summaryRouterMetricCount}").getValue() should be (1) - BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (3) + BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (4) } @@ -194,7 +191,7 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec { val task = new DruidValidatorStreamTask(druidValidatorConfig, mockKafkaUtil) task.process() - TelemetryEventsSink.values.size() should be (3) + TelemetryEventsSink.values.size() should be (4) SummaryEventsSink.values.size() should be (1) FailedEventsSink.values.size() should be (0) DupEventsSink.values.size() should be (1) @@ -203,10 +200,10 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec { BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationFailureMetricsCount}").getValue() should be (0) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.duplicate-event-count").getValue() should be (1) - BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (4) + BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (5) BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.summaryRouterMetricCount}").getValue() should be (1) - BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (3) + BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (4) } @@ -219,11 +216,13 @@ class DruidValidatorEventSource extends SourceFunction[Event] { val event1 = gson.fromJson(EventFixture.VALID_DENORM_TELEMETRY_EVENT, new util.LinkedHashMap[String, Any]().getClass) val event2 = gson.fromJson(EventFixture.INVALID_DENORM_TELEMETRY_EVENT, new util.LinkedHashMap[String, Any]().getClass) val event3 = gson.fromJson(EventFixture.VALID_DENORM_SUMMARY_EVENT, new util.LinkedHashMap[String, Any]().getClass) - val event4 = gson.fromJson(EventFixture.VALID_SERACH_EVENT, new util.LinkedHashMap[String, Any]().getClass) + val event4 = gson.fromJson(EventFixture.VALID_SEARCH_EVENT, new util.LinkedHashMap[String, Any]().getClass) + val event5 = gson.fromJson(EventFixture.SEARCH_EVENT_WITH_INCORRECT_DIALCODES_KEY, new util.LinkedHashMap[String, Any]().getClass) ctx.collect(new Event(event1)) ctx.collect(new Event(event2)) ctx.collect(new Event(event3)) ctx.collect(new Event(event4)) + ctx.collect(new Event(event5)) ctx.collect(new Event(event1)) } diff --git a/data-pipeline-flink/ingest-router/pom.xml b/data-pipeline-flink/ingest-router/pom.xml new file mode 100644 index 0000000000..103ddd5e14 --- /dev/null +++ b/data-pipeline-flink/ingest-router/pom.xml @@ -0,0 +1,217 @@ + + + + 4.0.0 + + org.sunbird.dp + pipeline-jobs + 1.0 + + + org.sunbird.dp.jobs + ingest-router + 1.0.0 + jar + IngestRouter + + Ingest router + + + + UTF-8 + 1.4.0 + + + + + org.apache.flink + flink-streaming-scala_${scala.maj.version} + ${flink.version} + provided + + + org.sunbird.dp + dp-core + 1.0.0 + + + org.sunbird.dp + dp-core + 1.0.0 + test-jar + test + + + org.apache.flink + flink-test-utils_2.12 + ${flink.version} + test + + + org.apache.flink + flink-runtime_2.12 + ${flink.version} + test + tests + + + it.ozimov + embedded-redis + 0.7.1 + test + + + org.apache.flink + flink-streaming-java_2.12 + ${flink.version} + test + tests + + + org.scalatest + scalatest_2.12 + 3.0.6 + test + + + org.mockito + mockito-core + 3.3.3 + test + + + com.fiftyonred + mock-jedis + 0.4.0 + test + + + + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + + package + + shade + + + + + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.sunbird.dp.ingestrouter.task.IngestRouterStreamTask + + + + reference.conf + + + + + + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + maven-surefire-plugin + 2.22.2 + + true + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + dp-duplication-testsuite.txt + + + + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + + \ No newline at end of file diff --git a/data-pipeline-flink/ingest-router/src/main/resources/ingest-router.conf b/data-pipeline-flink/ingest-router/src/main/resources/ingest-router.conf new file mode 100644 index 0000000000..05580b822a --- /dev/null +++ b/data-pipeline-flink/ingest-router/src/main/resources/ingest-router.conf @@ -0,0 +1,12 @@ +include "base-config.conf" + +kafka { + input.topic = ${job.env}".telemetry.ingestion" + output.success.topic = ${job.env}".telemetry.ingest" + groupId = ${job.env}"-ingest-router-group" +} + +task { + consumer.parallelism = 1 + downstream.operators.parallelism = 1 +} diff --git a/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterConfig.scala b/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterConfig.scala new file mode 100644 index 0000000000..ce185a0c12 --- /dev/null +++ b/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterConfig.scala @@ -0,0 +1,48 @@ +package org.sunbird.dp.ingestrouter.task + +import java.util + +import com.typesafe.config.Config +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.streaming.api.scala.OutputTag +import org.sunbird.dp.core.job.BaseJobConfig + + class IngestRouterConfig(override val config: Config) extends BaseJobConfig(config, "IngestRouterJob") { + + private val serialVersionUID = 2905979434303791379L + + implicit val bytesTypeInfo: TypeInformation[Array[Byte]] = TypeExtractor.getForClass(classOf[Array[Byte]]) + + // Kafka Topics Configuration + val kafkaIngestInputTopic: String = config.getString("kafka.ingest.input.topic") + val kafkaIngestSuccessTopic: String = config.getString("kafka.ingest.output.success.topic") + + val kafkaRawInputTopic: String = config.getString("kafka.raw.input.topic") + val kafkaRawSuccessTopic: String = config.getString("kafka.raw.output.success.topic") + + + override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") + val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.operators.parallelism") + val rawKafkaConsumerParallelism: Int = config.getInt("task.raw.consumer.parallelism") + val rawDownstreamOperatorsParallelism: Int = config.getInt("task.raw.downstream.operators.parallelism") + + val EVENTS_OUTPUT_TAG = "events" + + // Metric List + val ingestSuccessEventCount = "ingest-success-event-count" + val rawSuccessEventCount = "raw-success-event-count" + + val eventsOutputTag: OutputTag[Array[Byte]] = OutputTag[Array[Byte]](EVENTS_OUTPUT_TAG) + + // Consumers + val telemetryIngestProcessorConsumer = "ingest-router-consumer" + val telemetryRawProcessorConsumer = "raw-router-consumer" + + // Producers + val telemetryIngestProcessorProducer = "ingest-router-sink" + val telemetryRawProcessorProducer = "raw-router-sink" + + + + } diff --git a/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterStreamTask.scala b/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterStreamTask.scala new file mode 100644 index 0000000000..1d61d98f91 --- /dev/null +++ b/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterStreamTask.scala @@ -0,0 +1,55 @@ +package org.sunbird.dp.ingestrouter.task + +import java.io.File +import java.util + +import com.typesafe.config.ConfigFactory +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.api.java.utils.ParameterTool +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment +import org.sunbird.dp.core.job.FlinkKafkaConnector +import org.sunbird.dp.core.util.FlinkUtil + +class IngestRouterStreamTask(config: IngestRouterConfig, kafkaConnector: FlinkKafkaConnector) { + + private val serialVersionUID = -7729362727131516112L + + def process(): Unit = { + implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) + implicit val bytesTypeInfo: TypeInformation[Array[Byte]] = TypeExtractor.getForClass(classOf[Array[Byte]]) + + val ingestStream = + env.addSource(kafkaConnector.kafkaBytesSource(config.kafkaIngestInputTopic), config.telemetryIngestProcessorConsumer) + .uid(config.telemetryIngestProcessorConsumer).setParallelism(config.kafkaConsumerParallelism) + .addSink(kafkaConnector.kafkaBytesSink(config.kafkaIngestSuccessTopic)) + .name(config.telemetryIngestProcessorProducer).uid(config.telemetryIngestProcessorProducer) + .setParallelism(config.downstreamOperatorsParallelism) + val rawStream = + env.addSource(kafkaConnector.kafkaBytesSource(config.kafkaRawInputTopic), config.telemetryRawProcessorConsumer) + .uid(config.telemetryRawProcessorConsumer).setParallelism(config.rawKafkaConsumerParallelism) + .addSink(kafkaConnector.kafkaBytesSink(config.kafkaRawSuccessTopic)) + .name(config.telemetryRawProcessorProducer).uid(config.telemetryRawProcessorProducer) + .setParallelism(config.rawDownstreamOperatorsParallelism) + + + env.execute(config.jobName) + } +} + +// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster +object IngestRouterStreamTask { + + def main(args: Array[String]): Unit = { + val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) + val config = configFilePath.map { + path => ConfigFactory.parseFile(new File(path)).resolve() + }.getOrElse(ConfigFactory.load("ingest-router.conf").withFallback(ConfigFactory.systemEnvironment())) + val telemetryProcessorConfig = new IngestRouterConfig(config) + val kafkaUtil = new FlinkKafkaConnector(telemetryProcessorConfig) + val task = new IngestRouterStreamTask(telemetryProcessorConfig, kafkaUtil) + task.process() + } +} + +// $COVERAGE-ON$ diff --git a/data-pipeline-flink/ingest-router/src/test/resources/test.conf b/data-pipeline-flink/ingest-router/src/test/resources/test.conf new file mode 100644 index 0000000000..319f0131ce --- /dev/null +++ b/data-pipeline-flink/ingest-router/src/test/resources/test.conf @@ -0,0 +1,23 @@ +include "base-test.conf" + +kafka { + + groupId = "flink-ingest-router-group" + ingest { + input.topic = "flink.telemetry.ingestion" + output.success.topic = "flink.telemetry.ingest" + } + raw { + input.topic = "flink.telemetry.raw.event" + output.success.topic = "flink.telemetry.raw" + } +} + +task { + consumer.parallelism = 1 + downstream.operators.parallelism = 1 + raw { + consumer.parallelism = 1 + downstream.operators.parallelism = 1 + } +} \ No newline at end of file diff --git a/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala new file mode 100644 index 0000000000..daab76d0d5 --- /dev/null +++ b/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala @@ -0,0 +1,54 @@ +package org.sunbird.dp.fixture + +object EventFixture { + + val EVENT_WITH_MESSAGE_ID: String = + """ + |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} + |""".stripMargin + + val EVENT_WITHOUT_MESSAGE_ID: String = + """ + |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{"id":"do_21299582901864857613016","ver":"1.0","type":"AssessmentItem"},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test1","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_21299582901864857613016","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test2","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_312526125187809280139355","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test3","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"test-item-id","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} + |""".stripMargin + + val EMPTY_BATCH_EVENTS: String = + """ + |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} + |""".stripMargin + + val UNDEFINED_EVENTS_IN_BATCH: String = + """ + |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} + |""".stripMargin + + val INVALID_BATCH_EVENT: String = + """ + |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":["eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} + |""".stripMargin + + val MISSING_FIELDS_BATCH:String = { + """ + |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"mid":"5734970","channel":"98347593475834","params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","key":"","did":"0958743690"},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}]} + |""".stripMargin + } + + val RAW_LOG_EVENT:String = { + """ + |{"object":{"rollup":{},"ver":"","type":"content","id":"do_11307732367521382418684"},"ets":1596373063462,"ver":"3.0","eid":"LOG","actor":{"type":"service","id":"1"},"context":{"rollup":{},"sid":"","pdata":{"id":"local.dock.content-service","ver":"1.0","pid":"sunbird-content-service"},"env":"contentplayer","did":"","channel":"sunbird","cdata":[]},"mid":"LOG:89da1a341096f1e5e24806a0250c5c83","tags":[],"edata":{"params":[{"rid":"c1457c60-d4bf-11ea-9313-0912071b8abe"},{},{},{"url":"health"},{"method":"GET"}],"message":"successful","type":"api_access","level":"INFO"},"syncts":1596375302571,"@timestamp":"2020-08-02T13:35:02.571Z","flags":{"ex_processed":true}} + |""".stripMargin + } + + val RAW_SEARCH_EVENT:String = { + """ + |{"@timestamp":"2020-08-11T10:21:19.464Z","ets":1596001408756,"ver":"3.0","eid":"SEARCH","mid":"LP.1596001408756.045c0aa6-4b70-45d3-b43f-bc908e512175","actor":{"type":"System","id":"org.sunbird.learning.platform"},"context":{"channel":"in.ekstep","pdata":{"pid":"search-service","ver":"1.0","id":"sunbird.Sunbird_Dev.learning.platform"},"env":"search"},"syncts":1596001408756,"@version":"1","edata":{"filters":{"objectType":["Content"],"resourceType":{},"contentType":["course"],"framework":{},"channel":{},"mimeType":{},"status":"Live","identifier":["KP_FT_1564394114456","do_112476372676763648176","do_1124785353783377921154","do_112481468925796352148","do_1124834956377456641118","do_1124835868462612481134","do_1125083286221291521153","do_1125098016170639361238","do_1125110672521134081296","do_11269260687205990411105","do_112717173739429888132","do_112728688963133440110506","do_112736824142061568168","do_11283193441064550414","do_11307319880400076811160","do_2123251055368765441634"]},"query":"","size":14,"sort":{},"topn":[{"identifier":"do_112736824142061568168"},{"identifier":"do_1125110672521134081296"},{"identifier":"do_11269260687205990411105"},{"identifier":"do_112728688963133440110506"},{"identifier":"do_112717173739429888132"}],"type":"content"}} + |""".stripMargin + } + + val RAW_ERROR_EVENT:String = { + """ + |{"@timestamp":"2020-08-11T10:21:18.518Z","ets":1596483300965,"ver":"3.0","eid":"ERROR","mid":"LP.1596483300965.28e663f6-2994-442a-8f4a-eb1459c17a99","actor":{"type":"User","id":"org.ekstep.learning.platform"},"context":{"channel":"in.ekstep","pdata":{"pid":"learning-service","ver":"1.0","id":"dev.sunbird.learning.platform"},"env":"system"},"syncts":1596483300965,"@version":"1","edata":{"errtype":"system","err":"RESOURCE_NOT_FOUND","stacktrace":"RESOURCE_NOT_FOUND: Resource not found : KP_FT_1596483300820.img\n\tat org.ekstep.learning.actor.ContentStoreActor.onReceive(ContentStoreActor.java:77)\n\tat akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167)\n\tat akka.actor.Actor$class.aroundReceive(Actor.scala:539)\n\tat akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97)\n\tat akka.actor.ActorCell.receiveMessage(ActorCell.scala:610)\n\tat akka.actor.ActorCell.invoke(ActorCell.scala:579)\n\tat akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)\n\tat akka.dispatch.Mailbox.run(Mailbox.scala:229)\n\tat akka.dispatch.Mailbox.exec(Mailbox.scala:241)\n\tat akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)\n\tat akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)\n\tat akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)\n\tat akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)\n"}} + |""".stripMargin + } + +} \ No newline at end of file diff --git a/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/spec/IngestRouterStreamTaskTestSpec.scala b/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/spec/IngestRouterStreamTaskTestSpec.scala new file mode 100644 index 0000000000..0da773599b --- /dev/null +++ b/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/spec/IngestRouterStreamTaskTestSpec.scala @@ -0,0 +1,131 @@ +package org.sunbird.dp.spec + +import java.nio.charset.StandardCharsets +import java.util + +import com.google.gson.Gson +import com.typesafe.config.Config +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.mockito.Mockito +import org.mockito.Mockito._ +import org.sunbird.dp.fixture.EventFixture +import redis.embedded.RedisServer +import org.apache.flink.streaming.api.functions.sink.SinkFunction +import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration +import org.apache.flink.test.util.MiniClusterWithClientResource +import org.apache.flink.streaming.api.functions.source.SourceFunction +import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext +import com.typesafe.config.ConfigFactory +import org.sunbird.dp.core.cache.RedisConnect +import org.sunbird.dp.core.domain.Events +import org.sunbird.dp.core.job.FlinkKafkaConnector +import org.sunbird.dp.ingestrouter.task.{IngestRouterConfig, IngestRouterStreamTask} +import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} + +import collection.JavaConverters._ + +class IngestRouterStreamTaskTestSpec extends BaseTestSpec { + + implicit val bytesTypeInfo: TypeInformation[Array[Byte]] = TypeExtractor.getForClass(classOf[Array[Byte]]) + + val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() + .setConfiguration(testConfiguration()) + .setNumberSlotsPerTaskManager(1) + .setNumberTaskManagers(1) + .build) + + val config: Config = ConfigFactory.load("test.conf") + val ingestRouterConfig: IngestRouterConfig = new IngestRouterConfig(config) + val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) + + override protected def beforeAll(): Unit = { + super.beforeAll() + BaseMetricsReporter.gaugeMetrics.clear() + + when(mockKafkaUtil.kafkaBytesSource(ingestRouterConfig.kafkaIngestInputTopic)).thenReturn(new IngestRouterEventSource) + when(mockKafkaUtil.kafkaBytesSink(ingestRouterConfig.kafkaIngestSuccessTopic)).thenReturn(new IngestSuccessEventsSink) + + when(mockKafkaUtil.kafkaBytesSource(ingestRouterConfig.kafkaRawInputTopic)).thenReturn(new RawRouterEventSource) + when(mockKafkaUtil.kafkaBytesSink(ingestRouterConfig.kafkaRawSuccessTopic)).thenReturn(new RawSuccessEventsSink) + + flinkCluster.before() + } + + override protected def afterAll(): Unit = { + super.afterAll() + flinkCluster.after() + } + + "Ingest router job pipeline" should "just pass through ingest events" in { + + val task = new IngestRouterStreamTask(ingestRouterConfig, mockKafkaUtil) + task.process() + + IngestSuccessEventsSink.values.size() should be(3) + + RawSuccessEventsSink.values.size() should be(3) + + } + +} + +class IngestRouterEventSource extends SourceFunction[Array[Byte]] { + + override def run(ctx: SourceContext[Array[Byte]]) { + val gson = new Gson() + val event1 = EventFixture.EVENT_WITH_MESSAGE_ID + val event2 = EventFixture.EVENT_WITHOUT_MESSAGE_ID + val event3 = EventFixture.INVALID_BATCH_EVENT + ctx.collect(event1.getBytes(StandardCharsets.UTF_8)) + ctx.collect(event2.getBytes(StandardCharsets.UTF_8)) + ctx.collect(event3.getBytes(StandardCharsets.UTF_8)) + } + + override def cancel() = {} + +} + +class RawRouterEventSource extends SourceFunction[Array[Byte]] { + + override def run(ctx: SourceContext[Array[Byte]]) { + val gson = new Gson() + val event1 = EventFixture.RAW_LOG_EVENT + val event2 = EventFixture.RAW_SEARCH_EVENT + val event3 = EventFixture.RAW_ERROR_EVENT + ctx.collect(event1.getBytes(StandardCharsets.UTF_8)) + ctx.collect(event2.getBytes(StandardCharsets.UTF_8)) + ctx.collect(event3.getBytes(StandardCharsets.UTF_8)) + + } + + override def cancel() = {} + +} + + +class IngestSuccessEventsSink extends SinkFunction[Array[Byte]] { + + override def invoke(value: Array[Byte]): Unit = { + synchronized { + IngestSuccessEventsSink.values.add(value) + } + } +} + +object IngestSuccessEventsSink { + val values: util.List[Array[Byte]] = new util.ArrayList[Array[Byte]]() +} + +class RawSuccessEventsSink extends SinkFunction[Array[Byte]] { + + override def invoke(value: Array[Byte]): Unit = { + synchronized { + RawSuccessEventsSink.values.add(value) + } + } +} + +object RawSuccessEventsSink { + val values: util.List[Array[Byte]] = new util.ArrayList[Array[Byte]]() +} diff --git a/data-pipeline-flink/pipeline-preprocessor/pom.xml b/data-pipeline-flink/pipeline-preprocessor/pom.xml index d4c09e856e..c346c6f83b 100644 --- a/data-pipeline-flink/pipeline-preprocessor/pom.xml +++ b/data-pipeline-flink/pipeline-preprocessor/pom.xml @@ -26,7 +26,7 @@ org.apache.flink - flink-streaming-scala_${scala.version} + flink-streaming-scala_${scala.maj.version} ${flink.version} provided @@ -51,6 +51,11 @@ commons-lang3 3.9 + + io.github.classgraph + classgraph + 4.8.90 + org.sunbird.dp dp-core @@ -67,13 +72,13 @@ org.apache.flink flink-test-utils_2.12 - 1.10.0 + ${flink.version} test org.apache.flink flink-runtime_2.12 - 1.10.0 + ${flink.version} test tests @@ -86,7 +91,7 @@ org.apache.flink flink-streaming-java_2.12 - 1.10.0 + ${flink.version} test tests @@ -108,7 +113,14 @@ src/main/scala src/test/scala - + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + org.apache.maven.plugins maven-shade-plugin @@ -158,10 +170,11 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 4.4.0 - 1.8 - 1.8 + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} false diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/log4j.properties b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/log4j.properties deleted file mode 100644 index 77a86a6451..0000000000 --- a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=pipeline-preprocessor.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/pipeline-preprocessor.conf b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/pipeline-preprocessor.conf index b4e1550b60..ee3e522b8d 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/pipeline-preprocessor.conf +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/pipeline-preprocessor.conf @@ -5,22 +5,24 @@ kafka { output.failed.topic = ${job.env}".telemetry.failed" output.primary.route.topic = ${job.env}".telemetry.unique" output.log.route.topic = ${job.env}".druid.events.log" - output.error.route.topic = ${job.env}".druid.events.error" + output.error.route.topic = ${job.env}".telemetry.error" output.audit.route.topic = ${job.env}".telemetry.audit" output.duplicate.topic = ${job.env}".telemetry.duplicate" + output.denorm.secondary.route.topic = ${job.env}".telemetry.unique.secondary" + output.denorm.primary.route.topic = ${job.env}".telemetry.unique.primary" + output.cb.audit.route.topic = ${job.env}".telemetry.cb.audit" groupId = ${job.env}"-pipeline-preprocessor-group" } task { consumer.parallelism = 1 - telemetry.validation.parallelism = 1 - telemetry.router.parallelism = 1 - share.events.flattener.parallelism = 1 + downstream.operators.parallelism = 1 } telemetry.schema.path="schemas/telemetry/3.0" default.channel="b00bc992ef25f1a9a8d63291e20efc8d" -dedup.producer.included.ids = ["dev.sunbird.portal", "dev.sunbird.desktop"] +dedup.producer.included.ids = [${job.env}".sunbird.portal", ${job.env}".sunbird.desktop"] +secondary.events = ["INTERACT", "IMPRESSION", "SHARE_ITEM"] redis { database { diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_audit.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_audit.json new file mode 100644 index 0000000000..808235100d --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_audit.json @@ -0,0 +1,35 @@ +{ + "id": "http://api.ekstep.org/telemetry/audit", + "type": "object", + "required": [ + "eid", + "ets", + "ver", + "mid", + "actor", + "context", + "object", + "edata" + ], + "allOf": [ + { + "$ref": "resource:/schemas/telemetry/3.0/common.json" + }, + { + "properties": { + "eid": { + "id": "http://api.ekstep.org/telemetry/eid", + "enum": [ + "CB_AUDIT" + ] + }, + "edata": { + "oneOf": [ + {"$ref": "resource:/schemas/telemetry/3.0/cb_wat_edata.json"}, + {"$ref": "resource:/schemas/telemetry/3.0/cb_frac_edata.json"} + ] + } + } + } + ] +} \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_data.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_data.json new file mode 100644 index 0000000000..5241e53e71 --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_data.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "required": ["data"], + "additionalProperties": false, + "properties": { + "data": {"type": "object"} + } +} \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_frac_edata.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_frac_edata.json new file mode 100644 index 0000000000..b6a86e7177 --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_frac_edata.json @@ -0,0 +1,37 @@ +{ + "type": "object", + "required": [ + "state", + "props", + "cb_object" + ], + "additionalProperties": false, + "properties": { + "type": { + "id": "http://api.ekstep.org/telemetry/cb-frac-edata/type", + "type": "string" + }, + "props": { + "id": "http://api.ekstep.org/telemetry/cb-frac-edata/props", + "type": "array", + "items": { + "type": "string" + } + }, + "state": { + "id": "http://api.ekstep.org/telemetry/cb-frac-edata/state", + "type": "string" + }, + "prevstate": { + "id": "http://api.ekstep.org/telemetry/cb-frac-edata/prevstate", + "type": "string" + }, + "duration": { + "id": "http://api.ekstep.org/telemetry/cb-frac-edata/duration", + "type": "number" + }, + "cb_object": { + "$ref": "resource:/schemas/telemetry/3.0/cb_frac_object.json" + } + } +} \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_frac_object.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_frac_object.json new file mode 100644 index 0000000000..dbecda421b --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_frac_object.json @@ -0,0 +1,41 @@ +{ + "type": "object", + "required": [ + "id", + "type", + "ver", + "name", + "org", + "sub_type" + ], + "additionalProperties": false, + "properties": { + "id": { + "id": "http://api.ekstep.org/telemetry/cb-frac-object/id", + "type": "string" + }, + "type": { + "id": "http://api.ekstep.org/telemetry/cb-frac-object/type", + "enum": ["Competency", "Activity", "Role", "Position", "Officer"] + }, + "ver": { + "id": "http://api.ekstep.org/telemetry/cb-frac-object/ver", + "type": "string" + }, + "rollup": { + "$ref": "resource:/schemas/telemetry/3.0/rollup.json" + }, + "name": { + "id": "http://api.ekstep.org/telemetry/cb-frac-object/name", + "type": "string" + }, + "org": { + "id": "http://api.ekstep.org/telemetry/cb-frac-object/org", + "type": "string" + }, + "sub_type": { + "id": "http://api.ekstep.org/telemetry/cb-frac-object/sub_type", + "type": "string" + } + } +} \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_wat_edata.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_wat_edata.json new file mode 100644 index 0000000000..a1405347ae --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_wat_edata.json @@ -0,0 +1,41 @@ +{ + "type": "object", + "required": [ + "state", + "props", + "cb_object", + "cb_data" + ], + "additionalProperties": false, + "properties": { + "type": { + "id": "http://api.ekstep.org/telemetry/cb-wat-edata/type", + "type": "string" + }, + "props": { + "id": "http://api.ekstep.org/telemetry/cb-wat-edata/props", + "type": "array", + "items": { + "type": "string" + } + }, + "state": { + "id": "http://api.ekstep.org/telemetry/cb-wat-edata/state", + "type": "string" + }, + "prevstate": { + "id": "http://api.ekstep.org/telemetry/cb-wat-edata/prevstate", + "type": "string" + }, + "duration": { + "id": "http://api.ekstep.org/telemetry/cb-wat-edata/duration", + "type": "number" + }, + "cb_object": { + "$ref": "resource:/schemas/telemetry/3.0/cb_wat_object.json" + }, + "cb_data": { + "$ref": "resource:/schemas/telemetry/3.0/cb_data.json" + } + } +} \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_wat_object.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_wat_object.json new file mode 100644 index 0000000000..c1b62e098c --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_wat_object.json @@ -0,0 +1,36 @@ +{ + "type": "object", + "required": [ + "id", + "type", + "ver", + "name", + "org" + ], + "additionalProperties": false, + "properties": { + "id": { + "id": "http://api.ekstep.org/telemetry/cb-wat-object/id", + "type": "string" + }, + "type": { + "id": "http://api.ekstep.org/telemetry/cb-wat-object/type", + "enum": ["WorkOrder"] + }, + "ver": { + "id": "http://api.ekstep.org/telemetry/cb-wat-object/ver", + "type": "string" + }, + "rollup": { + "$ref": "resource:/schemas/telemetry/3.0/rollup.json" + }, + "name": { + "id": "http://api.ekstep.org/telemetry/cb-wat-object/name", + "type": "string" + }, + "org": { + "id": "http://api.ekstep.org/telemetry/cb-wat-object/org", + "type": "string" + } + } +} \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/event.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/event.json index e77e5225c2..ef520c5825 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/event.json +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/event.json @@ -46,6 +46,15 @@ }, { "$ref": "resource:/schemas/telemetry/3.0/exdata.json" + }, + { + "$ref": "resource:/schemas/telemetry/3.0/summary.json" + }, + { + "$ref": "resource:/schemas/telemetry/3.0/metrics.json" + }, + { + "$ref": "resource:/schemas/telemetry/3.0/cb_audit.json" } ] } \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/metrics.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/metrics.json new file mode 100644 index 0000000000..db2802812d --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/metrics.json @@ -0,0 +1,63 @@ +{ + "id": "http://api.ekstep.org/telemetry/metrics", + "type": "object", + "required": [ + "eid", + "ets", + "ver", + "mid", + "actor", + "context", + "edata" + ], + "allOf": [ + { + "$ref": "resource:/schemas/telemetry/3.0/common.json" + }, + { + "properties": { + "eid": { + "id": "http://api.ekstep.org/telemetry/eid", + "enum": [ + "METRICS" + ] + }, + "edata": { + "id": "http://api.ekstep.org/telemetry/edata", + "type": "object", + "additionalProperties": false, + "required": [ + "system", + "subsystem" + ], + "properties": { + "system": { + "id": "http://api.ekstep.org/telemetry/edata/system", + "type": "string" + }, + "subsystem": { + "id": "http://api.ekstep.org/telemetry/edata/subsystem", + "type": "string" + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "properties": { + "metric": { + "id": "http://api.ekstep.org/telemetry/edata/metrics/metric", + "type": "string" + }, + "value": { + "id": "http://api.ekstep.org/telemetry/edata/metrics/value", + "type": "number" + } + } + } + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/summary.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/summary.json new file mode 100644 index 0000000000..4c2c2b3db5 --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/summary.json @@ -0,0 +1,87 @@ +{ + "id": "http://api.ekstep.org/telemetry/summary", + "type": "object", + "required": [ + "eid", + "ets", + "ver", + "mid", + "actor", + "context", + "edata" + ], + "allOf": [ + { + "$ref": "resource:/schemas/telemetry/3.0/common.json" + }, + { + "properties": { + "eid": { + "id": "http://api.ekstep.org/telemetry/eid", + "enum": [ + "SUMMARY" + ] + }, + "edata": { + "id": "http://api.ekstep.org/telemetry/edata", + "type": "object", + "additionalProperties": false, + "required": [ + "type", + "starttime", + "endtime", + "timespent", + "pageviews", + "interactions" + ], + "properties": { + "type": { + "id": "http://api.ekstep.org/telemetry/edata/type", + "type": "string" + }, + "mode": { + "id": "http://api.ekstep.org/telemetry/edata/mode", + "type": "string" + }, + "starttime": { + "id": "http://api.ekstep.org/telemetry/edata/starttime", + "type": "number" + }, + "endtime": { + "id": "http://api.ekstep.org/telemetry/edata/endtime", + "type": "number" + }, + "timespent": { + "id": "http://api.ekstep.org/telemetry/edata/timespent", + "type": "number" + }, + "pageviews": { + "id": "http://api.ekstep.org/telemetry/edata/pageviews", + "type": "number" + }, + "interactions": { + "id": "http://api.ekstep.org/telemetry/edata/interactions", + "type": "number" + }, + "extra": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "id": "http://api.ekstep.org/telemetry/edata/extra/id", + "type": "string" + }, + "value": { + "id": "http://api.ekstep.org/telemetry/edata/extra/value", + "type": "string" + } + } + } + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/domain/Event.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/domain/Event.scala index 5da5d952b9..44f27bc06e 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/domain/Event.scala +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/domain/Event.scala @@ -86,5 +86,8 @@ class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { def rollup: util.Map[String, AnyRef] = telemetry.read[util.Map[String, AnyRef]](EventsPath.CONTEXT_ROLLUP_PATH).orNull + def updateHub(hub: String): Unit = { + telemetry.add("context.hub", hub) + } } diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/PipelinePreprocessorFunction.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/PipelinePreprocessorFunction.scala new file mode 100644 index 0000000000..d191415e8c --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/PipelinePreprocessorFunction.scala @@ -0,0 +1,133 @@ +package org.sunbird.dp.preprocessor.functions + +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.dp.core.cache.{DedupEngine, RedisConnect} +import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics} +import org.sunbird.dp.preprocessor.domain.Event +import org.sunbird.dp.preprocessor.task.PipelinePreprocessorConfig +import org.sunbird.dp.preprocessor.util.{ShareEventsFlattener, TelemetryValidator} + +class PipelinePreprocessorFunction(config: PipelinePreprocessorConfig, + @transient var telemetryValidator: TelemetryValidator = null, + @transient var shareEventsFlattener: ShareEventsFlattener = null, + @transient var dedupEngine: DedupEngine = null) + (implicit val eventTypeInfo: TypeInformation[Event]) + extends BaseProcessFunction[Event, Event](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[PipelinePreprocessorFunction]) + + override def metricsList(): List[String] = { + List(config.validationFailureMetricsCount, + config.validationSuccessMetricsCount, + config.duplicationSkippedEventMetricsCount, + config.primaryRouterMetricCount, + config.logEventsRouterMetricsCount, + config.errorEventsRouterMetricsCount, + config.auditEventRouterMetricCount, + config.shareEventsRouterMetricCount, + config.shareItemEventsMetircsCount, + config.denormSecondaryEventsRouterMetricsCount, + config.denormPrimaryEventsRouterMetricsCount, + config.cbAuditEventRouterMetricCount + ) ::: deduplicationMetrics + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + if (dedupEngine == null) { + val redisConnect = new RedisConnect(config.redisHost, config.redisPort, config) + dedupEngine = new DedupEngine(redisConnect, config.dedupStore, config.cacheExpirySeconds) + } + + if (telemetryValidator == null) { + telemetryValidator = new TelemetryValidator(config) + } + + if(shareEventsFlattener == null) { + shareEventsFlattener = new ShareEventsFlattener(config) + } + } + + override def close(): Unit = { + super.close() + dedupEngine.closeConnectionPool() + } + + def isDuplicateCheckRequired(producerId: String): Boolean = { + // config.includedProducersForDedup.contains(producerId) + true + } + + def addHubField(event: Event): Unit = { + val clientEventTypes = Set("IMPRESSION", "INTERACT", "START", "END") + val envHubMap = Map("learn" -> "learn", "course" -> "learn", "discuss" -> "discuss", "network" -> "network", + "careers" -> "careers", "competency" -> "competency", "events" -> "events") + if (clientEventTypes.contains(event.eid().toUpperCase())) { + val hub = envHubMap.getOrElse(event.env.toLowerCase(), "other") + event.updateHub(hub) + } + } + + override def processElement(event: Event, + context: ProcessFunction[Event, Event]#Context, + metrics: Metrics): Unit = { + + val isValid = telemetryValidator.validate(event, context, metrics) + + if (isValid) { + if (event.eid().equalsIgnoreCase("LOG")) { + context.output(config.logEventsOutputTag, event) + metrics.incCounter(metric = config.logEventsRouterMetricsCount) + } + else { + val isUnique = if (isDuplicateCheckRequired(event.producerId())) { + deDuplicate[Event, Event](event.mid(), event, context, config.duplicateEventsOutputTag, + flagName = config.DEDUP_FLAG_NAME)(dedupEngine, metrics) + } else { + event.markSkipped(config.DEDUP_SKIP_FLAG_NAME) + metrics.incCounter(uniqueEventMetricCount) + true + } + + if (isUnique) { + + // add hub info + addHubField(event) + + if("ERROR".equalsIgnoreCase(event.eid())) { + metrics.incCounter(metric = config.errorEventsRouterMetricsCount) + } else if (config.secondaryEvents.contains(event.eid())) { + context.output(config.denormSecondaryEventsRouteOutputTag, event) + metrics.incCounter(metric = config.denormSecondaryEventsRouterMetricsCount) + } + else { + context.output(config.denormPrimaryEventsRouteOutputTag, event) + metrics.incCounter(metric = config.denormPrimaryEventsRouterMetricsCount) + } + event.eid() match { + case "AUDIT" => + context.output(config.auditRouteEventsOutputTag, event) + metrics.incCounter(metric = config.auditEventRouterMetricCount) + metrics.incCounter(metric = config.primaryRouterMetricCount) // Since we are are sinking the AUDIT Event into primary router topic + case "SHARE" => + shareEventsFlattener.flatten(event, context, metrics) + metrics.incCounter(metric = config.shareEventsRouterMetricCount) + metrics.incCounter(metric = config.primaryRouterMetricCount) // // Since we are are sinking the SHARE Event into primary router topic + case "ERROR" => + context.output(config.errorEventOutputTag, event) + case "CB_AUDIT" => + context.output(config.cbAuditRouteEventsOutputTag, event) // cbAudit event are not routed to denorm topic + metrics.incCounter(metric = config.cbAuditEventRouterMetricCount) // // metric for cb_audit events + case _ => context.output(config.primaryRouteEventsOutputTag, event) + metrics.incCounter(metric = config.primaryRouterMetricCount) + } + } + } + } + + } + +} diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/ShareEventsFlattenerFunction.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/ShareEventsFlattenerFunction.scala index a76ae85627..d7e7b5d6cf 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/ShareEventsFlattenerFunction.scala +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/ShareEventsFlattenerFunction.scala @@ -38,12 +38,12 @@ class ShareEventsFlattenerFunction(config: PipelinePreprocessorConfig) val edataType = if (transfers == 0) "download" else "import" val shareItemEvent = generateShareItemEvents(shareEvent, EventObject(id = identifier, ver = version, `type` = contentType, rollup = Rollup(shareEvent.objectID())), Some(edataType), Some(size)) - context.output(config.shareItemEventOutputTag, new Gson().toJson(shareItemEvent)) + context.output(config.shareItemEventOutputTag, new Event(new Gson().fromJson(shareItemEvent, new util.LinkedHashMap[String, Any]().getClass))) }) } else { val shareItemEvent = generateShareItemEvents(shareEvent, EventObject(id = identifier, ver = version, `type` = contentType, rollup = Rollup(shareEvent.objectID())), edataType = Some(shareEvent.edataType())) - context.output(config.shareItemEventOutputTag, new Gson().toJson(shareItemEvent)) + context.output(config.shareItemEventOutputTag, new Event(new Gson().fromJson(shareItemEvent, new util.LinkedHashMap[String, Any]().getClass))) } metrics.incCounter(config.shareItemEventsMetircsCount) }) @@ -56,8 +56,8 @@ class ShareEventsFlattenerFunction(config: PipelinePreprocessorConfig) eventObj: EventObject, edataType: Option[String], paramSize: Option[Double] = None - ): ShareEvent = { - ShareEvent( + ): String = { + val shareItemEvent = ShareEvent( ActorObject(event.actorId(), event.actorType()), "SHARE_ITEM", EData(event.edataDir, edataType.orNull, paramSize.getOrElse(0)), @@ -69,6 +69,8 @@ class ShareEventsFlattenerFunction(config: PipelinePreprocessorConfig) eventObj, event.eventTags ) + + new Gson().toJson(shareItemEvent) } override def metricsList(): List[String] = { diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorConfig.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorConfig.scala index 618d53feaf..ef84c3eb5e 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorConfig.scala +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorConfig.scala @@ -28,10 +28,14 @@ class PipelinePreprocessorConfig(override val config: Config) extends BaseJobCon val kafkaLogRouteTopic: String = config.getString("kafka.output.log.route.topic") val kafkaErrorRouteTopic: String = config.getString("kafka.output.error.route.topic") val kafkaAuditRouteTopic: String = config.getString("kafka.output.audit.route.topic") + val kafkaCbAuditRouteTopic: String = config.getString("kafka.output.cb.audit.route.topic") val kafkaFailedTopic: String = config.getString("kafka.output.failed.topic") val kafkaDuplicateTopic: String = config.getString("kafka.output.duplicate.topic") + val kafkaDenormSecondaryRouteTopic: String = config.getString("kafka.output.denorm.secondary.route.topic") + val kafkaDenormPrimaryRouteTopic: String = config.getString("kafka.output.denorm.primary.route.topic") + val defaultChannel: String = config.getString("default.channel") val includedProducersForDedup: List[String] = config.getStringList("dedup.producer.included.ids").asScala.toList @@ -44,19 +48,22 @@ class PipelinePreprocessorConfig(override val config: Config) extends BaseJobCon // Router stream out put tags val primaryRouteEventsOutputTag: OutputTag[Event] = OutputTag[Event]("primary-route-events") + // Spliting events on priority for faster denorm processing + val denormSecondaryEventsRouteOutputTag: OutputTag[Event] = OutputTag[Event]("denorm-secondary-events") + val denormPrimaryEventsRouteOutputTag: OutputTag[Event] = OutputTag[Event]("denorm-primary-events") + // Audit, Log & Error Events output tag val auditRouteEventsOutputTag: OutputTag[Event] = OutputTag[Event]("audit-route-events") + val cbAuditRouteEventsOutputTag: OutputTag[Event] = OutputTag[Event]("cb-audit-route-events") val logEventsOutputTag: OutputTag[Event] = OutputTag[Event]("log-route-events") val errorEventOutputTag: OutputTag[Event] = OutputTag[Event]("error-route-events") // Share events out put tags val shareRouteEventsOutputTag: OutputTag[Event] = OutputTag[Event]("share-route-events") - val shareItemEventOutputTag: OutputTag[String] = OutputTag[String]("share-item-events") + val shareItemEventOutputTag: OutputTag[Event] = OutputTag[Event]("share-item-events") override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - val validationParallelism: Int = config.getInt("task.telemetry.validation.parallelism") - val routerParallelism: Int = config.getInt("task.telemetry.router.parallelism") - val shareEventsFlattnerParallelism: Int = config.getInt("task.share.events.flattener.parallelism") + val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.operators.parallelism") val VALIDATION_FLAG_NAME = "pp_validation_processed" val DEDUP_FLAG_NAME = "pp_duplicate" @@ -69,7 +76,9 @@ class PipelinePreprocessorConfig(override val config: Config) extends BaseJobCon val shareEventsRouterMetricCount = "share-route-success-count" val logEventsRouterMetricsCount = "log-route-success-count" val errorEventsRouterMetricsCount = "error-route-success-count" - + val denormSecondaryEventsRouterMetricsCount = "denorm-secondary-route-success-count" + val denormPrimaryEventsRouterMetricsCount = "denorm-primary-route-success-count" + val cbAuditEventRouterMetricCount = "cb-audit-route-success-count" // Validation job metrics val validationSuccessMetricsCount = "validation-success-event-count" @@ -98,12 +107,15 @@ class PipelinePreprocessorConfig(override val config: Config) extends BaseJobCon val logRouterProducer = "log-route-sink" val errorRouterProducer = "error-route-sink" val auditRouterProducer = "audit-route-sink" + val cbAuditRouterProducer = "cb-audit-route-sink" val invalidEventProducer = "invalid-events-sink" val duplicateEventProducer = "duplicate-events-sink" + val denormSecondaryEventProducer = "denorm-secondary-events-sink" + val denormPrimaryEventProducer = "denorm-primary-events-sink" val defaultSchemaFile = "envelope.json" - + val secondaryEvents: List[String] = config.getStringList("secondary.events").asScala.toList } diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorStreamTask.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorStreamTask.scala index c8006d7153..19c07a7f52 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorStreamTask.scala +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorStreamTask.scala @@ -11,7 +11,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.sunbird.dp.core.job.FlinkKafkaConnector import org.sunbird.dp.core.util.FlinkUtil import org.sunbird.dp.preprocessor.domain.Event -import org.sunbird.dp.preprocessor.functions.{ShareEventsFlattenerFunction, TelemetryRouterFunction, TelemetryValidationFunction} +import org.sunbird.dp.preprocessor.functions.{PipelinePreprocessorFunction, ShareEventsFlattenerFunction, TelemetryRouterFunction, TelemetryValidationFunction} /** * Telemetry Pipeline processor stream task does the following pipeline processing in a sequence: @@ -61,52 +61,79 @@ class PipelinePreprocessorStreamTask(config: PipelinePreprocessorConfig, kafkaCo * 4. Share Events Flattener */ - val validationStream: SingleOutputStreamOperator[Event] = + val eventStream: SingleOutputStreamOperator[Event] = env.addSource(kafkaConsumer, config.pipelinePreprocessorConsumer) .uid(config.pipelinePreprocessorConsumer).setParallelism(config.kafkaConsumerParallelism) .rebalance() - .process(new TelemetryValidationFunction(config)).name(config.telemetryValidationFunction).uid(config.telemetryValidationFunction) - .setParallelism(config.validationParallelism) - - val routerStream: SingleOutputStreamOperator[Event] = - validationStream.getSideOutput(config.uniqueEventsOutputTag) - .process(new TelemetryRouterFunction(config)).name(config.telemetryRouterFunction).uid(config.telemetryRouterFunction) - .setParallelism(config.routerParallelism) - - val shareEventsFlattener: SingleOutputStreamOperator[Event] = - routerStream.getSideOutput(config.shareRouteEventsOutputTag) - .process(new ShareEventsFlattenerFunction(config)).name(config.shareEventsFlattenerFunction).uid(config.shareEventsFlattenerFunction) - .setParallelism(config.shareEventsFlattnerParallelism) + .process(new PipelinePreprocessorFunction(config)).setParallelism(config.downstreamOperatorsParallelism) /** * Sink for invalid events, duplicate events, log events, audit events and telemetry events */ - validationStream.getSideOutput(config.validationFailedEventsOutputTag).addSink(kafkaConnector.kafkaEventSink(config.kafkaFailedTopic)).name(config.invalidEventProducer).uid(config.invalidEventProducer) - validationStream.getSideOutput(config.duplicateEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDuplicateTopic)).name(config.duplicateEventProducer).uid(config.duplicateEventProducer) + eventStream.getSideOutput(config.validationFailedEventsOutputTag).addSink(kafkaConnector.kafkaEventSink(config.kafkaFailedTopic)).name(config.invalidEventProducer).uid(config.invalidEventProducer).setParallelism(config.downstreamOperatorsParallelism) + eventStream.getSideOutput(config.duplicateEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDuplicateTopic)).name(config.duplicateEventProducer).uid(config.duplicateEventProducer).setParallelism(config.downstreamOperatorsParallelism) /** * Routing LOG & ERROR Events to "event.log" & "events.error" topic respectively. */ - routerStream.getSideOutput(config.logEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaLogRouteTopic)).name(config.logRouterProducer).uid(config.logRouterProducer) - routerStream.getSideOutput(config.errorEventOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaErrorRouteTopic)).name(config.errorRouterProducer).uid(config.errorRouterProducer) + eventStream.getSideOutput(config.logEventsOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaLogRouteTopic)) + .name(config.logRouterProducer).uid(config.logRouterProducer) + .setParallelism(config.downstreamOperatorsParallelism) + + eventStream.getSideOutput(config.errorEventOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaErrorRouteTopic)) + .name(config.errorRouterProducer).uid(config.errorRouterProducer) + .setParallelism(config.downstreamOperatorsParallelism) /** * Pushing "AUDIT" event into both sink and audit topic */ - routerStream.getSideOutput(config.auditRouteEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaAuditRouteTopic)).name(config.auditRouterProducer).uid(config.auditRouterProducer) - routerStream.getSideOutput(config.auditRouteEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)).name(config.auditEventsPrimaryRouteProducer).uid(config.auditEventsPrimaryRouteProducer) + eventStream.getSideOutput(config.auditRouteEventsOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaAuditRouteTopic)) + .name(config.auditRouterProducer).uid(config.auditRouterProducer) + .setParallelism(config.downstreamOperatorsParallelism) + + eventStream.getSideOutput(config.auditRouteEventsOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)) + .name(config.auditEventsPrimaryRouteProducer).uid(config.auditEventsPrimaryRouteProducer) + .setParallelism(config.downstreamOperatorsParallelism) /** * Pushing all the events to unique topic (next stream = denorm) , except LOG, ERROR, AUDIT Events, */ - routerStream.getSideOutput(config.primaryRouteEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)).name(config.primaryRouterProducer).uid(config.primaryRouterProducer) + eventStream.getSideOutput(config.primaryRouteEventsOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)) + .name(config.primaryRouterProducer).uid(config.primaryRouterProducer) + .setParallelism(config.downstreamOperatorsParallelism) + + /** + * Splitting events based on priority and route to different topics (next stream = denorm) + */ + eventStream.getSideOutput(config.denormSecondaryEventsRouteOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDenormSecondaryRouteTopic)) + .name(config.denormSecondaryEventProducer).uid(config.denormSecondaryEventProducer) + .setParallelism(config.downstreamOperatorsParallelism) + + eventStream.getSideOutput(config.denormPrimaryEventsRouteOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDenormPrimaryRouteTopic)) + .name(config.denormPrimaryEventProducer).uid(config.denormPrimaryEventProducer) + .setParallelism(config.downstreamOperatorsParallelism) + /** * Pushing "SHARE and SHARE_ITEM" event into out put topic unique topic(next_streaming_process = denorm) */ + eventStream.getSideOutput(config.shareItemEventOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)).name(config.shareItemsPrimaryRouterProducer).uid(config.shareItemsPrimaryRouterProducer).setParallelism(config.downstreamOperatorsParallelism) + + /** + * pushing cbAudit events into cbAudit topic + */ - shareEventsFlattener.getSideOutput(config.primaryRouteEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)).name(config.shareEventsPrimaryRouteProducer).uid(config.shareEventsPrimaryRouteProducer) - shareEventsFlattener.getSideOutput(config.shareItemEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaPrimaryRouteTopic)).name(config.shareItemsPrimaryRouterProducer).uid(config.shareItemsPrimaryRouterProducer) + eventStream.getSideOutput(config.cbAuditRouteEventsOutputTag) + .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaCbAuditRouteTopic)) + .name(config.cbAuditRouterProducer).uid(config.cbAuditRouterProducer) + .setParallelism(config.downstreamOperatorsParallelism) env.execute(config.jobName) } diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/SchemaValidator.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/SchemaValidator.scala index c6a6e4a295..550448dfdf 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/SchemaValidator.scala +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/SchemaValidator.scala @@ -1,23 +1,20 @@ package org.sunbird.dp.preprocessor.util import java.io.{File, IOException} -import java.nio.file.{FileSystems, Files, Paths} +import java.nio.charset.StandardCharsets +import java.nio.file._ import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import com.github.fge.jackson.JsonLoader import com.github.fge.jsonschema.core.exceptions.ProcessingException import com.github.fge.jsonschema.core.report.ProcessingReport import com.github.fge.jsonschema.main.{JsonSchema, JsonSchemaFactory} -import com.google.common.io.ByteStreams -import com.google.gson.Gson +import io.github.classgraph.{ClassGraph, Resource} import org.slf4j.LoggerFactory import org.sunbird.dp.preprocessor.domain.Event import org.sunbird.dp.preprocessor.task.PipelinePreprocessorConfig -import scala.collection.JavaConverters._ import scala.collection.mutable -import scala.collection.mutable.ListBuffer -import scala.util.Try class SchemaValidator(config: PipelinePreprocessorConfig) extends java.io.Serializable { @@ -28,39 +25,29 @@ class SchemaValidator(config: PipelinePreprocessorConfig) extends java.io.Serial logger.info("Initializing schema for telemetry objects...") private val schemaJsonMap: Map[String, JsonSchema] = { - val schamaMap = new mutable.HashMap[String, JsonSchema]() - val schemaFactory = JsonSchemaFactory.byDefault - val schemaUrl = this.getClass.getClassLoader.getResource(s"${config.schemaPath}").toURI - // $COVERAGE-OFF$ Disabling code coverage for below code, It can be testable only incase of jar - val schemaFiles = if (schemaUrl.getScheme.equalsIgnoreCase("jar")) { - val fileSystem = FileSystems.newFileSystem(schemaUrl, Map[String, AnyRef]().asJava) - val files = loadSchemaFiles(fileSystem.getPath(s"${config.schemaPath}")) - fileSystem.close() - files - } else { - loadSchemaFiles(Paths.get(schemaUrl)) - } + readResourceFiles(s"${config.schemaPath}") + } - logger.info(s"Loaded ${schemaFiles.size} telemetry schema files...") - schemaFiles.map { schemaFile => - val schemaJson = - new String(ByteStreams.toByteArray( - this.getClass.getClassLoader.getResourceAsStream(s"${config.schemaPath}/$schemaFile") - )) - schamaMap += schemaFile.toString -> schemaFactory.getJsonSchema(JsonLoader.fromString(schemaJson)) + def readResourceFiles(schemaUrl: String): Map[String, JsonSchema] = { + val classGraphResult = new ClassGraph().acceptPaths(schemaUrl).scan() + val schemaFactory = JsonSchemaFactory.byDefault + val schemaMap = new mutable.HashMap[String, JsonSchema]() + try { + val resources = classGraphResult.getResourcesWithExtension("json") + resources.forEachByteArrayIgnoringIOException((res: Resource, content: Array[Byte]) => { + schemaMap += Paths.get(res.getPath).getFileName.toString -> schemaFactory.getJsonSchema(JsonLoader.fromString(new String(content, StandardCharsets.UTF_8))) + }) + } catch { + case ex: Exception => ex.printStackTrace() + throw ex + } finally { + classGraphResult.close() } - schamaMap.toMap + schemaMap.toMap } logger.info("Schema initialization completed for telemetry objects...") - def loadSchemaFiles(schemaDirPath: java.nio.file.Path): List[String] = { - val schemaFiles = Try(Files.newDirectoryStream(schemaDirPath)).map { stream => - stream.iterator().asScala.toList.map(path => path.getFileName.toString) - }.getOrElse(List[String]()) - schemaFiles - } - def schemaFileExists(event: Event): Boolean = schemaJsonMap.contains(event.schemaName) @throws[IOException] diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/ShareEventsFlattener.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/ShareEventsFlattener.scala new file mode 100644 index 0000000000..085c80ac66 --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/ShareEventsFlattener.scala @@ -0,0 +1,91 @@ +package org.sunbird.dp.preprocessor.util + +import java.util +import java.util.UUID + +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.gson.Gson +import org.sunbird.dp.core.util.JSONUtil +// import com.google.gson.Gson +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.sunbird.dp.core.job.Metrics +import org.sunbird.dp.preprocessor.domain._ +import org.sunbird.dp.preprocessor.task.PipelinePreprocessorConfig +import org.sunbird.dp.preprocessor.domain.{Context => EventContext} + +import scala.util.Try + +class ShareEventsFlattener(config: PipelinePreprocessorConfig) extends java.io.Serializable { + + private val serialVersionUID = 1167435095740381669L + val objectMapper = new ObjectMapper() + + def flatten(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics) = { + + event.edataItems().forEach(items => { + val version = items.get("ver").asInstanceOf[String] + val identifier = items.get("id").asInstanceOf[String] + val contentType = items.get("type").asInstanceOf[String] + val paramsList = Option(items.get("params").asInstanceOf[util.ArrayList[Map[String, AnyRef]]]) + + if (paramsList.isDefined) { + paramsList.getOrElse(new util.ArrayList[util.Map[String, AnyRef]]()).forEach( + param => { + val shareEventParams = param.asInstanceOf[util.Map[String, Any]] + val transfers = Try(shareEventParams.get("transfers").toString.toDouble).getOrElse(0d) + val size = Try(shareEventParams.get("size").toString.toDouble).getOrElse(0d) + val edataType = if (transfers == 0) "download" else "import" + val shareItemStr = generateShareItemEvents(event, EventObject(id = identifier, ver = version, + `type` = contentType, rollup = Rollup(event.objectID())), Some(edataType), Some(size)) + // context.output(config.shareItemEventOutputTag, new Event(new Gson().fromJson(shareItemEvent, new util.LinkedHashMap[String, Any]().getClass))) + val shareItemEvent = new Event(JSONUtil.deserialize[util.LinkedHashMap[String, Any]](shareItemStr)) + context.output(config.shareItemEventOutputTag, shareItemEvent) + routeShareItemEvents(shareItemEvent, context, metrics) + } + ) + } else { + val shareItemStr = generateShareItemEvents(event, EventObject(id = identifier, ver = version, + `type` = contentType, rollup = Rollup(event.objectID())), edataType = Some(event.edataType())) + // context.output(config.shareItemEventOutputTag, new Event(new Gson().fromJson(shareItemEvent, new util.LinkedHashMap[String, Any]().getClass))) + val shareItemEvent = new Event(JSONUtil.deserialize[util.LinkedHashMap[String, Any]](shareItemStr)) + context.output(config.shareItemEventOutputTag, shareItemEvent) + routeShareItemEvents(shareItemEvent, context, metrics) + } + metrics.incCounter(config.shareItemEventsMetircsCount) + }) + + event.markSuccess(config.SHARE_EVENTS_FLATTEN_FLAG_NAME) + context.output(config.primaryRouteEventsOutputTag, event) + + } + + + def generateShareItemEvents(event: Event, eventObj: EventObject, edataType: Option[String], paramSize: Option[Double] = None): String = { + val shareItemEvent = ShareEvent( + ActorObject(event.actorId(), event.actorType()), + "SHARE_ITEM", + EData(event.edataDir, edataType.orNull, paramSize.getOrElse(0)), + ver = "3.0", + syncts = event.eventSyncTs.asInstanceOf[Number].longValue, + ets = event.ets(), + context = EventContext(event.channel(), event.env, event.sessionId, event.did(), event.eventPData, event.cdata, event.rollup), + mid = event.mid() + "-" + UUID.randomUUID().toString, + eventObj, + event.eventTags + ) + + new Gson().toJson(shareItemEvent) + } + + def routeShareItemEvents(shareItem: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics) = { + if (config.secondaryEvents.contains("SHARE_ITEM")) { + context.output(config.denormSecondaryEventsRouteOutputTag, shareItem) + metrics.incCounter(metric = config.denormSecondaryEventsRouterMetricsCount) + } + else { + context.output(config.denormPrimaryEventsRouteOutputTag, shareItem) + metrics.incCounter(metric = config.denormPrimaryEventsRouterMetricsCount) + } + } + +} diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/TelemetryValidator.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/TelemetryValidator.scala new file mode 100644 index 0000000000..6717a56c51 --- /dev/null +++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/TelemetryValidator.scala @@ -0,0 +1,64 @@ +package org.sunbird.dp.preprocessor.util + +import com.github.fge.jsonschema.core.report.ProcessingReport +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.dp.core.job.Metrics +import org.sunbird.dp.preprocessor.domain.Event +import org.sunbird.dp.preprocessor.task.PipelinePreprocessorConfig + +class TelemetryValidator(config: PipelinePreprocessorConfig) extends java.io.Serializable { + + private val serialVersionUID = - 2903685244618349843L + private[this] val logger = LoggerFactory.getLogger(classOf[TelemetryValidator]) + + private val schemaValidator = new SchemaValidator(config) + + def validate(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics): Boolean = { + val isSchemaPresent: Boolean = schemaValidator.schemaFileExists(event) + if (isSchemaPresent) { + val validationReport = schemaValidator.validate(event, isSchemaPresent = isSchemaPresent) + if (validationReport.isSuccess) { + onValidationSuccess(event, metrics, context) + } else { + onValidationFailure(event, metrics, context, validationReport) + } + validationReport.isSuccess + } else { + onMissingSchema(event, metrics, context, "Schema not found: eid looks incorrect, sending to failed") + false + } + } + + private def dataCorrection(event: Event): Unit = { + // Remove prefix from federated userIds + val eventActorId = event.actorId() + if (eventActorId != null && !eventActorId.isEmpty && eventActorId.startsWith("f:")) + event.updateActorId(eventActorId.substring(eventActorId.lastIndexOf(":") + 1)) + if (event.objectFieldsPresent && (event.objectType().equalsIgnoreCase("DialCode") || event.objectType().equalsIgnoreCase("qr"))) event.correctDialCodeValue() + } + + def onValidationSuccess(event: Event, metrics: Metrics, context: ProcessFunction[Event, Event]#Context): Unit = { + logger.debug(s"Telemetry schema validation is success: ${event.mid()}") + dataCorrection(event) + event.markSuccess(config.VALIDATION_FLAG_NAME) + metrics.incCounter(config.validationSuccessMetricsCount) + event.updateDefaults(config) + } + + def onValidationFailure(event: Event, metrics: Metrics, context: ProcessFunction[Event, Event]#Context, validationReport: ProcessingReport): Unit = { + val failedErrorMsg = schemaValidator.getInvalidFieldName(validationReport.toString) + logger.debug(s"Telemetry schema validation is failed for: ${event.mid()} and error message is: ${validationReport.toString}") + event.markValidationFailure(failedErrorMsg, config.VALIDATION_FLAG_NAME) + metrics.incCounter(config.validationFailureMetricsCount) + context.output(config.validationFailedEventsOutputTag, event) + } + + def onMissingSchema(event: Event, metrics: Metrics, context: ProcessFunction[Event, Event]#Context, message: String): Unit = { + logger.debug(s"Telemetry schema validation is failed for: ${event.mid()} and error message is: $message") + event.markValidationFailure(message, config.VALIDATION_FLAG_NAME) + metrics.incCounter(config.validationFailureMetricsCount) + context.output(config.validationFailedEventsOutputTag, event) + } + +} diff --git a/data-pipeline-flink/pipeline-preprocessor/src/test/resources/test.conf b/data-pipeline-flink/pipeline-preprocessor/src/test/resources/test.conf index 2e1cb89cec..b2d53b3c64 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/test/resources/test.conf +++ b/data-pipeline-flink/pipeline-preprocessor/src/test/resources/test.conf @@ -5,21 +5,22 @@ kafka { output.failed.topic = "flink.telemetry.failed" output.primary.route.topic = "flink.telemetry.unique" output.log.route.topic = "flink.druid.events.log" - output.error.route.topic = "flink.druid.events.error" + output.error.route.topic = "flink.telemetry.error" output.audit.route.topic = "flink.telemetry.audit" output.duplicate.topic = "flink.telemetry.duplicate" + output.denorm.secondary.route.topic = "flink.telemetry.unique.secondary" + output.denorm.primary.route.topic = "flink.telemetry.unique.primary" groupId = "flink-pipeline-preprocessor-group" } task { - telemetry.validation.parallelism = 1 - telemetry.router.parallelism = 1 - share.events.flattener.parallelism = 1 + downstream.operators.parallelism = 1 } telemetry.schema.path="schemas/telemetry/3.0" default.channel="b00bc992ef25f1a9a8d63291e20efc8d" dedup.producer.included.ids = ["dev.sunbird.portal", "dev.sunbird.desktop"] +secondary.events = ["INTERACT", "IMPRESSION", "SHARE_ITEM"] redis { host = 127.0.0.1 diff --git a/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala index 7616a76ebf..46f11c6d82 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala +++ b/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala @@ -123,4 +123,38 @@ object EventFixtures { |{"ver":"3.0","ets":1577278681178,"actor":{"type":"User","id":"7c3ea1bb-4da1-48d0-9cc0-c4f150554149"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.sunbird.desktop","pid":"sunbird.app","ver":"2.3.162"},"env":"app","sid":"82e41d87-e33f-4269-aeae-d56394985599","did":"1b17c32bad61eb9e33df281eecc727590d739b2b"},"edata":{"dir":"In","type":"File","items":[{"origin":{"id":"1b17c32bad61eb9e33df281eecc727590d739b2b","type":"Device"},"id":"do_312785709424099328114191","type":"CONTENT","ver":"1","params":[{"transfers":0,"size":21084308}]},{"origin":{"id":"1b17c32bad61eb9e33df281eecc727590d739b2b","type":"Device"},"id":"do_31277435209002188818711","type":"CONTENT","ver":"18","params":[{"transfers":12,"size":"123"}]},{"origin":{"id":"1b17c32bad61eb9e33df281eecc727590d739b2b","type":"Device"},"id":"do_31278794857559654411554","type":"TextBook","ver":"1"}]},"object":{"id":"do_312528116260749312248818","type":"TextBook","version":"10","rollup":{}},"mid":"02ba33e5-15fe-4ec5-b32.1084308E760-3d03429fae84","syncts":1577278682630,"@timestamp":"2019-12-25T12:58:02.630Z","type":"events"} |""".stripMargin + // Invalid SUMMARY event with edata.extra.[].value datatype mismatch + val EVENT_13 = + """ + |{"eid":"SUMMARY","ets":1625043400402,"ver":"3.0","mid":"SUMMARY:a3e517153c4ba392297e70521aa5e17a","actor":{"id":"70e496fe83dee324009a847ea222bc5c","type":"User"},"context":{"channel":"01268904781886259221","pdata":{"id":"staging.sunbird.portal","ver":"4.1.0","pid":"sunbird-portal"},"env":"contentplayer","sid":"73d82044-8ea5-dffc-1af5-6cdf2a1fa1da","did":"70e496fe83dee324009a847ea222bc5c","cdata":[{"id":"kubXMwcsJK2JANa0PeYc00GK5CSXoS1q","type":"ContentSession"},{"id":"xcFG0rntKUGltu2m8zJh7ZqattT9u3Ix","type":"PlaySession"},{"id":"2.0","type":"PlayerVersion"}],"rollup":{"l1":"01268904781886259221"},"uid":"anonymous"},"object":{"id":"do_213302422998196224166","ver":"1","type":"Content","rollup":{}},"tags":["01268904781886259221"],"edata":{"type":"content","mode":"play","starttime":1625043385301,"endtime":1625043401236,"timespent":16,"pageviews":2,"interactions":2,"extra":[{"id":"progress","value":100},{"id":"endpageseen","value":true},{"id":"score","value":2},{"id":"correct","value":2},{"id":"incorrect","value":0},{"id":"partial","value":0},{"id":"skipped","value":0}]}} + | + |""".stripMargin + + // Invalid SUMMARY event with edata.timespent datatype mismatch + val EVENT_14 = + """ + |{"eid":"SUMMARY","ets":1625043400402,"ver":"3.0","mid":"SUMMARY:a3e517153c4ba392297e70521aa5e17a","actor":{"id":"70e496fe83dee324009a847ea222bc5c","type":"User"},"context":{"channel":"01268904781886259221","pdata":{"id":"staging.sunbird.portal","ver":"4.1.0","pid":"sunbird-portal"},"env":"contentplayer","sid":"73d82044-8ea5-dffc-1af5-6cdf2a1fa1da","did":"70e496fe83dee324009a847ea222bc5c","cdata":[{"id":"kubXMwcsJK2JANa0PeYc00GK5CSXoS1q","type":"ContentSession"},{"id":"xcFG0rntKUGltu2m8zJh7ZqattT9u3Ix","type":"PlaySession"},{"id":"2.0","type":"PlayerVersion"}],"rollup":{"l1":"01268904781886259221"},"uid":"anonymous"},"object":{"id":"do_213302422998196224166","ver":"1","type":"Content","rollup":{}},"tags":["01268904781886259221"],"edata":{"type":"content","mode":"play","starttime":1625043385301,"endtime":1625043401236,"timespent":"0:16","pageviews":2,"interactions":2,"extra":[{"id":"progress","value":"100"},{"id":"endpageseen","value":"true"},{"id":"score","value":"2"},{"id":"correct","value":"2"},{"id":"incorrect","value":"0"},{"id":"partial","value":"0"},{"id":"skipped","value":"0"}]}} + | + |""".stripMargin + + // Valid SUMMARY event + val EVENT_15 = + """ + |{"eid":"SUMMARY","ets":1625043400402,"ver":"3.0","mid":"SUMMARY:a3e517153c4ba392297e70521aa5e17a","actor":{"id":"70e496fe83dee324009a847ea222bc5c","type":"User"},"context":{"channel":"01268904781886259221","pdata":{"id":"staging.sunbird.portal","ver":"4.1.0","pid":"sunbird-portal"},"env":"contentplayer","sid":"73d82044-8ea5-dffc-1af5-6cdf2a1fa1da","did":"70e496fe83dee324009a847ea222bc5c","cdata":[{"id":"kubXMwcsJK2JANa0PeYc00GK5CSXoS1q","type":"ContentSession"},{"id":"xcFG0rntKUGltu2m8zJh7ZqattT9u3Ix","type":"PlaySession"},{"id":"2.0","type":"PlayerVersion"}],"rollup":{"l1":"01268904781886259221"},"uid":"anonymous"},"object":{"id":"do_213302422998196224166","ver":"1","type":"Content","rollup":{}},"tags":["01268904781886259221"],"edata":{"type":"content","mode":"play","starttime":1625043385301,"endtime":1625043401236,"timespent":16,"pageviews":2,"interactions":2,"extra":[{"id":"progress","value":"100"},{"id":"endpageseen","value":"true"},{"id":"score","value":"2"},{"id":"correct","value":"2"},{"id":"incorrect","value":"0"},{"id":"partial","value":"0"},{"id":"skipped","value":"0"}]}} + | + |""".stripMargin + + // Invalid METRICS event with edata.subsystem missing + val EVENT_16 = + """ + |{"@timestamp":"2021-05-09T06:53:58.317Z","actor":{"id":"5e78cb4a62f5c60d3a007ff366c740727afa2aec42f9e63e26fb1a879b87513b","type":"User"},"context":{"cdata":[],"did":"5e78cb4a62f5c60d3a007ff366c740727afa2aec42f9e63e26fb1a879b87513b","channel":"desktop","env":"DesktopApp","pdata":{"id":"desktop,app","pid":"desktop.app","ver":"1.0"},"sid":"0e2dcdd2-e81c-4e9c-9c43-b41a6fdd6c48"},"edata":{"metrics":[{"metric":"minApiTime","value":0.002482144},{"metric":"maxApiTime","value":7.236265204},{"metric":"avgApiTime","value":0.5468935359999998},{"metric":"totalApiS","value":50.0},{"metric":"minAppStartupTime","value":6.566},{"metric":"maxAppStartupTime","value":11.422},{"metric":"avgAppStartupTime","value":9.755999999999998},{"metric":"totalAppStartupS","value":3.0},{"metric":"minImportTime","value":1.196166354944613},{"metric":"maxImportTime","value":1.196166354944613},{"metric":"avgImportTime","value":1.196166354944613},{"metric":"totalImportS","value":1.0},{"metric":"createdDate","value":1620431999999.0}],"system":"DesktopApp"},"eid":"METRICS","ets":1620542437965.0,"mid":"METRICS:1711d2706507cbd2e22ec88f7660a569","object":{},"syncts":1620543238317,"tags":[],"ver":"3.0"} + | + |""".stripMargin + + // Valid METRICS event + val EVENT_17 = + """ + |{"@timestamp":"2021-05-09T06:53:58.317Z","actor":{"id":"5e78cb4a62f5c60d3a007ff366c740727afa2aec42f9e63e26fb1a879b87513b","type":"User"},"context":{"cdata":[],"did":"5e78cb4a62f5c60d3a007ff366c740727afa2aec42f9e63e26fb1a879b87513b","channel":"desktop","env":"DesktopApp","pdata":{"id":"desktop,app","pid":"desktop.app","ver":"1.0"},"sid":"0e2dcdd2-e81c-4e9c-9c43-b41a6fdd6c48"},"edata":{"metrics":[{"metric":"minApiTime","value":0.002482144},{"metric":"maxApiTime","value":7.236265204},{"metric":"avgApiTime","value":0.5468935359999998},{"metric":"totalApiS","value":50.0},{"metric":"minAppStartupTime","value":6.566},{"metric":"maxAppStartupTime","value":11.422},{"metric":"avgAppStartupTime","value":9.755999999999998},{"metric":"totalAppStartupS","value":3.0},{"metric":"minImportTime","value":1.196166354944613},{"metric":"maxImportTime","value":1.196166354944613},{"metric":"avgImportTime","value":1.196166354944613},{"metric":"totalImportS","value":1.0},{"metric":"createdDate","value":1620431999999.0}],"subsystem":"DesktopApp","system":"DesktopApp"},"eid":"METRICS","ets":1620542437965.0,"mid":"METRICS:1711d2706507cbd2e22ec88f7660a569","object":{},"syncts":1620543238317,"tags":[],"ver":"3.0"} + | + |""".stripMargin } diff --git a/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/spec/PipelineProcessorStreamTaskTestSpec.scala b/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/spec/PipelineProcessorStreamTaskTestSpec.scala index 894b9a5ebd..94ea9dee1b 100644 --- a/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/spec/PipelineProcessorStreamTaskTestSpec.scala +++ b/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/spec/PipelineProcessorStreamTaskTestSpec.scala @@ -16,10 +16,15 @@ import org.mockito.Mockito.when import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} import org.sunbird.dp.fixture.EventFixtures import org.sunbird.dp.core.job.FlinkKafkaConnector +import org.sunbird.dp.core.util.JSONUtil import org.sunbird.dp.preprocessor.domain.Event import org.sunbird.dp.preprocessor.task.{PipelinePreprocessorConfig, PipelinePreprocessorStreamTask} import redis.embedded.RedisServer +import scala.collection.JavaConverters._ + +case class SHARE_ITEM_EVENT(objectId: String, objectType: String) + class PipelineProcessorStreamTaskTestSpec extends BaseTestSpec { implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) @@ -53,7 +58,9 @@ class PipelineProcessorStreamTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaErrorRouteTopic)).thenReturn(new TelemetryErrorEventSink) when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaFailedTopic)).thenReturn(new TelemetryFailedEventsSink) when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaAuditRouteTopic)).thenReturn(new TelemetryAuditEventSink) - when(mockKafkaUtil.kafkaStringSink(ppConfig.kafkaPrimaryRouteTopic)).thenReturn(new ShareItemEventSink) + + when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaDenormSecondaryRouteTopic)).thenReturn(new TelemetryDenormSecondaryEventSink) + when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaDenormPrimaryRouteTopic)).thenReturn(new TelemetryDenormPrimaryEventSink) flinkCluster.before() } @@ -69,14 +76,18 @@ class PipelineProcessorStreamTaskTestSpec extends BaseTestSpec { val task = new PipelinePreprocessorStreamTask(ppConfig, mockKafkaUtil) task.process() - ShareItemEventSink.values.size() should be(3) - TelemetryPrimaryEventSink.values.size() should be(5) - TelemetryFailedEventsSink.values.size() should be(4) + // 5 telemetry and 3 SHARE_ITEM + TelemetryPrimaryEventSink.values.size() should be(10) + TelemetryPrimaryEventSink.values.asScala.count(event => event.eid().equals("SHARE_ITEM")) should be (3) + TelemetryFailedEventsSink.values.size() should be(7) DupEventsSink.values.size() should be(1) TelemetryAuditEventSink.values.size() should be(1) TelemetryLogEventSink.values.size() should be(1) TelemetryErrorEventSink.values.size() should be(1) + TelemetryDenormSecondaryEventSink.values.size() should be(4) // 1 INTERACT and 3 SHARE_ITEM + TelemetryDenormPrimaryEventSink.values.size() should be(6) + /** * * 1. primary-route-success-count -> 05 * * 2. audit-route-success-count -> 01 @@ -90,22 +101,38 @@ class PipelineProcessorStreamTaskTestSpec extends BaseTestSpec { * * 10. unique-event-count -> 02 * * 12. share-item-event-success-count -> 03 */ - - BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.primaryRouterMetricCount}").getValue() should be(5) + val expectedShareItems: List[SHARE_ITEM_EVENT] = List( + SHARE_ITEM_EVENT(objectId = "do_312785709424099328114191", objectType = "CONTENT"), + SHARE_ITEM_EVENT(objectId = "do_31277435209002188818711", objectType = "CONTENT"), + SHARE_ITEM_EVENT(objectId = "do_31278794857559654411554", objectType = "TextBook") + ) + + val shareItems = TelemetryPrimaryEventSink.values.asScala.filter(event => event.eid().equals("SHARE_ITEM")) + shareItems.foreach { + event => + val shareItemObject = event.getTelemetry.read[util.HashMap[String, AnyRef]]("object").getOrElse(new util.HashMap()).asScala + val actualShareItem = SHARE_ITEM_EVENT(objectId = shareItemObject("id").asInstanceOf[String], shareItemObject("type").asInstanceOf[String]) + expectedShareItems should contain (actualShareItem) + } + + BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.primaryRouterMetricCount}").getValue() should be(7) BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.shareItemEventsMetircsCount}").getValue() should be(3) BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.auditEventRouterMetricCount}").getValue() should be(1) BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.shareEventsRouterMetricCount}").getValue() should be(1) BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.logEventsRouterMetricsCount}").getValue() should be(1) BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.errorEventsRouterMetricsCount}").getValue() should be(1) + BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.validationSuccessMetricsCount}").getValue() should be(10) + BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.validationFailureMetricsCount}").getValue() should be(7) - BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.validationSuccessMetricsCount}").getValue() should be(8) - BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.validationFailureMetricsCount}").getValue() should be(4) - - BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.unique-event-count").getValue() should be(7) + BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.unique-event-count").getValue() should be(8) // ONLY LOG events are skipped from dedup BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.duplicate-event-count").getValue() should be(1) + + BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.denormSecondaryEventsRouterMetricsCount}").getValue() should be(4) + BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.denormPrimaryEventsRouterMetricsCount}").getValue() should be(6) + + } } -} class PipeLineProcessorEventSource extends SourceFunction[Event] { @@ -123,6 +150,11 @@ class PipeLineProcessorEventSource extends SourceFunction[Event] { val event10 = gson.fromJson(EventFixtures.EVENT_10, new util.LinkedHashMap[String, Any]().getClass) val event11 = gson.fromJson(EventFixtures.EVENT_11, new util.LinkedHashMap[String, Any]().getClass) val event12 = gson.fromJson(EventFixtures.EVENT_12, new util.LinkedHashMap[String, Any]().getClass) + val event13 = gson.fromJson(EventFixtures.EVENT_13, new util.LinkedHashMap[String, Any]().getClass) + val event14 = gson.fromJson(EventFixtures.EVENT_14, new util.LinkedHashMap[String, Any]().getClass) + val event15 = gson.fromJson(EventFixtures.EVENT_15, new util.LinkedHashMap[String, Any]().getClass) + val event16 = gson.fromJson(EventFixtures.EVENT_16, new util.LinkedHashMap[String, Any]().getClass) + val event17 = gson.fromJson(EventFixtures.EVENT_17, new util.LinkedHashMap[String, Any]().getClass) ctx.collect(new Event(event1)) ctx.collect(new Event(event2)) ctx.collect(new Event(event3)) @@ -135,25 +167,17 @@ class PipeLineProcessorEventSource extends SourceFunction[Event] { ctx.collect(new Event(event10)) ctx.collect(new Event(event11)) ctx.collect(new Event(event12)) + ctx.collect(new Event(event13)) + ctx.collect(new Event(event14)) + ctx.collect(new Event(event15)) + ctx.collect(new Event(event16)) + ctx.collect(new Event(event17)) } override def cancel() = {} } -class ShareItemEventSink extends SinkFunction[String] { - - override def invoke(value: String): Unit = { - synchronized { - ShareItemEventSink.values.add(value) - } - } -} - -object ShareItemEventSink { - val values: util.List[String] = new util.ArrayList() -} - class TelemetryFailedEventsSink extends SinkFunction[Event] { override def invoke(value: Event): Unit = { @@ -233,4 +257,30 @@ class DupEventsSink extends SinkFunction[Event] { object DupEventsSink { val values: util.List[Event] = new util.ArrayList() +} + +class TelemetryDenormSecondaryEventSink extends SinkFunction[Event] { + + override def invoke(value: Event): Unit = { + synchronized { + TelemetryDenormSecondaryEventSink.values.add(value) + } + } +} + +object TelemetryDenormSecondaryEventSink { + val values: util.List[Event] = new util.ArrayList() +} + +class TelemetryDenormPrimaryEventSink extends SinkFunction[Event] { + + override def invoke(value: Event): Unit = { + synchronized { + TelemetryDenormPrimaryEventSink.values.add(value) + } + } +} + +object TelemetryDenormPrimaryEventSink { + val values: util.List[Event] = new util.ArrayList() } \ No newline at end of file diff --git a/data-pipeline-flink/pom.xml b/data-pipeline-flink/pom.xml index 9bd5dd9139..8c337d009a 100644 --- a/data-pipeline-flink/pom.xml +++ b/data-pipeline-flink/pom.xml @@ -21,26 +21,30 @@ dp-core + ingest-router telemetry-extractor pipeline-preprocessor de-normalization druid-events-validator device-profile-updater - user-cache-updater content-cache-updater assessment-aggregator + user-cache-updater-2.0 + cb-preprocessor + rating UTF-8 - 2.12 - 2.12.11 - 1.10.1 + 2.12 + 2.12.11 + 1.13.5 2.4.0 + 11 1.9.13 1.4.0 - release-3.0.2 + release-4.6.0 @@ -73,10 +77,10 @@ org.apache.maven.plugins maven-compiler-plugin - 3.1 + 3.8.1 - 1.8 - 1.8 + ${java.target.runtime} + ${java.target.runtime} diff --git a/data-pipeline-flink/rating/pom.xml b/data-pipeline-flink/rating/pom.xml new file mode 100644 index 0000000000..d5ba245d9b --- /dev/null +++ b/data-pipeline-flink/rating/pom.xml @@ -0,0 +1,284 @@ + + + + 4.0.0 + + 3.0.1 + + + + org.sunbird.dp + pipeline-jobs + 1.0 + + + org.sunbird.dp.jobs + rating + 2.0.0 + jar + Rating + + Rating Feature + + + + UTF-8 + 1.4.0 + + + + + + + + + + + + + + + + + + + + + org.apache.flink + flink-streaming-scala_${scala.maj.version} + ${flink.version} + provided + + + org.sunbird.dp + dp-core + 1.0.0 + + + org.sunbird.dp + dp-core + 1.0.0 + test-jar + test + + + org.apache.flink + flink-test-utils_2.12 + ${flink.version} + test + + + org.apache.flink + flink-runtime_2.12 + ${flink.version} + test + tests + + + it.ozimov + embedded-redis + 0.7.1 + test + + + com.google.guava + guava + + + + + org.cassandraunit + cassandra-unit + 3.11.2.0 + test + + + org.apache.flink + flink-streaming-java_2.12 + ${flink.version} + test + tests + + + org.scalatest + scalatest_2.12 + 3.0.6 + test + + + org.mockito + mockito-core + 3.3.3 + test + + + com.fiftyonred + mock-jedis + 0.4.0 + test + + + com.squareup.okhttp3 + mockwebserver + 4.4.0 + test + + + + + + + + + + + + + + + + + + + + + + + + + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + + package + + shade + + + false + + + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.sunbird.dp.rating.task.RatingTask + + + + reference.conf + + + + + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + maven-surefire-plugin + 2.22.2 + + true + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + rating-testsuite.txt + + + + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + + + + + + + + + + + + diff --git a/data-pipeline-flink/rating/src/main/resources/rating-feature.conf b/data-pipeline-flink/rating/src/main/resources/rating-feature.conf new file mode 100644 index 0000000000..90c2ab334f --- /dev/null +++ b/data-pipeline-flink/rating/src/main/resources/rating-feature.conf @@ -0,0 +1,26 @@ +include "base-config.conf" + +kafka { +# input.topic = "rating" +# output.topic = "failedRating" + + output.topic = ${job.env}."rating.failedRating" + input.topic = ${job.env}."dev.rating.event" + groupId = ${job.env}"-rating-group" + +} + +task { + rating.parallelism = 1 +} + +ext-cassandra { + keyspace = "sunbird" + courses_keyspace="sunbird_courses" + course_table = "user_enrolments" + ratings_table = "ratings" + ratings_lookup_table = "ratings_lookup" + ratings_summary_table = "ratings_summary" + host = "localhost" + port = "9042" +} diff --git a/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/domain/Event.scala b/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/domain/Event.scala new file mode 100644 index 0000000000..b8738cd0ae --- /dev/null +++ b/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/domain/Event.scala @@ -0,0 +1,50 @@ +package org.sunbird.dp.rating.domain + +import java.util +import org.sunbird.dp.core.domain.{Events, EventsPath} + + +class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { + private val jobName = "ratingFeature" + + def versionNumber: Int = { + telemetry.read[Int]("version").get + + } + + def action: String = { + telemetry.read[String]("action").get + } + + def activityId: String = { + telemetry.read[String]("activity_id").get + } + + def activityType: String = { + telemetry.read[String]("activity_Type").get + } + + def userId: String = { + telemetry.read[String]("user_id").get + } + + def createdDate: String = { + telemetry.read[String]("created_Date").get + } + + def prevValues: util.Map[String, AnyRef] = { + telemetry.read[util.Map[String, AnyRef]]("prevValues").getOrElse(null) + } + + def updatedValues: util.Map[String, AnyRef] = { + telemetry.read[util.Map[String, AnyRef]]("updatedValues").getOrElse(null) + } + + def markFailed(errorMsg: String): Unit = { + telemetry.addFieldIfAbsent(EventsPath.FLAGS_PATH, new util.HashMap[String, Boolean]) + telemetry.addFieldIfAbsent("metadata", new util.HashMap[String, AnyRef]) + telemetry.add("metadata.validation_error", errorMsg) + telemetry.add("metadata.src", jobName) + } + +} diff --git a/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/function/RatingFunction.scala b/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/function/RatingFunction.scala new file mode 100644 index 0000000000..d34e15fbdb --- /dev/null +++ b/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/function/RatingFunction.scala @@ -0,0 +1,350 @@ +package org.sunbird.dp.rating.function + +import com.datastax.driver.core.Row +import com.datastax.driver.core.querybuilder.QueryBuilder +import com.google.gson.Gson +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.dp.contentupdater.core.util.RestUtil +import org.sunbird.dp.core.cache.DataCache +import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics} +import org.sunbird.dp.core.util.{CassandraUtil, JSONUtil} +import org.sunbird.dp.rating.domain.Event +import org.sunbird.dp.rating.task.RatingConfig + +import java.util.{HashMap, UUID} +import scala.collection.mutable + +import scala.collection.JavaConverters._ + +class RatingFunction(config: RatingConfig, @transient var cassandraUtil: CassandraUtil = null)(implicit val mapTypeInfo: TypeInformation[Event]) + extends BaseProcessFunction[Event, Event](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[RatingFunction]) + + private var dataCache: DataCache = _ + private var restUtil: RestUtil = _ + + override def metricsList(): List[String] = { + List() + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + } + + case class RatingJson(objectType: String, var user_id: String, var date: String, var rating: Float, var review: String) + + override def processElement(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics): Unit = { + var userStatus: Boolean = false + try { + val query = QueryBuilder.select().column("userid").from(config.dbCoursesKeyspace, config.courseTable) + .where(QueryBuilder.eq(config.userId, event.userId)).and(QueryBuilder.eq(config.courseId, event.activityId)) + val rows: java.util.List[Row] = cassandraUtil.find(query.toString); + if (null != rows && !rows.isEmpty) { + userStatus = true + var delta = 0.0f + val prevRatingValue = event.prevValues + if(prevRatingValue!=null){ + delta = event.updatedValues.get("rating").asInstanceOf[Double].toFloat - event.prevValues.get("rating").asInstanceOf[Double].toFloat + }else{ + delta = event.updatedValues.get("rating").asInstanceOf[Double].toFloat + } + val validReview = event.updatedValues.get("review").asInstanceOf[String] + var tempRow: Row = null + val ratingQuery = QueryBuilder.select().all().from(config.dbKeyspace, config.ratingsSummaryTable) + .where(QueryBuilder.eq(config.activityId, event.activityId)) + .and(QueryBuilder.eq(config.activityType, event.activityType)).toString + val ratingRows: java.util.List[Row] = cassandraUtil.find(ratingQuery.toString); + + if (delta != 0.0f || (validReview!=null && validReview.size > 100)) { + var updatedRating: Float = 0.0f + var updatedRatingValues: HashMap[Float, Float] = new HashMap[Float, Float]() + var prevRating: Float = 0.0f + var x = 0.0f + var sumOfTotalRating: Float = 0.0f + var totalNumberOfRatings: Float = 0.0f + var summary: String = null + if (null != ratingRows && !ratingRows.isEmpty) { + tempRow = ratingRows.asScala.toList(0) + if (delta != 0.0f) { + if(prevRatingValue!=null) { + prevRating = event.prevValues.get("rating").asInstanceOf[Double].toFloat + } + updatedRating = event.updatedValues.get("rating").asInstanceOf[Double].toFloat + updatedRatingValues = update_ratings_count(tempRow, prevRating, updatedRating) + sumOfTotalRating = tempRow.getFloat("sum_of_total_ratings") + delta + totalNumberOfRatings = tempRow.getFloat("total_number_of_ratings") + if(prevRatingValue==null){ + totalNumberOfRatings = totalNumberOfRatings + 1.0f + } + if(tempRow.getString("latest50reviews")!=null) { + summary = tempRow.getString("latest50reviews") + } + } + + if ((validReview!=null && validReview.size > 100) && delta == 0.0f) { + sumOfTotalRating = tempRow.getFloat("sum_of_total_ratings") + totalNumberOfRatings = tempRow.getFloat("total_number_of_ratings") + prevRating = event.prevValues.get("rating").asInstanceOf[Double].toFloat + updatedRating = event.updatedValues.get("rating").asInstanceOf[Double].toFloat + updatedRatingValues = update_ratings_count(tempRow, prevRating, updatedRating) + if(tempRow.getString("latest50reviews")!=null) { + summary = tempRow.getString("latest50reviews") + } + } + } + else { + updatedRating = event.updatedValues.get("rating").asInstanceOf[Double].toFloat + updatedRatingValues = update_ratings_count(tempRow, 0.0f, updatedRating) + sumOfTotalRating = event.updatedValues.get("rating").asInstanceOf[Double].toFloat + totalNumberOfRatings = 1.0f + } + updateDB(event, updatedRatingValues, sumOfTotalRating, + totalNumberOfRatings, + summary) + + } + if ((validReview!=null && validReview.size < 100) && delta == 0.0f) { + tempRow = ratingRows.asScala.toList(0) + val sumOfTotalRating = tempRow.getFloat("sum_of_total_ratings") + val totalNumberOfRatings = tempRow.getFloat("total_number_of_ratings") + val updatedRatingValues = update_ratings_count(tempRow, event.prevValues.get("rating").asInstanceOf[Double].toFloat, event.updatedValues.get("rating").asInstanceOf[Double].toFloat) + var summary: String = null + if(tempRow.getString("latest50reviews")!=null) { + summary = tempRow.getString("latest50reviews") + } + updateDB(event, updatedRatingValues, sumOfTotalRating, + totalNumberOfRatings, + summary) + } + if (null != getRatingLookUp(event)) { + deleteRatingLookup(event) + } + saveRatingLookup(event) + } else { + context.output(config.failedEvent, event) + } + } + catch { + case ex: Exception => { + ex.printStackTrace() + context.output(config.failedEvent, event) + logger.info("Event throwing exception: ", ex.getMessage) + } + } + } + + def updateDB(event: Event, updatedRatingValues: HashMap[Float, Float], + sumOfTotalRating: Float, totalRating: Float, + summary: String): Unit = { + val ratingDBResult = getRatingSummary(event) + val validReview = event.updatedValues.get("review").asInstanceOf[String] + + var updatedReviews = "" + if (null == ratingDBResult) { + updatedReviews = update_Top50_Review_Summary(null, event) + saveRatingSummary(event, updatedRatingValues, updatedReviews, sumOfTotalRating, totalRating) + } + else { + updatedReviews = update_Top50_Review_Summary(summary, event) + updateRatingSummary(event, updatedRatingValues, updatedReviews, sumOfTotalRating, totalRating) + } + } + + def update_ratings_count(tempRow: Row, prevRating: Float, updatedRating: Float): HashMap[Float, Float] = { + var ratingMap: HashMap[Float, Float] = new HashMap() + var x = 0; + val defaultCount = 0.0f + for (i <- 0 to 4) { + ratingMap.put(i + 1.0f, 0.0f) + } + if (tempRow != null) { + ratingMap.put(1.0f, (if (tempRow.getFloat("totalcount1stars")!=null) tempRow.getFloat("totalcount1stars") else defaultCount)) + ratingMap.put(2.0f, (if (tempRow.getFloat("totalcount2stars")!=null) tempRow.getFloat("totalcount2stars") else defaultCount)) + ratingMap.put(3.0f, (if (tempRow.getFloat("totalcount3stars")!=null) tempRow.getFloat("totalcount3stars") else defaultCount)) + ratingMap.put(4.0f, (if (tempRow.getFloat("totalcount4stars")!=null) tempRow.getFloat("totalcount4stars") else defaultCount)) + ratingMap.put(5.0f, (if (tempRow.getFloat("totalcount5stars")!=null) tempRow.getFloat("totalcount5stars") else defaultCount)) + + } + val newRating = (updatedRating).floor + val oldRating = (prevRating).floor + if (ratingMap.containsKey(newRating) && newRating != oldRating) { + ratingMap.put(newRating, ratingMap.get(newRating) + 1) + } + if (oldRating!=0.0f && ratingMap.get(oldRating) != 0.0f && newRating != oldRating) { + if (ratingMap.containsKey(oldRating) && newRating != oldRating) { + ratingMap.put(oldRating, ratingMap.get(oldRating) - 1) + } + } + ratingMap + + } + + def update_Top50_Review_Summary(summary: String, event: Event): String = { + var gson: Gson = new Gson() + var ratingQueue = mutable.Queue[RatingJson]() + var updatedReviewSize: Int = 0 + if (event.updatedValues.get("review") != null) + updatedReviewSize = event.updatedValues.get("review").asInstanceOf[String].size + + if (summary!=null) { + var ratingJson: Array[RatingJson] = gson.fromJson(summary, classOf[Array[RatingJson]]) + ratingJson.foreach( + row => { + if(!row.user_id.asInstanceOf[String].equals(event.userId.asInstanceOf[String])){ + ratingQueue.enqueue(row) + } + }); + if (ratingQueue.size >= 50) { + ratingQueue.dequeue() + } + } + + if (updatedReviewSize != 0 && updatedReviewSize > 100) { + + ratingQueue.enqueue(RatingJson("review", + event.userId.asInstanceOf[String], + event.updatedValues.get("updatedOn").asInstanceOf[String], + event.updatedValues.get("rating").asInstanceOf[Double].toFloat, + event.updatedValues.get("review").asInstanceOf[String])) + val finalResult = ratingQueue.toList + gson.toJson(finalResult.toArray) + } else { + if(!ratingQueue.isEmpty){ + gson.toJson(ratingQueue.toList.toArray) + }else { + null + } + } + } + + def getRatingSummary(event: Event): Row = { + val query = QueryBuilder.select.all() + .from(config.dbKeyspace, config.ratingsSummaryTable). + where(QueryBuilder.eq("activityid", event.activityId)) + .and(QueryBuilder.eq("activitytype", event.activityType)).toString + + val row = cassandraUtil.findOne(query) + logger.info("Successfully retrieved the rating for summary - activityId: " + + event.activityId + " ,activityType: " + event.activityType + " ,userId: " + + event.userId + "with " + "prevValues: " + event.prevValues + "and updateValues" + event.updatedValues) + row + } + + def getRatingLookUp(event: Event): Row = { + var prevRating = 0.0f + + if(event.prevValues!=null) { + prevRating = event.prevValues.get("rating").asInstanceOf[Double].toFloat + } + val query = QueryBuilder.select.all() + .from(config.dbKeyspace, config.ratingsLookupTable). + where(QueryBuilder.eq("activityid", event.activityId)) + .and(QueryBuilder.eq("activitytype", event.activityType)) + .and(QueryBuilder.eq("rating", prevRating)).toString + + val row = cassandraUtil.findOne(query) + logger.info("Successfully retrieved the rating for summary - activityId: " + + event.activityId + " ,activityType: " + event.activityType + " ,userId: " + + event.userId + "with " + "and updateValues" + event.updatedValues) + row + } + + def saveRatingSummary(event: Event, updatedRatingValues: HashMap[Float, Float], + summary: String, sumOfTotalRating: Float, totalRating: Float): Unit = { + val query = QueryBuilder.insertInto(config.dbKeyspace, config.ratingsSummaryTable) + .value("activityid", event.activityId) + .value("activitytype", event.activityType) + .value("latest50reviews", summary) + .value("sum_of_total_ratings", sumOfTotalRating) + .value("total_number_of_ratings", totalRating) + .value("totalcount1stars", updatedRatingValues.get(1.0f)) + .value("totalcount2stars", updatedRatingValues.get(2.0f)) + .value("totalcount3stars", updatedRatingValues.get(3.0f)) + .value("totalcount4stars", updatedRatingValues.get(4.0f)) + .value("totalcount5stars", updatedRatingValues.get(5.0f)).toString + + cassandraUtil.upsert(query) + logger.info("Successfully processed the rating event - activityId: " + + event.activityId + " ,activityType: " + event.activityType + " ,userId: " + + event.userId + "with "+ "and updateValues" + event.updatedValues) + } + + def deleteRatingLookup(event: Event): Unit = { + val updatingTime = event.prevValues.get("updatedOn").asInstanceOf[String] + val timeBasedUuid = UUID.fromString(updatingTime) + + val query = QueryBuilder.delete() + .from(config.dbKeyspace, config.ratingsLookupTable) + .where(QueryBuilder.eq("activityid", event.activityId)) + .and(QueryBuilder.eq("activitytype", event.activityType)) + .and(QueryBuilder.eq("rating", event.prevValues.get("rating").asInstanceOf[Double].toFloat)) + .and(QueryBuilder.eq("updatedon", timeBasedUuid)).toString + + cassandraUtil.upsert(query) + logger.info("Successfully saved the rating for lookup - activityId: " + + event.activityId + " ,activityType: " + event.activityType + " ,userId: " + + event.userId + "with " + "prevValues: " + event.prevValues + "and updateValues" + event.updatedValues) + } + + def saveRatingLookup(event: Event): Unit = { + val updatingTime = event.updatedValues.get("updatedOn").asInstanceOf[String] + val timeBasedUuid = UUID.fromString(updatingTime) + var review_value: String = null; + if(event.updatedValues.get("review") != null) + review_value = event.updatedValues.get("review").toString; + val query = QueryBuilder.insertInto(config.dbKeyspace, config.ratingsLookupTable) + .value("activityid", event.activityId) + .value("activitytype", event.activityType) + .value("rating", event.updatedValues.get("rating").asInstanceOf[Double].toFloat) + .value("updatedon", timeBasedUuid) + .value("review", review_value) + .value("userid", event.userId).toString + + cassandraUtil.upsert(query) + logger.info("Successfully saved the rating for lookup - activityId: " + + event.activityId + " ,activityType: " + event.activityType + " ,userId: " + + event.userId + "with " + "prevValues: " + event.prevValues + "and updateValues" + event.updatedValues) + } + + def createRatingLookup(event: Event): Unit = { + val updatingTime = event.updatedValues.get("updatedOn").asInstanceOf[String] + val timeBasedUuid = UUID.fromString(updatingTime) + val query = QueryBuilder.insertInto(config.dbKeyspace, config.ratingsLookupTable) + .value("activityid", event.activityId) + .value("activitytype", event.activityType) + .value("rating", event.updatedValues.get("rating").asInstanceOf[Float]) + .value("updatedon", timeBasedUuid) + .value("userid", event.userId).toString + + cassandraUtil.upsert(query) + logger.info("Successfully saved the rating for lookup - activityId: " + + event.activityId + " ,activityType: " + event.activityType + " ,userId: " + + event.userId + "with " + "prevValues: " + event.prevValues + "and updateValues" + event.updatedValues) + } + + def updateRatingSummary(event: Event, updatedRatingValues: HashMap[Float, Float], + summary: String, sumOfTotalRating: Float, totalRating: Float): Unit = { + val test = updatedRatingValues.get(1.0f) + + val updateQuery = QueryBuilder.update(config.dbKeyspace, config.ratingsSummaryTable) + .`with`(QueryBuilder.set("latest50reviews", summary)) + .and(QueryBuilder.set("sum_of_total_ratings", sumOfTotalRating)) + .and(QueryBuilder.set("total_number_of_ratings", totalRating)) + .and(QueryBuilder.set("totalcount1stars", updatedRatingValues.get(1.0f))) + .and(QueryBuilder.set("totalcount2stars", updatedRatingValues.get(2.0f))) + .and(QueryBuilder.set("totalcount3stars", updatedRatingValues.get(3.0f))) + .and(QueryBuilder.set("totalcount4stars", updatedRatingValues.get(4.0f))) + .and(QueryBuilder.set("totalcount5stars", updatedRatingValues.get(5.0f))) + .where(QueryBuilder.eq("activityid", event.activityId)) + .and(QueryBuilder.eq("activitytype", event.activityType)) + cassandraUtil.upsert(updateQuery.toString) + logger.info("Successfully updated ratings in rating summary - activity_id: " + + event.activityId + " ,activity_type: " + event.activityType) + } +} + diff --git a/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/task/RatingConfig.scala b/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/task/RatingConfig.scala new file mode 100644 index 0000000000..aa1aef2b78 --- /dev/null +++ b/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/task/RatingConfig.scala @@ -0,0 +1,47 @@ +package org.sunbird.dp.rating.task +import com.typesafe.config.Config +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.streaming.api.scala.OutputTag +import org.sunbird.dp.core.job.BaseJobConfig +import org.sunbird.dp.rating.domain.Event + +class RatingConfig (override val config: Config) extends BaseJobConfig(config, "RatingJob") { + private val serialVersionUID = 2905979434303791379L + + implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + + // Kafka Topics Configuration + val inputTopic: String = config.getString("kafka.input.topic") + + // rating specific + val ratingParallelism: Int = config.getInt("task.rating.parallelism") + val kafkaIssueTopic: String = config.getString("kafka.output.topic") + val issueEventSink = "rating-issue-event-sink" + val issueOutputTagName = "rating-issue-events" + val failedEvent: OutputTag[Event] = OutputTag[Event]("failed-rating-events") + + //Cassandra + val courseTable: String = config.getString("ext-cassandra.course_table") + val ratingsTable: String = config.getString("ext-cassandra.ratings_table") + val ratingsSummaryTable: String = config.getString("ext-cassandra.ratings_summary_table") + val ratingsLookupTable: String = config.getString("ext-cassandra.ratings_lookup_table") + val dbKeyspace: String = config.getString("ext-cassandra.keyspace") + val dbHost: String = config.getString("ext-cassandra.host") + val dbPort: Int = config.getInt("ext-cassandra.port") + val dbCoursesKeyspace: String = config.getString("ext-cassandra.courses_keyspace") + + // constants + val courseId = "courseid" + val userId = "userid" + val activityId = "activityid" + val activityType = "activitytype" + + // Consumers + val RatingConsumer = "rating-consumer" + + // Functions + val ratingFunction = "RatingFunction" + + +} diff --git a/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/task/RatingTask.scala b/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/task/RatingTask.scala new file mode 100644 index 0000000000..c6711e5b40 --- /dev/null +++ b/data-pipeline-flink/rating/src/main/scala/org/sunbird/dp/rating/task/RatingTask.scala @@ -0,0 +1,51 @@ +package org.sunbird.dp.rating.task + +import java.io.File +import com.typesafe.config.ConfigFactory +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.api.java.utils.ParameterTool +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment +import org.sunbird.dp.core.job.FlinkKafkaConnector +import org.sunbird.dp.core.util.FlinkUtil +import org.sunbird.dp.rating.domain.Event +import org.sunbird.dp.rating.function.RatingFunction + +class RatingTask(config: RatingConfig, kafkaConnector: FlinkKafkaConnector) { + + private val serialVersionUID = -7729362727131516112L + + def process(): Unit = { + + implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) + implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + + val source = kafkaConnector.kafkaEventSource[Event](config.inputTopic) + + val stream = + env.addSource(source, config.RatingConsumer).uid(config.RatingConsumer).rebalance() + .process(new RatingFunction(config)).setParallelism(config.ratingParallelism) + .name(config.ratingFunction).uid(config.ratingFunction) + stream.getSideOutput(config.failedEvent).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaIssueTopic)) + .name(config.issueEventSink).uid(config.issueEventSink) + .setParallelism(config.ratingParallelism) + env.execute(config.jobName) + } + +} + +// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster +object RatingTask { + + def main(args: Array[String]): Unit = { + val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) + val config = configFilePath.map { + path => ConfigFactory.parseFile(new File(path)).resolve() + }.getOrElse(ConfigFactory.load("rating-feature.conf").withFallback(ConfigFactory.systemEnvironment())) + val ratingConfig = new RatingConfig(config) + val kafkaUtil = new FlinkKafkaConnector(ratingConfig) + val task = new RatingTask(ratingConfig, kafkaUtil) + task.process() + } +} + diff --git a/data-pipeline-flink/sunbird-dp-distribution/Dockerfile b/data-pipeline-flink/sunbird-dp-distribution/Dockerfile index b00c143bbe..c220a7daa1 100644 --- a/data-pipeline-flink/sunbird-dp-distribution/Dockerfile +++ b/data-pipeline-flink/sunbird-dp-distribution/Dockerfile @@ -1,7 +1,9 @@ -FROM anandp504/flink:1.10.1-scala_2.12 +FROM sunbird/flink:1.13.5-scala_2.12-java11 COPY target/sunbird-dp-distribution-1.0.tar.gz /tmp USER flink RUN tar -xvf /tmp/sunbird-dp-distribution-1.0.tar.gz -C $FLINK_HOME/lib/ +RUN mkdir $FLINK_HOME/plugins/s3-fs-presto +RUN cp $FLINK_HOME/opt/flink-s3-fs-presto-1.13.5.jar $FLINK_HOME/plugins/s3-fs-presto/ USER root RUN rm -f /tmp/sunbird-dp-distribution-1.0.tar.gz USER flink diff --git a/data-pipeline-flink/sunbird-dp-distribution/pom.xml b/data-pipeline-flink/sunbird-dp-distribution/pom.xml index 2cc9a4c876..1aa7bd0956 100644 --- a/data-pipeline-flink/sunbird-dp-distribution/pom.xml +++ b/data-pipeline-flink/sunbird-dp-distribution/pom.xml @@ -16,10 +16,16 @@ org.apache.flink - flink-streaming-scala_${scala.version} + flink-streaming-scala_${scala.maj.version} ${flink.version} jar + + org.sunbird.dp.jobs + ingest-router + 1.0.0 + jar + org.sunbird.dp.jobs telemetry-extractor @@ -52,8 +58,8 @@ org.sunbird.dp.jobs - user-cache-updater - 1.0.0 + user-cache-updater-2.0 + 2.0.0 jar @@ -68,6 +74,18 @@ 1.0.0 jar + + org.sunbird.dp.jobs + cb-preprocessor + 1.0.0 + jar + + + org.sunbird.dp.jobs + rating + 2.0.0 + jar + diff --git a/data-pipeline-flink/telemetry-extractor/pom.xml b/data-pipeline-flink/telemetry-extractor/pom.xml index 6776dfbfed..5c9be58489 100644 --- a/data-pipeline-flink/telemetry-extractor/pom.xml +++ b/data-pipeline-flink/telemetry-extractor/pom.xml @@ -27,7 +27,7 @@ org.apache.flink - flink-streaming-scala_${scala.version} + flink-streaming-scala_${scala.maj.version} ${flink.version} provided @@ -46,13 +46,13 @@ org.apache.flink flink-test-utils_2.12 - 1.10.0 + ${flink.version} test org.apache.flink flink-runtime_2.12 - 1.10.0 + ${flink.version} test tests @@ -65,7 +65,7 @@ org.apache.flink flink-streaming-java_2.12 - 1.10.0 + ${flink.version} test tests @@ -93,6 +93,14 @@ src/main/scala src/test/scala + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + org.apache.maven.plugins maven-shade-plugin @@ -141,10 +149,11 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 4.4.0 - 1.8 - 1.8 + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} false diff --git a/data-pipeline-flink/telemetry-extractor/src/main/resources/log4j.properties b/data-pipeline-flink/telemetry-extractor/src/main/resources/log4j.properties deleted file mode 100644 index dda4b21056..0000000000 --- a/data-pipeline-flink/telemetry-extractor/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=telemetry-extractor.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/data-pipeline-flink/telemetry-extractor/src/main/resources/telemetry-extractor.conf b/data-pipeline-flink/telemetry-extractor/src/main/resources/telemetry-extractor.conf index f4801f93b7..9518da6df3 100644 --- a/data-pipeline-flink/telemetry-extractor/src/main/resources/telemetry-extractor.conf +++ b/data-pipeline-flink/telemetry-extractor/src/main/resources/telemetry-extractor.conf @@ -3,6 +3,7 @@ include "base-config.conf" kafka { input.topic = ${job.env}".telemetry.ingest" output.success.topic = ${job.env}".telemetry.raw" + output.log.route.topic = ${job.env}".druid.events.log" output.duplicate.topic = ${job.env}".telemetry.extractor.duplicate" output.failed.topic = ${job.env}".telemetry.failed" output.batch.failed.topic = ${job.env}".telemetry.extractor.failed" @@ -16,9 +17,7 @@ kafka { task { consumer.parallelism = 1 - dedup.parallelism = 1 - extraction.parallelism = 1 - redactor.parallelism = 1 + downstream.operators.parallelism = 1 } redis { diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/domain/Models.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/domain/Models.scala index 471f04a9c3..fa4dbda67e 100644 --- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/domain/Models.scala +++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/domain/Models.scala @@ -12,14 +12,5 @@ case class Pdata(ver: String, pid: String, id: String = "data-pipeline") case class Object(id:String, ver: String, `type`:String, rollup: Option[Map[String, String]]) -case class LogEvent(actor: Actor, - eid: String, - edata: EData, - ver: String = "3.0", - syncts: Long, - ets: Long = System.currentTimeMillis(), - context: Context, - mid: String, - `object`: Object, - tags: Seq[AnyRef] - ) \ No newline at end of file +case class LogEvent(actor: Actor, eid: String, edata: EData, ver: String = "3.0", syncts: Long, ets: Long = System.currentTimeMillis(), + context: Context, mid: String, `object`: Object, tags: Seq[AnyRef]) \ No newline at end of file diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/DeduplicationFunction.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/DeduplicationFunction.scala index 8e305286d6..d9d5d242d9 100644 --- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/DeduplicationFunction.scala +++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/DeduplicationFunction.scala @@ -2,7 +2,8 @@ package org.sunbird.dp.extractor.functions import java.util -import com.google.gson.Gson +import org.sunbird.dp.core.util.JSONUtil +// import com.google.gson.Gson import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.ProcessFunction @@ -53,13 +54,15 @@ class DeduplicationFunction(config: TelemetryExtractorConfig, @transient var ded throw jedisEx } case ex: Exception => { + logger.error("Unexpected Error", ex) metrics.incCounter(config.failedBatchCount) context.output(config.failedBatchEventOutputTag, batchEvents) } } def getMsgIdentifier(batchEvents: String): String = { - val event = new Gson().fromJson(batchEvents, new util.LinkedHashMap[String, AnyRef]().getClass) + // val event = new Gson().fromJson(batchEvents, new util.LinkedHashMap[String, AnyRef]().getClass) + val event = JSONUtil.deserialize[util.LinkedHashMap[String, AnyRef]](batchEvents) val paramsObj = Option(event.get("params")) val messageId = paramsObj.map { params => params.asInstanceOf[util.Map[String, AnyRef]].get("msgid").asInstanceOf[String] diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/ExtractionFunction.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/ExtractionFunction.scala index e9091b1722..97b7ddd8a1 100644 --- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/ExtractionFunction.scala +++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/ExtractionFunction.scala @@ -11,6 +11,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.streaming.api.functions.ProcessFunction import org.joda.time.format.DateTimeFormat import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics} +import org.sunbird.dp.core.util.JSONUtil import org.sunbird.dp.extractor.domain._ import org.sunbird.dp.extractor.domain.{Context => EventContext} import org.sunbird.dp.extractor.task.TelemetryExtractorConfig @@ -40,25 +41,28 @@ class ExtractionFunction(config: TelemetryExtractorConfig)(implicit val stringTy eventsList.forEach(event => { val eventId = event.get("eid").asInstanceOf[String] val eventData = updateEvent(event, syncTs) - val eventJson = gson.toJson(eventData) + val eventJson = JSONUtil.serialize(eventData) val eventSize = eventJson.getBytes("UTF-8").length if (eventSize > config.eventMaxSize) { metrics.incCounter(config.failedEventCount) context.output(config.failedEventsOutputTag, markFailed(eventData)) } else { metrics.incCounter(config.successEventCount) - if (config.redactEventsList.contains(eventId)) + if (config.redactEventsList.contains(eventId)) { context.output(config.assessRedactEventsOutputTag, markSuccess(eventData)) - else + } else if ("LOG".equalsIgnoreCase(eventId)) { + context.output(config.logEventsOutputTag, markSuccess(eventData)) + } else { context.output(config.rawEventsOutputTag, markSuccess(eventData)) + } } }) /** * Generating Audit events to compute the number of events in the batch. */ - context.output(config.logEventsOutputTag, - gson.fromJson(gson.toJson(generateAuditEvents(eventsList.size(), batchEvent)), mapType)) + // context.output(config.logEventsOutputTag, gson.fromJson(gson.toJson(generateAuditEvents(eventsList.size(), batchEvent)), mapType)) + context.output(config.auditEventsOutputTag, gson.toJson(generateAuditEvents(eventsList.size(), batchEvent))) metrics.incCounter(config.auditEventCount) } diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorConfig.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorConfig.scala index a3f6003982..6ff583997c 100644 --- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorConfig.scala +++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorConfig.scala @@ -22,6 +22,7 @@ class TelemetryExtractorConfig(override val config: Config) extends BaseJobConfi // Kafka Topics Configuration val kafkaInputTopic: String = config.getString("kafka.input.topic") val kafkaSuccessTopic: String = config.getString("kafka.output.success.topic") + val kafkaLogRouteTopic: String = config.getString("kafka.output.log.route.topic") val kafkaDuplicateTopic: String = config.getString("kafka.output.duplicate.topic") val kafkaFailedTopic: String = config.getString("kafka.output.failed.topic") val kafkaBatchFailedTopic: String = config.getString("kafka.output.batch.failed.topic") @@ -29,21 +30,21 @@ class TelemetryExtractorConfig(override val config: Config) extends BaseJobConfi val eventMaxSize: Long = config.getLong("kafka.event.max.size") override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - val deDupParallelism: Int = config.getInt("task.dedup.parallelism") - val extractionParallelism: Int = config.getInt("task.extraction.parallelism") - val redactorParallelism: Int = config.getInt("task.redactor.parallelism") + val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.operators.parallelism") val redactEventsList: List[String] = config.getStringList("redact.events.list").asScala.toList val contentStore: Int = config.getInt("redis-meta.database.contentstore.id") - val UNIQUE_EVENTS_OUTPUT_TAG = "unique-events" + val UNIQUE_EVENTS_OUTPUT_TAG = "unique-batch-events" val RAW_EVENTS_OUTPUT_TAG = "raw-events" + val LOG_EVENTS_OUTPUT_TAG = "log-events" + val ERROR_EVENTS_OUTPUT_TAG = "error-events" val ASSESS_REDACT_EVENTS_OUTPUT_TAG = "assess-redact-events" val ASSESS_RAW_EVENTS_OUTPUT_TAG = "assess-raw-events" val FAILED_EVENTS_OUTPUT_TAG = "failed-events" val FAILED_BATCH_EVENTS_OUTPUT_TAG = "failed-batch-events" - val LOG_EVENTS_OUTPUT_TAG = "log-events" - val DUPLICATE_EVENTS_OUTPUT_TAG = "duplicate-events" + val AUDIT_EVENTS_OUTPUT_TAG = "audit-events" + val DUPLICATE_EVENTS_OUTPUT_TAG = "duplicate-batch-events" // Metric List val successEventCount = "success-event-count" @@ -56,11 +57,13 @@ class TelemetryExtractorConfig(override val config: Config) extends BaseJobConfi val skippedEventCount = "skipped-event-count" val rawEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](RAW_EVENTS_OUTPUT_TAG) + val logEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](LOG_EVENTS_OUTPUT_TAG) + val errorEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](ERROR_EVENTS_OUTPUT_TAG) val assessRawEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](ASSESS_RAW_EVENTS_OUTPUT_TAG) val assessRedactEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](ASSESS_REDACT_EVENTS_OUTPUT_TAG) val failedEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](FAILED_EVENTS_OUTPUT_TAG) val failedBatchEventOutputTag: OutputTag[String] = OutputTag[String](FAILED_BATCH_EVENTS_OUTPUT_TAG) - val logEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](LOG_EVENTS_OUTPUT_TAG) + val auditEventsOutputTag: OutputTag[String] = OutputTag[String](AUDIT_EVENTS_OUTPUT_TAG) val duplicateEventOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](id = DUPLICATE_EVENTS_OUTPUT_TAG) val uniqueEventOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](id = UNIQUE_EVENTS_OUTPUT_TAG) @@ -76,6 +79,8 @@ class TelemetryExtractorConfig(override val config: Config) extends BaseJobConfi val extractorBatchFailedEventsProducer = "extractor-batch-failed-events-sink" val extractorRawEventsProducer = "extractor-raw-events-sink" val extractorAuditEventsProducer = "extractor-audit-events-sink" + val extractorLogEventsProducer = "extractor-log-events-sink" + val extractorErrorEventsProducer = "extractor-error-events-sink" val extractorFailedEventsProducer = "extractor-failed-events-sink" val assessEventsProducer = "assess-events-sink" val assessRawEventsProducer = "assess-raw-events-sink" diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorStreamTask.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorStreamTask.scala index c4a9ebed62..e0e226f843 100644 --- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorStreamTask.scala +++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorStreamTask.scala @@ -61,7 +61,7 @@ class TelemetryExtractorStreamTask(config: TelemetryExtractorConfig, kafkaConnec .rebalance() .process(new DeduplicationFunction(config)) .name("ExtractorDeduplicationFn").uid("ExtractorDeduplicationFn") - .setParallelism(config.deDupParallelism) + .setParallelism(config.downstreamOperatorsParallelism) /** * After - De-Duplication process. * 1. Extract the batch events. @@ -71,22 +71,36 @@ class TelemetryExtractorStreamTask(config: TelemetryExtractorConfig, kafkaConnec deDupStream.getSideOutput(config.uniqueEventOutputTag) .process(new ExtractionFunction(config)) .name(config.extractionFunction).uid(config.extractionFunction) - .setParallelism(config.extractionParallelism) + .setParallelism(config.downstreamOperatorsParallelism) val redactorStream = extractionStream.getSideOutput(config.assessRedactEventsOutputTag) .process(new RedactorFunction(config)).name(config.redactorFunction).uid(config.redactorFunction) - .setParallelism(config.redactorParallelism) + .setParallelism(config.downstreamOperatorsParallelism) - deDupStream.getSideOutput(config.duplicateEventOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaDuplicateTopic)).name(config.extractorDuplicateProducer).uid(config.extractorDuplicateProducer) - deDupStream.getSideOutput(config.failedBatchEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaBatchFailedTopic)).name(config.extractorBatchFailedEventsProducer).uid(config.extractorBatchFailedEventsProducer) + deDupStream.getSideOutput(config.duplicateEventOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaDuplicateTopic)) + .name(config.extractorDuplicateProducer).uid(config.extractorDuplicateProducer).setParallelism(config.downstreamOperatorsParallelism) - extractionStream.getSideOutput(config.rawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic)).name(config.extractorRawEventsProducer).uid(config.extractorRawEventsProducer) - extractionStream.getSideOutput(config.logEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic)).name(config.extractorAuditEventsProducer).uid(config.extractorAuditEventsProducer) - extractionStream.getSideOutput(config.failedEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaFailedTopic)).name(config.extractorFailedEventsProducer).uid(config.extractorFailedEventsProducer) + deDupStream.getSideOutput(config.failedBatchEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaBatchFailedTopic)) + .name(config.extractorBatchFailedEventsProducer).uid(config.extractorBatchFailedEventsProducer).setParallelism(config.downstreamOperatorsParallelism) - redactorStream.getSideOutput(config.rawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic)).name(config.assessEventsProducer).uid(config.assessEventsProducer) - redactorStream.getSideOutput(config.assessRawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaAssessRawTopic)).name(config.assessRawEventsProducer).uid(config.assessRawEventsProducer) + extractionStream.getSideOutput(config.rawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic)) + .name(config.extractorRawEventsProducer).uid(config.extractorRawEventsProducer).setParallelism(config.downstreamOperatorsParallelism) + + extractionStream.getSideOutput(config.logEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaLogRouteTopic)) + .name(config.extractorLogEventsProducer).uid(config.extractorLogEventsProducer).setParallelism(config.downstreamOperatorsParallelism) + + extractionStream.getSideOutput(config.auditEventsOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaLogRouteTopic)) + .name(config.extractorAuditEventsProducer).uid(config.extractorAuditEventsProducer).setParallelism(config.downstreamOperatorsParallelism) + + extractionStream.getSideOutput(config.failedEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaFailedTopic)) + .name(config.extractorFailedEventsProducer).uid(config.extractorFailedEventsProducer).setParallelism(config.downstreamOperatorsParallelism) + + redactorStream.getSideOutput(config.rawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic)) + .name(config.assessEventsProducer).uid(config.assessEventsProducer).setParallelism(config.downstreamOperatorsParallelism) + + redactorStream.getSideOutput(config.assessRawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaAssessRawTopic)) + .name(config.assessRawEventsProducer).uid(config.assessRawEventsProducer).setParallelism(config.downstreamOperatorsParallelism) env.execute(config.jobName) } diff --git a/data-pipeline-flink/telemetry-extractor/src/test/resources/test.conf b/data-pipeline-flink/telemetry-extractor/src/test/resources/test.conf index 1c37949cb0..cb85dc1f76 100644 --- a/data-pipeline-flink/telemetry-extractor/src/test/resources/test.conf +++ b/data-pipeline-flink/telemetry-extractor/src/test/resources/test.conf @@ -3,6 +3,7 @@ include "base-test.conf" kafka { input.topic = "flink.telemetry.ingest" output.success.topic = "flink.telemetry.raw" + output.log.route.topic = "flink.druid.events.log" output.duplicate.topic = "flink.telemetry.extractor.duplicate" output.failed.topic = "flink.telemetry.failed" output.batch.failed.topic = "flink.telemetry.extractor.failed" @@ -13,9 +14,7 @@ kafka { task { consumer.parallelism = 1 - dedup.parallelism = 1 - extraction.parallelism = 1 - redactor.parallelism = 1 + downstream.operators.parallelism = 1 } redis { diff --git a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/Event.scala b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/Event.scala new file mode 100644 index 0000000000..64caa9cfe1 --- /dev/null +++ b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/Event.scala @@ -0,0 +1,9 @@ +package org.sunbird.dp.fixture + +import java.util + +import org.sunbird.dp.core.domain.Events + +class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { + +} diff --git a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala index 774c80756e..894859bf67 100644 --- a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala +++ b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala @@ -4,12 +4,12 @@ object EventFixture { val EVENT_WITH_MESSAGE_ID: String = """ - |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} + |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"ERROR","ets":1604733538282,"ver":"3.0","mid":"ERROR:36855629","actor":{"id":"479ea1e9","type":"User"},"context":{"channel":"1234","pdata":{"id":"local.diksha.app","ver":"3.2.485","pid":"local.app.contentplayer"},"env":"contentplayer","sid":"f55869e7-1f87","did":"a3aa6eed374c1a","cdata":[{"id":"streaming","type":"PlayerLaunch"},{"id":"a29d203eb","type":"ContentSession"},{"id":"bbc96db","type":"PlaySession"},{"id":"f55869e7-1","type":"UserSession"}],"rollup":{"l1":"0126684405014522"}},"object":{"id":"do_313134032380190720","type":"Content","ver":"1","rollup":{"l1":"do_3131397259090575361"}},"tags":[],"edata":{"err":"Missing PDF","errtype":"CONTENT","stacktrace":"Error\n at MissingPDFExceptionClosure","pageid":""}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} |""".stripMargin val EVENT_WITHOUT_MESSAGE_ID: String = """ - |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{"id":"do_21299582901864857613016","ver":"1.0","type":"AssessmentItem"},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_21299582901864857613016","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_312526125187809280139355","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"test-item-id","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} + |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{"id":"do_21299582901864857613016","ver":"1.0","type":"AssessmentItem"},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test1","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_21299582901864857613016","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test2","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_312526125187809280139355","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test3","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"test-item-id","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} |""".stripMargin val EMPTY_BATCH_EVENTS: String = diff --git a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/spec/TelemetryExtractionStreamTaskTestSpec.scala b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/spec/TelemetryExtractionStreamTaskTestSpec.scala index 2a67f7897d..a0a8afe6e9 100644 --- a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/spec/TelemetryExtractionStreamTaskTestSpec.scala +++ b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/spec/TelemetryExtractionStreamTaskTestSpec.scala @@ -8,7 +8,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.TypeExtractor import org.mockito.Mockito import org.mockito.Mockito._ -import org.sunbird.dp.fixture.EventFixture +import org.sunbird.dp.fixture.{Event, EventFixture} import redis.embedded.RedisServer import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration @@ -17,6 +17,7 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext import com.typesafe.config.ConfigFactory import org.sunbird.dp.core.cache.RedisConnect +import org.sunbird.dp.core.domain.Events import org.sunbird.dp.core.job.FlinkKafkaConnector import org.sunbird.dp.extractor.task.{TelemetryExtractorConfig, TelemetryExtractorStreamTask} import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} @@ -48,11 +49,13 @@ class ExtractionStreamTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSource(extractorConfig.kafkaInputTopic)).thenReturn(new ExtractorEventSource) when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaDuplicateTopic)).thenReturn(new DupEventsSink) when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaSuccessTopic)).thenReturn(new RawEventsSink) + when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaLogRouteTopic)).thenReturn(new LogEventsSink) + when(mockKafkaUtil.kafkaStringSink(extractorConfig.kafkaLogRouteTopic)).thenReturn(new AuditEventsSink) when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink) when(mockKafkaUtil.kafkaStringSink(extractorConfig.kafkaBatchFailedTopic)).thenReturn(new FailedBatchEventsSink) when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaAssessRawTopic)).thenReturn(new AssessRawEventsSink) - setupRedisTestData + setupRedisTestData() flinkCluster.before() } @@ -78,31 +81,50 @@ class ExtractionStreamTaskTestSpec extends BaseTestSpec { val task = new TelemetryExtractorStreamTask(extractorConfig, mockKafkaUtil) task.process() - RawEventsSink.values.size() should be (45) // 43 events + 2 log events generated for auditing + // RawEventsSink.values.size() should be (43) // 43 events + 2 log events generated for auditing + RawEventsSink.values.size() should be (6) // includes 1 Error Event + AuditEventsSink.values.size() should be (2) + LogEventsSink.values.size() should be (38) FailedEventsSink.values.size() should be (2) FailedBatchEventsSink.values.size() should be (1) DupEventsSink.values.size() should be (1) AssessRawEventsSink.values.size() should be (2) - val rawEvent = gson.fromJson(gson.toJson(RawEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala + // val rawEvent = gson.fromJson(gson.toJson(RawEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala + val rawEvent = RawEventsSink.values.get(0) val dupEvent = gson.fromJson(gson.toJson(DupEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala val failedEvent = gson.fromJson(gson.toJson(FailedEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala - rawEvent("flags").asInstanceOf[util.Map[String, Boolean]].get("ex_processed") should be(true) + // rawEvent("flags").asInstanceOf[util.Map[String, Boolean]].get("ex_processed") should be(true) + rawEvent.flags().asInstanceOf[util.Map[String, Boolean]].get("ex_processed") should be(true) dupEvent("flags").asInstanceOf[util.Map[String, Boolean]].get("extractor_duplicate") should be(true) failedEvent("flags").asInstanceOf[util.Map[String, Boolean]].get("ex_processed") should be(false) // Assertions for redactor logic + val responseEventWithoutValues = RawEventsSink.values.asScala.toList.filter(ev => ev.mid().equals("RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc")).head + responseEventWithoutValues.getTelemetry.read[util.Map[String, AnyRef]]("edata").get.get("values").asInstanceOf[util.ArrayList[AnyRef]].size should be (0) + + val assessEventWithoutResValues = RawEventsSink.values.asScala.filter(ev => ev.mid().equals("ASSESS:12159f2827880221eef12a6be9560379:test1")).head + assessEventWithoutResValues.getTelemetry.read[util.Map[String, AnyRef]]("edata").get.get("resvalues").asInstanceOf[util.ArrayList[AnyRef]].size should be (0) + + + val assessEventWithResValues = RawEventsSink.values.asScala.filter(ev => ev.mid().equals("ASSESS:12159f2827880221eef12a6be9560379:test2")).head + assessEventWithResValues.getTelemetry.read[util.Map[String, AnyRef]]("edata").get.get("resvalues").asInstanceOf[util.ArrayList[AnyRef]].size should be (1) + + /* val responseEventWithoutValues = gson.fromJson(gson.toJson(RawEventsSink.values.get(39)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala responseEventWithoutValues("edata").asInstanceOf[util.Map[String, AnyRef]].get("values").asInstanceOf[util.ArrayList[AnyRef]].size should be (0) + val assessEventWithoutResValues = gson.fromJson(gson.toJson(RawEventsSink.values.get(40)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala assessEventWithoutResValues("edata").asInstanceOf[util.Map[String, AnyRef]].get("resvalues").asInstanceOf[util.ArrayList[AnyRef]].size should be (0) + val assessEventWithResValues = gson.fromJson(gson.toJson(RawEventsSink.values.get(42)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala assessEventWithResValues("edata").asInstanceOf[util.Map[String, AnyRef]].get("resvalues").asInstanceOf[util.ArrayList[AnyRef]].size should be (1) + */ BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.successBatchCount}").getValue() should be (3) BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.failedBatchCount}").getValue() should be (1) BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.failedEventCount}").getValue() should be (2) - BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.successEventCount}").getValue() should be (43) + BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.successEventCount}").getValue() should be (44) BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.unique-event-count").getValue() should be (2) BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.duplicate-event-count").getValue() should be (1) BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.auditEventCount}").getValue() should be (2) @@ -134,13 +156,14 @@ class RawEventsSink extends SinkFunction[util.Map[String, AnyRef]] { override def invoke(value: util.Map[String, AnyRef]): Unit = { synchronized { - RawEventsSink.values.add(value) + val res = new Event(value.asInstanceOf[util.Map[String, Any]]) + RawEventsSink.values.add(new Event(value.asInstanceOf[util.Map[String, Any]])) } } } object RawEventsSink { - val values: util.List[util.Map[String, AnyRef]] = new util.ArrayList() + val values: util.List[Event] = new util.ArrayList[Event]() } class FailedEventsSink extends SinkFunction[util.Map[String, AnyRef]] { @@ -172,13 +195,26 @@ class LogEventsSink extends SinkFunction[util.Map[String, AnyRef]] { override def invoke(value: util.Map[String, AnyRef]): Unit = { synchronized { - LogEventsSink.values.add(value) + LogEventsSink.values.add(new Event(value.asInstanceOf[util.Map[String, Any]])) } } } object LogEventsSink { - val values: util.List[util.Map[String, AnyRef]] = new util.ArrayList() + val values: util.List[Event] = new util.ArrayList[Event]() +} + + +class AuditEventsSink extends SinkFunction[String] { + override def invoke(value: String): Unit = { + synchronized { + AuditEventsSink.values.add(value) + } + } +} + +object AuditEventsSink { + val values: util.List[String] = new util.ArrayList() } class DupEventsSink extends SinkFunction[util.Map[String, AnyRef]] { diff --git a/data-pipeline-flink/user-cache-updater/README.md b/data-pipeline-flink/user-cache-updater-2.0/README.md similarity index 100% rename from data-pipeline-flink/user-cache-updater/README.md rename to data-pipeline-flink/user-cache-updater-2.0/README.md diff --git a/data-pipeline-flink/user-cache-updater-2.0/pom.xml b/data-pipeline-flink/user-cache-updater-2.0/pom.xml new file mode 100644 index 0000000000..d976893de8 --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/pom.xml @@ -0,0 +1,240 @@ + + + + 4.0.0 + + 3.0.1 + + + + org.sunbird.dp + pipeline-jobs + 1.0 + + + org.sunbird.dp.jobs + user-cache-updater-2.0 + 2.0.0 + jar + UserCacheUpdaterV2 + + Updating user details into readis cache + + + + UTF-8 + 1.4.0 + + + + + org.apache.flink + flink-streaming-scala_${scala.maj.version} + ${flink.version} + provided + + + org.sunbird.dp + dp-core + 1.0.0 + + + org.sunbird.dp + dp-core + 1.0.0 + test-jar + test + + + org.apache.flink + flink-test-utils_2.12 + ${flink.version} + test + + + org.apache.flink + flink-runtime_2.12 + ${flink.version} + test + tests + + + it.ozimov + embedded-redis + 0.7.1 + test + + + com.google.guava + guava + + + + + org.cassandraunit + cassandra-unit + 3.11.2.0 + test + + + org.apache.flink + flink-streaming-java_2.12 + ${flink.version} + test + tests + + + org.scalatest + scalatest_2.12 + 3.0.6 + test + + + org.mockito + mockito-core + 3.3.3 + test + + + com.fiftyonred + mock-jedis + 0.4.0 + test + + + com.squareup.okhttp3 + mockwebserver + 4.4.0 + test + + + + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + + package + + shade + + + false + + + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.sunbird.dp.usercache.task.UserCacheUpdaterStreamTaskV2 + + + + reference.conf + + + + + + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + ${java.target.runtime} + ${java.target.runtime} + ${scala.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + maven-surefire-plugin + 2.22.2 + + true + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + usercache-testsuite.txt + + + + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + + \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/resources/user-cache-updater-v2.conf b/data-pipeline-flink/user-cache-updater-2.0/src/main/resources/user-cache-updater-v2.conf new file mode 100644 index 0000000000..37f5162f60 --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/resources/user-cache-updater-v2.conf @@ -0,0 +1,33 @@ +include "base-config.conf" + +kafka { + input.topic = ${job.env}".telemetry.audit" + groupId = ${job.env}"-user-cache-updater-group" +} + +task { + usercache.updater.parallelism = 1 +} + +# redis-metadata +redis-meta { + database { + userstore.id = 12 + key.expiry.seconds = 3600 + } +} + +user-read { + api { + url = "/learner/private/user/v1/read" + } +} + +regd.user.producer.pid = "learner-service" +user.self.signin.types = ["google","self"] +user.validated.types = ["sso"] +user.self.signin.key = "Self-Signed-In" +user.valid.key = "Validated" +user.read.url.fields = "locations,organisations" +user.read.api.error = ["CLIENT_ERROR"] +user.redis.registration-count.path = "mdo_registered_officer_count" \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala new file mode 100644 index 0000000000..2dbd4dd28b --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala @@ -0,0 +1,42 @@ +package org.sunbird.dp.usercache.domain + +import java.util + +import org.sunbird.dp.core.domain.Events +import org.sunbird.dp.usercache.util.UserReadResult + +class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { + + override def kafkaKey(): String = { + did() + } + + def getId: String = { + Option(objectType()).map({ t => if (t.equalsIgnoreCase("User")) objectID() else null + }).getOrElse(null) + } + + def getState: String = { + telemetry.read[String]("edata.state").getOrElse(null) + } + + def getContextDataId(cDataType: String): String = { + val cdata = telemetry.read[util.ArrayList[util.Map[String, AnyRef]]]("context.cdata").getOrElse(null) + var signInType: String = null + Option(cdata).map(data => { + data.forEach(cdataMap => { + if (cdataMap.get("type").asInstanceOf[String].equalsIgnoreCase(cDataType)) signInType = cdataMap.get("id").toString else signInType + }) + }).getOrElse(signInType) + signInType + } + + def userMetaData(): util.ArrayList[String] = { + telemetry.read[util.ArrayList[String]]("edata.props").getOrElse(new util.ArrayList[String]()) + } + + def isValid(userReadRes: UserReadResult) = { + if (userReadRes.responseCode.toUpperCase.equalsIgnoreCase("OK") && !userReadRes.result.isEmpty && userReadRes.result.containsKey("response")) true else false + } + +} diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunctionV2.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunctionV2.scala new file mode 100644 index 0000000000..9d0b3efc79 --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunctionV2.scala @@ -0,0 +1,81 @@ +package org.sunbird.dp.usercache.functions + +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.dp.contentupdater.core.util.RestUtil +import org.sunbird.dp.core.cache.{DataCache, RedisConnect} +import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics} +import org.sunbird.dp.core.util.JSONUtil +import org.sunbird.dp.usercache.domain.Event +import org.sunbird.dp.usercache.task.UserCacheUpdaterConfigV2 +import org.sunbird.dp.usercache.util.UserMetadataUpdater + +import scala.collection.JavaConverters.mapAsJavaMap +import scala.collection.mutable + +class UserCacheUpdaterFunctionV2(config: UserCacheUpdaterConfigV2)(implicit val mapTypeInfo: TypeInformation[Event]) + extends BaseProcessFunction[Event, Event](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[UserCacheUpdaterFunctionV2]) + private var dataCache: DataCache = _ + + private var restUtil: RestUtil = _ + + override def metricsList(): List[String] = { + List(config.userCacheHit, config.skipCount, config.successCount, config.totalEventsCount, config.apiReadMissCount, config.apiReadSuccessCount) + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.userStore, config.userFields) + dataCache.init() + restUtil = new RestUtil() + } + + override def close(): Unit = { + super.close() + dataCache.close() + } + + override def processElement(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics): Unit = { + metrics.incCounter(config.totalEventsCount) + val userId = event.getId + try { + Option(userId).map(id => { + Option(event.getState).map(name => { + val userData: mutable.Map[String, AnyRef] = name.toUpperCase match { + case "CREATE" | "CREATED" | "UPDATE" | "UPDATED" => { + UserMetadataUpdater.execute(id, event, metrics, config, dataCache, restUtil) + } + case _ => { + logger.info(s"Invalid event state name either it should be(Create/Created/Update/Updated) but found $name for ${event.mid()}") + metrics.incCounter(config.skipCount) + mutable.Map[String, AnyRef]() + } + } + if (!userData.isEmpty) { + if (config.regdUserProducerPid.equals(event.producerPid())) UserMetadataUpdater.removeEmptyFields(config.userStoreKeyPrefix + id, dataCache, userData) + dataCache.hmSet(config.userStoreKeyPrefix + id, mapAsJavaMap(UserMetadataUpdater.stringify(userData))) + logger.info(s"Data inserted into cache for user: ${userId} having mid: ${event.mid()}") + metrics.incCounter(config.successCount) + metrics.incCounter(config.userCacheHit) + } else { + logger.info(s"User Data to be updated is empty for user: ${userId}") + metrics.incCounter(config.skipCount) + } + }).getOrElse(metrics.incCounter(config.skipCount)) + }).getOrElse(metrics.incCounter(config.skipCount)) + } catch { + case ex: Exception => { + ex.printStackTrace() + logger.info(s"Processing event for user: ${userId} having mid: ${event.mid()}") + logger.info("Event throwing exception: SKIPPING!!!") + // logger.info("Event throwing exception: ", JSONUtil.serialize(event)) + // throw ex + } + } + } +} + diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfigV2.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfigV2.scala new file mode 100644 index 0000000000..1a0a5118f6 --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfigV2.scala @@ -0,0 +1,84 @@ +package org.sunbird.dp.usercache.task + +import com.typesafe.config.Config +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.sunbird.dp.core.job.BaseJobConfig +import org.sunbird.dp.usercache.domain.Event + +import java.util.{List => JList} + +class UserCacheUpdaterConfigV2(override val config: Config) extends BaseJobConfig(config, "UserCacheUpdaterJobV2") { + + private val serialVersionUID = 2905979434303791379L + + implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + + // Kafka Topics Configuration + val inputTopic: String = config.getString("kafka.input.topic") + val userFields: List[String] = if (config.hasPath("user.redis.removeable-fields")) + config.getStringList("user.redis.removeable-fields").asInstanceOf[List[String]] + else List[String]("state", "district", "block", "cluster", "schooludisecode", "schoolname") + + // User cache updater job metrics + val userCacheHit = "user-cache-hit" + val skipCount = "skipped-message-count" + val successCount = "success-message-count" + val dbReadSuccessCount = "db-read-success-count" + val dbReadMissCount = "db-read-miss-count" + val apiReadSuccessCount = "api-read-success-count" + val apiReadMissCount = "api-read-miss-count" + val totalEventsCount ="total-audit-events-count" + + val userSelfSignedInTypeList: JList[String] = config.getStringList("user.self.signin.types") + val userValidatedTypeList: JList[String] = config.getStringList("user.validated.types") + val userSelfSignedKey: String = config.getString("user.self.signin.key") + val userValidatedKey: String = config.getString("user.valid.key") + val regdUserProducerPid: String = config.getString("regd.user.producer.pid") + + // Redis + val userStore: Int = config.getInt("redis-meta.database.userstore.id") + + val userCacheParallelism: Int = config.getInt("task.usercache.updater.parallelism") + + // constants + val userSignInTypeKey = "usersignintype" + val userLoginTypeKey = "userlogintype" + val firstName = "firstname" + val lastName = "lastname" + val rootOrgId = "rootorgid" + val stateKey = "state" + val districtKey = "district" + val blockKey = "block" + val clusterKey = "cluster" + val orgnameKey = "orgname" + val schoolKey = "school" + val schoolUdiseCodeKey = "schooludisecode" + val schoolNameKey = "schoolname" + val `type` = "type" + val subtype = "subType" + val userTypeKey = "usertype" + val userSubtypeKey = "usersubtype" + val userId = "userid" + val language = "language" + val email = "email" + val phone = "phone" + val profileUserTypesKey = "profileusertypes" + + //user store key prefix + val userStoreKeyPrefix = "user:" + // Consumers + val userCacheConsumer = "user-cache-consumer" + + // Functions + val userCacheUpdaterFunction = "UserCacheUpdaterFunctionV2" + + //User Read API + val userReadApiUrl = config.getString("user-read.api.url") + val userReadApiFields = config.getString("user.read.url.fields") + val userReadApiErrors: JList[String] = config.getStringList("user.read.api.error") + + val userRegistrationCountPath = config.getString("user.redis.registration-count.path") + + val userAccBlockedErrCode = "UOS_USRRED0006" +} \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTaskV2.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTaskV2.scala new file mode 100644 index 0000000000..dc99cf4ebc --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTaskV2.scala @@ -0,0 +1,48 @@ +package org.sunbird.dp.usercache.task + +import java.io.File + +import com.typesafe.config.ConfigFactory +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.api.java.utils.ParameterTool +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment +import org.sunbird.dp.core.job.FlinkKafkaConnector +import org.sunbird.dp.core.util.FlinkUtil +import org.sunbird.dp.usercache.domain.Event +import org.sunbird.dp.usercache.functions.UserCacheUpdaterFunctionV2 + +class UserCacheUpdaterStreamTaskV2(config: UserCacheUpdaterConfigV2, kafkaConnector: FlinkKafkaConnector) { + + private val serialVersionUID = -7729362727131516112L + + def process(): Unit = { + + implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) + implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + + val source = kafkaConnector.kafkaEventSource[Event](config.inputTopic) + env.addSource(source, config.userCacheConsumer).uid(config.userCacheConsumer).rebalance() + .process(new UserCacheUpdaterFunctionV2(config)).setParallelism(config.userCacheParallelism) + .name(config.userCacheUpdaterFunction).uid(config.userCacheUpdaterFunction) + env.execute(config.jobName) + } + +} + +// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster +object UserCacheUpdaterStreamTaskV2 { + + def main(args: Array[String]): Unit = { + val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) + val config = configFilePath.map { + path => ConfigFactory.parseFile(new File(path)).resolve() + }.getOrElse(ConfigFactory.load("user-cache-updater-v2.conf").withFallback(ConfigFactory.systemEnvironment())) + val userCacheUpdaterConfig = new UserCacheUpdaterConfigV2(config) + val kafkaUtil = new FlinkKafkaConnector(userCacheUpdaterConfig) + val task = new UserCacheUpdaterStreamTaskV2(userCacheUpdaterConfig, kafkaUtil) + task.process() + } +} + +// $COVERAGE-ON$ diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/util/UserMetadataUpdater.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/util/UserMetadataUpdater.scala new file mode 100644 index 0000000000..d2c56ad476 --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/util/UserMetadataUpdater.scala @@ -0,0 +1,159 @@ +package org.sunbird.dp.usercache.util + +import com.google.gson.Gson +import org.slf4j.LoggerFactory +import org.sunbird.dp.contentupdater.core.util.RestUtil +import org.sunbird.dp.core.cache.DataCache +import org.sunbird.dp.core.job.Metrics +import org.sunbird.dp.usercache.domain.Event +import org.sunbird.dp.usercache.task.UserCacheUpdaterConfigV2 + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +case class UserReadResult(result: java.util.HashMap[String, Any], responseCode: String, params: Params) +case class Response(firstName: String, lastName: String, encEmail: String, encPhone: String, language: java.util.List[String], rootOrgId: String, profileUserType: java.util.HashMap[String, String], + userLocations: java.util.ArrayList[java.util.Map[String, AnyRef]], rootOrg: RootOrgInfo, userId: String, framework: java.util.LinkedHashMap[String, java.util.List[String]], profileUserTypes: java.util.List[java.util.HashMap[String, String]]) +case class RootOrgInfo(orgName: String) +case class Params(msgid: String, err: String, status: String, errmsg: String) + +object UserMetadataUpdater { + + private lazy val gson = new Gson() + + val logger = LoggerFactory.getLogger("UserMetadataUpdater") + + def execute(userId: String, event: Event, metrics: Metrics, config: UserCacheUpdaterConfigV2, dataCache: DataCache, restUtil: RestUtil): mutable.Map[String, AnyRef] = { + + val generalInfo = getGeneralInfo(userId, event, metrics, config, dataCache); + val regdInfo = if (config.regdUserProducerPid.equals(event.producerPid())) { + getRegisteredUserInfo(userId, event, metrics, config, dataCache, restUtil) + } else mutable.Map[String, String]() + generalInfo.++:(regdInfo); + } + + def getGeneralInfo(userId: String, event: Event, metrics: Metrics, config: UserCacheUpdaterConfigV2, dataCache: DataCache): mutable.Map[String, String] = { + val userCacheData: mutable.Map[String, String] = mutable.Map[String, String]() + Option(event.getContextDataId(cDataType = "SignupType")).map(signInType => { + if (config.userSelfSignedInTypeList.contains(signInType)) { + userCacheData.put(config.userSignInTypeKey, config.userSelfSignedKey) + } + if (config.userValidatedTypeList.contains(signInType)) { + userCacheData.put(config.userSignInTypeKey, config.userValidatedKey) + } + }).orNull + Option(event.getContextDataId(cDataType = "UserRole")).map(loginType => { + userCacheData.put(config.userLoginTypeKey, loginType) + }) + userCacheData; + } + + @throws(classOf[Exception]) + def getRegisteredUserInfo(userId: String, event: Event, metrics: Metrics, config: UserCacheUpdaterConfigV2, dataCache: DataCache, + restUtil: RestUtil): mutable.Map[String, AnyRef] = { + var userCacheData: mutable.Map[String, AnyRef] = mutable.Map[String, AnyRef]() + + //?fields=locations is appended in url to get userLocation in API response + val userReadRes = gson.fromJson[UserReadResult](restUtil.get(String.format("%s%s",config.userReadApiUrl, userId + "?fields=" + config.userReadApiFields)), classOf[UserReadResult]) + if(event.isValid(userReadRes)) { + // Inc API Read metrics + metrics.incCounter(config.apiReadSuccessCount) + + val response = gson.fromJson[Response](gson.toJson(userReadRes.result.get("response")), classOf[Response]) + val framework = response.framework + //flatten BGMS value + /** + * Assumption: Board and Framework-id is single valued + */ + if (!framework.isEmpty) { + val boardList = framework.getOrDefault("board", List().asJava) + val board = if (!boardList.isEmpty) boardList.get(0) else "" + val medium = framework.getOrDefault("medium", List().asJava) + val grade = framework.getOrDefault("gradeLevel", List().asJava) + val subject = framework.getOrDefault("subject", List().asJava) + val frameworkIdList = framework.getOrDefault("id", List().asJava) + val id = if (!frameworkIdList.isEmpty) frameworkIdList.get(0) else "" + userCacheData.+=("board" -> board, "medium" -> medium, "grade" -> grade, "subject" -> subject, "framework" -> id) + } + + //Location and School Information + val locationInfo = response.userLocations + if(null != locationInfo && !locationInfo.isEmpty) { + locationInfo.forEach(location => { + location.getOrDefault("type", "").asInstanceOf[String].toLowerCase match { + case config.schoolKey => userCacheData.put(config.schoolNameKey, location.getOrDefault("name", "").asInstanceOf[String]) + userCacheData.put(config.schoolUdiseCodeKey, location.getOrDefault("code", "").asInstanceOf[String]) + case _ => userCacheData.put(location.getOrDefault("type", "").asInstanceOf[String], location.getOrDefault("name", "").asInstanceOf[String]) + } + }) + } + + //Flatten User Type and subType + val profileUserTypes = response.profileUserTypes + if (null != profileUserTypes && !profileUserTypes.isEmpty) { + val List(userTypeString, userSubtypeString) = makeUsertypeStrings(profileUserTypes, config) + + userCacheData.+=(config.userTypeKey -> userTypeString, config.userSubtypeKey -> userSubtypeString) + userCacheData.+=(config.profileUserTypesKey -> new Gson().toJson(profileUserTypes)) + } + + //Personal information + userCacheData.+=(config.firstName -> response.firstName, config.lastName -> response.lastName, + config.language -> response.language, + config.orgnameKey -> response.rootOrg.orgName, + config.rootOrgId -> response.rootOrgId, + config.phone -> response.encPhone, + config.email -> response.encEmail, + config.userId -> response.userId) + + // update user registration count + event.getState.toUpperCase match { + case "CREATE" | "CREATED" => dataCache.hIncByWithRetry(config.userRegistrationCountPath, response.rootOrgId, 1) + case _ => + } + + } else if (config.userReadApiErrors.contains(userReadRes.responseCode.toUpperCase) && userReadRes.params.err.equalsIgnoreCase(config.userAccBlockedErrCode)) { //Skip the events for which response is 400 Bad request + logger.info(s"User Read API has response as ${userReadRes.responseCode.toUpperCase} for user: ${userId}") + metrics.incCounter(config.apiReadMissCount) + } else { + logger.info(s"User Read API does not have details for user: ${userId}") + metrics.incCounter(config.apiReadMissCount) + throw new Exception(s"User Read API does not have details for user: ${userId}") + } + userCacheData + } + + def makeUsertypeStrings(profileUserTypes: java.util.List[java.util.HashMap[String, String]], config: UserCacheUpdaterConfigV2): List[String] = { + val userTypeValue = mutable.ListBuffer[String]() + val userSubtypeValue = mutable.ListBuffer[String]() + profileUserTypes.forEach(userType => { + val typeVal:String = userType.get(config.`type`) + val subTypeVal:String = userType.get(config.subtype) + + if (typeVal != null && typeVal.nonEmpty && !userTypeValue.contains(typeVal)) userTypeValue.append(typeVal) + if (subTypeVal != null && subTypeVal.nonEmpty && !userSubtypeValue.contains(subTypeVal)) userSubtypeValue.append(subTypeVal) + }) + + List(userTypeValue.mkString(","), userSubtypeValue.mkString(",")) + } + + def removeEmptyFields(key: String, dataCache: DataCache, userMetaData: mutable.Map[String, AnyRef]):Unit = { + val redisRec = dataCache.hgetAllWithRetry(key) + val removableKeys = redisRec.keySet.diff(userMetaData.keySet) + if(removableKeys.nonEmpty) dataCache.hdelWithRetry(key, removableKeys.toSeq) + } + + def stringify(userData: mutable.Map[String, AnyRef]): mutable.Map[String, String] = { + userData.map { f => + (f._1, if (!f._2.isInstanceOf[String]) { + if (null != f._2) { + new Gson().toJson(f._2) + } else { + "" + } + } else { + f._2.asInstanceOf[String].replaceAll("\\[", "\\\\[").replaceAll("\\]", "\\\\]") + }) + } + } +} diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/test/resources/test.conf b/data-pipeline-flink/user-cache-updater-2.0/src/test/resources/test.conf new file mode 100644 index 0000000000..8352c40663 --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/src/test/resources/test.conf @@ -0,0 +1,32 @@ +include "base-test.conf" + +kafka { + input.topic = "flink.telemetry.audit" + groupId = "flink-user-cache-updater-group" +} + +task { + usercache.updater.parallelism = 1 +} + +redis-meta { + database { + userstore.id = 1 + key.expiry.seconds = 3600 + } +} + +user-read { + api { + url = "http://127.0.0.1:3000/learner/private/user/v1/read/" + } +} + +regd.user.producer.pid = "learner-service" +user.self.signin.types = ["google","self"] +user.validated.types = ["sso"] +user.self.signin.key = "Self-Signed-In" +user.valid.key = "Validated" +user.read.url.fields = "locations,organisations" +user.read.api.error = ["CLIENT_ERROR"] +user.redis.registration-count.path = "mdo_registered_officer_count" \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala new file mode 100644 index 0000000000..e25223e54b --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala @@ -0,0 +1,106 @@ +package org.sunbird.dp.fixture + +object EventFixture { + + val telemetrEvents: List[String] = List( + + /** + * User-Id : user-1 + * Edata.state is update + * Having all the user data + */ + """ + {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-1","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} + | + |""".stripMargin, + /** + * UserId: user-9 + * Client_Error + */ + """{"actor":{"id":"cb0defc7-9f5f-4c0a-bb73-207d34ed2cf0","type":"User"},"eid":"AUDIT","edata":{"state":"Update","type":"mergeUser","props":["fromAccountId","toAccountId","type"]},"ver":"3.0","syncts":1621593310099,"@timestamp":"2021-05-21T10:35:10.099Z","ets":1621593307552,"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.learning.service","pid":"learner-service","ver":"3.9.0"},"env":"User","cdata":[{"id":"9e6f3392-af99-425b-9404-e8e4078a1b65","type":"FromAccountId"},{"id":"cb0defc7-9f5f-4c0a-bb73-207d34ed2cf0","type":"ToAccountId"},{"id":"8f0d93fd-33ac-4a64-a39c-f008011c976a","type":"Request"}],"rollup":{"l1":"01275678925675724817"}},"flags":{"pp_duplicate_skipped":true,"pp_validation_processed":true},"mid":"8f0d93fd-33ac-4a64-a39c-f008011c976a","type":"events","object":{"id":"user-8","type":"User"}}""", + /** + * User-Id : user-2 + * Edata.state is update + * user's data missed from API + */ + """ + {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-2","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} + | + |""".stripMargin, + /** + * User-Id : user-3 + * Edata.state is update + * framework and block info not present + */ + """ + {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-3","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} + | + |""".stripMargin, + /** + * User-Id : user-4 + * Edata.state is update + * framework and block info not present + */ + """ + {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-4","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} + | + |""".stripMargin, + /** + * UserId = user-5 + * EData state is "Created" + * User SignupType is "sso" + * It should able to insert The Map(usersignintype, Validated) + * + */ + """ + |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Created","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"user-5"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"}""".stripMargin, + + /** + * AUDIT Event with object type as content. (not related to user) + * Props are not defined - Skip Count should incr (skipCount =+ 1) + */ + """ + |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"do_r8r8rew97we9r8","type":"content"},"edata":{"state":"Updated"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} + | + |""".stripMargin, + + /** + * AUDIT Event, But having invalid state(other than create/update) + * userid= user-6 + */ + + """ + | + |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-7","type":"user"},"edata":{"state":"wrongState"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} + |""".stripMargin, + /** + * User-Id : user-10 + * schoolname starting with '[' + * Having all the user data + */ + """ + {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-10","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} + | + |""".stripMargin + + ) + + /** + * User-Id : user-1 + * Edata.state is update + * Having pid as sunbird.app + */ + val telemetryEventWithAppPid: String = """ + |{"actor":{"type":"System","id":"cb0defc7-9f5f-4c0a-bb73-207d34ed2cf0"},"eid":"AUDIT","edata":{"state":"Updated","props":[]},"ver":"3.0","syncts":1631551986826,"@timestamp":"2021-09-13T16:53:06.826Z","ets":1.63155195956E12,"context":{"cdata":[{"id":"teacher","type":"UserRole"},{"type":"Tabs","id":"Library-Course"},{"id":"a9d1d5df-da2e-4006-8495-363cce1ceec2","type":"UserSession"}],"env":"sdk","channel":"0126684405014528002","pdata":{"id":"dev.sunbird.app","pid":"sunbird.app","ver":"4.1.892"},"sid":"a9d1d5df-da2e-4006-8495-363cce1ceec2","did":"575fc6d8a853d2f543e4ef54d57aa3b6cd0110cf","rollup":{"l1":"0126684405014528002"}},"flags":{"ex_processed":true,"pp_validation_processed":true,"pp_duplicate_skipped":true},"mid":"bb429169-f320-4da3-80ad-660093aafe3b","type":"events","object":{"id":"user-1","type":"user","version":"","rollup":{}}} + |""".stripMargin + + /** + * User-Id : user-1 + * Edata.state is update + * Having pid as learner-service + */ + val telemetryEventWithLearnerPid: String = """ + |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-1","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} + |""".stripMargin + +} \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpecV2.scala b/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpecV2.scala new file mode 100644 index 0000000000..e411a703f4 --- /dev/null +++ b/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpecV2.scala @@ -0,0 +1,348 @@ +package org.sunbird.dp.spec + +import java.io.IOException +import java.util +import com.google.gson.Gson +import com.typesafe.config.{Config, ConfigFactory} +import okhttp3.mockwebserver.{MockResponse, MockWebServer} +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration +import org.apache.flink.streaming.api.functions.source.SourceFunction +import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext +import org.apache.flink.test.util.MiniClusterWithClientResource +import org.mockito.Mockito +import org.mockito.Mockito._ +import org.scalatest.BeforeAndAfterEach +import org.sunbird.dp.core.cache.RedisConnect +import org.sunbird.dp.core.job.FlinkKafkaConnector +import org.sunbird.dp.fixture.EventFixture +import org.sunbird.dp.usercache.domain.Event +import org.sunbird.dp.usercache.task.{UserCacheUpdaterConfigV2, UserCacheUpdaterStreamTaskV2} +import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} +import redis.clients.jedis.Jedis +import redis.embedded.RedisServer + +class UserCacheUpdatetStreamTaskSpecV2 extends BaseTestSpec with BeforeAndAfterEach { + + implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + + val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() + .setConfiguration(testConfiguration()) + .setNumberSlotsPerTaskManager(1) + .setNumberTaskManagers(1) + .build) + + var redisServer: RedisServer = _ + val config: Config = ConfigFactory.load("test.conf") + val userCacheConfig: UserCacheUpdaterConfigV2 = new UserCacheUpdaterConfigV2(config) + val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) + val gson = new Gson() + var jedis: Jedis = _ + var server = new MockWebServer() + + override protected def beforeAll(): Unit = { + super.beforeAll() + redisServer = new RedisServer(6340) + redisServer.start() + BaseMetricsReporter.gaugeMetrics.clear() + val redisConnect = new RedisConnect(userCacheConfig.metaRedisHost, userCacheConfig.metaRedisPort, userCacheConfig) + jedis = redisConnect.getConnection(userCacheConfig.userStore) + flinkCluster.before() + } + + override protected def afterAll(): Unit = { + super.afterAll() + redisServer.stop() + server.close() + flinkCluster.after() + } + + override protected def beforeEach(): Unit = { + server.close() + server = new MockWebServer() + super.beforeEach() + } + + def setupRestUtilData(): Unit = { + + val user1 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":["English"],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","recoveryEmail":"","identifier":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Utkarsha","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Kapoor","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"administrator","subType":"deo"},"profileUserTypes":[{"type":"administrator","subType":"hm"},{"type":"teacher"},{"type":"administrator","subType":"crp"},{"type":"other"},{"type":"parent"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"DPS, MATHURA","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":["IGOT-Health"],"gradeLevel":["Volunteers"],"id":["igot_health"],"medium":["English"],"subject":["IRCS"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}""" + val clientError = """{"id":".private.user.v1.read.bcda3d80-0de0-4179-a702-9f89e6eeec56","ver":"private","ts":"2021-05-24 07:10:48:908+0000","params":{"resmsgid":null,"msgid":"f17cf782-6f67-4575-9152-21cf2daabe85","err":"USER_ACCOUNT_BLOCKED","status":"USER_ACCOUNT_BLOCKED","errmsg":"User account has been blocked ."},"responseCode":"CLIENT_ERROR","result":{}}""" + val user2 = """{"id":".private.user.v1.read.95e4942d-cbe8-477d-aebd-ad8e6de4bfc81","ver":"private","ts":"2021-03-16 11:42:24:074+0000","params":{"resmsgid":null,"msgid":"e082153a-159c-4d18-a2d3-a5dc509f4d9f","err":"USER_NOT_FOUND","status":"USER_NOT_FOUND","errmsg":"user not found."},"responseCode":"RESOURCE_NOT_FOUND","result":{}}""" + val user3 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":[],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"user-3","recoveryEmail":"","identifier":"user-3","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"01285019302823526477","prevUsedEmail":"","firstName":"Isha","lastName":"Wakankar","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"user-3","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"administrator","subType":"deo"},"profileUserTypes":[{"type":"administrator","subType":"deo"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"DPS, MATHURA","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"user-3","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}""" + val user4 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"thumbnail":null,"encPhone": "LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==", "encEmail": "LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==","updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Manju","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Davanam","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"teacher"},"profileUserTypes":[{"type":"teacher"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"type":"location","name":""}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":[],"gradeLevel":["Volunteers"],"medium":["English"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}""" + val invalid_json = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-04-21T02:51:39ZZ\",\"params\":{\"resmsgid\":\"4544fce4-efee-4ee2-8816-fdb3f60ac492\",\"msgid\":null,\"err\":null,\"status\":\"successful\",\"errmsg\":null},\"responseCode\":\"OK\",\"result\":{\"status\":\"No dialcodeFound\"}}" + val user10 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":["English"],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","recoveryEmail":"","identifier":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Utkarsha","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Kapoor","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"administrator-hm","subType":null},"profileUserTypes":[{"type":"administrator","subType":null}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1]","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"[RPMMAT M.S UDHADIH","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":["IGOT-Health"],"gradeLevel":["Volunteers"],"id":["igot_health"],"medium":["English"],"subject":["IRCS"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}""" + try + server.start(3000) + catch { + case e: IOException => + System.out.println("Exception" + e) + } + server.enqueue(new MockResponse().setBody(user1)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-1") + server.enqueue(new MockResponse().setBody(user1)) +// server.enqueue(new MockResponse().setBody(clientError)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-9") + server.enqueue(new MockResponse().setBody(user1)) +// server.enqueue(new MockResponse().setBody(user2)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-2") + server.enqueue(new MockResponse().setBody(user3)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-3") + server.enqueue(new MockResponse().setBody(user4)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-4") + server.enqueue(new MockResponse().setBody(user1)) +// server.enqueue(new MockResponse().setBody(user2)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-5") + server.enqueue(new MockResponse().setBody(user10)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-10") +// server.enqueue(new MockResponse().setBody(invalid_json)) +// server.url("http://127.0.0.1:3000/api/dialcode/v3/read/X3J6W2") +// server.enqueue(new MockResponse().setBody(invalid_json)) +// server.url("http://127.0.0.1:3000/api/dialcode/v3/read/X3J6W3") + server.enqueue(new MockResponse().setHeader("Authorization", "auth_token")) + + } + + def setupRestUtilDataWithErrors(): Unit = { + + val user1 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":["English"],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","recoveryEmail":"","identifier":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Utkarsha","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Kapoor","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"administrator","subType":"deo"},"profileUserTypes":[{"type":"administrator","subType":"hm"},{"type":"teacher"},{"type":"administrator","subType":"crp"},{"type":"other"},{"type":"parent"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"DPS, MATHURA","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":["IGOT-Health"],"gradeLevel":["Volunteers"],"id":["igot_health"],"medium":["English"],"subject":["IRCS"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}""" + val clientError = """{"id":".private.user.v1.read.bcda3d80-0de0-4179-a702-9f89e6eeec56","ver":"private","ts":"2022-03-17 08:33:32:188+0000","params":{"resmsgid":"4896831f-ef38-4731-bb0b-564a05e98576","msgid":"4896831f-ef38-4731-bb0b-564a05e98576","err":"UOS_USRRED0006","status":"FAILED","errmsg":"User account has been blocked ."},"responseCode":"CLIENT_ERROR","result":{}}""" + val user2 = """{"id":".private.user.v1.read.95e4942d-cbe8-477d-aebd-ad8e6de4bfc81","ver":"private","ts":"2021-03-16 11:42:24:074+0000","params":{"resmsgid":null,"msgid":"e082153a-159c-4d18-a2d3-a5dc509f4d9f","err":"USER_NOT_FOUND","status":"USER_NOT_FOUND","errmsg":"user not found."},"responseCode":"RESOURCE_NOT_FOUND","result":{}}""" + val user3 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":[],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"user-3","recoveryEmail":"","identifier":"user-3","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"01285019302823526477","prevUsedEmail":"","firstName":"Isha","lastName":"Wakankar","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"user-3","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserTypes":[{"type":"administrator","subType":"deo"}],"profileUserType":{"type":"administrator","subType":"deo"},"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"DPS, MATHURA","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"user-3","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}""" + val user4 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"thumbnail":null,"encPhone": "LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==", "encEmail": "LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==","updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Manju","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Davanam","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"teacher"},"profileUserTypes":[{"type":"teacher"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"type":"location","name":""}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":[],"gradeLevel":["Volunteers"],"medium":["English"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}""" + val invalid_json = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-04-21T02:51:39ZZ\",\"params\":{\"resmsgid\":\"4544fce4-efee-4ee2-8816-fdb3f60ac492\",\"msgid\":null,\"err\":null,\"status\":\"successful\",\"errmsg\":null},\"responseCode\":\"OK\",\"result\":{\"status\":\"No dialcodeFound\"}}" + try + server.start(3000) + catch { + case e: IOException => + System.out.println("Exception" + e) + } + server.enqueue(new MockResponse().setBody(user1)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-1") + server.enqueue(new MockResponse().setBody(clientError)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-9") + server.enqueue(new MockResponse().setBody(user2)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-2") + server.enqueue(new MockResponse().setBody(user3)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-3") + server.enqueue(new MockResponse().setBody(user4)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-4") + server.enqueue(new MockResponse().setBody(user2)) + server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-5") + server.enqueue(new MockResponse().setHeader("Authorization", "auth_token")) + + } + + "UserCacheUpdater" should "be able to add user to cache with all information" in { + setupRestUtilData() + when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new InputSource) + + val task = new UserCacheUpdaterStreamTaskV2(userCacheConfig, mockKafkaUtil) + task.process() + + /** + * Metrics Assertions + */ + BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.userCacheHit}").getValue() should be(7) + BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.totalEventsCount}").getValue() should be(9) + BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.skipCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.successCount}").getValue() should be(7) + BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.apiReadSuccessCount}").getValue() should be(7) + BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.apiReadMissCount}").getValue() should be(0) + + // select index: 12 + jedis.select(userCacheConfig.userStore) + + //user information: user-1 + var userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix + "user-1") + userInfo.get("firstname") should be ("Utkarsha") + userInfo.get("lastname") should be ("Kapoor") + userInfo.get("state") should be ("Odisha") + userInfo.get("district") should be ("CUTTACK") + userInfo.get("block") should be ("BLOCK1") + userInfo.get("cluster") should be ("CLUSTER1") + userInfo.get("schooludisecode") should be ("3183211") + userInfo.get("schoolname") should be ("DPS, MATHURA") + userInfo.get("usertype") should be ("administrator,teacher,other,parent") + userInfo.get("usersubtype") should be ("hm,crp") + userInfo.get("board") should be ("IGOT-Health") + userInfo.get("rootorgid") should be ("0127738024883077121") + userInfo.get("orgname") should be ("Root Org2") + userInfo.get("subject") should be ("""["IRCS"]""") + userInfo.get("language") should be ("""["English"]""") + userInfo.get("grade") should be ("""["Volunteers"]""") + userInfo.get("framework") should be ("igot_health") + userInfo.get("medium") should be ("""["English"]""") + userInfo.get("profileusertypes") should be ("""\[{"subType":"hm","type":"administrator"},{"type":"teacher"},{"subType":"crp","type":"administrator"},{"type":"other"},{"type":"parent"}\]""") + + //user information: user-2 + userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix + "user-2") + userInfo.get("usersignintype") should be ("Self-Signed-In") + userInfo.keySet() should not contain theSameElementsAs(Seq("firstname", "lastname", "schoolname", "schooludisecode", "schoolname")) + + /*** + * user information: user-3 + * Framework and block information is stored as empty string + */ + userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix + "user-3") + userInfo.get("firstname") should be ("Isha") + userInfo.get("lastname") should be ("Wakankar") + userInfo.get("state") should be ("Odisha") + userInfo.get("district") should be ("CUTTACK") + userInfo.get("block") should be (null) + userInfo.get("cluster") should be ("CLUSTER1") + userInfo.get("schooludisecode") should be ("3183211") + userInfo.get("schoolname") should be ("DPS, MATHURA") + userInfo.get("usertype") should be ("administrator") + userInfo.get("usersubtype") should be ("deo") + userInfo.get("rootorgid") should be ("01285019302823526477") + userInfo.get("orgname") should be ("Root Org2") + userInfo.get("board") should be (null) + userInfo.get("subject") should be (null) + userInfo.get("language") should be ("""[]""") + userInfo.get("grade") should be (null) + userInfo.get("framework") should be (null) + userInfo.get("medium") should be (null) + userInfo.get("profileusertypes") should be ("""\[{"subType":"deo","type":"administrator"}\]""") + + /*** + * userinformation: user-4 + * school information not present and subject not present in framework + */ + userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix + "user-4") + userInfo.get("firstname") should be ("Manju") + userInfo.get("lastname") should be ("Davanam") + userInfo.get("state") should be ("Odisha") + userInfo.get("district") should be ("CUTTACK") + userInfo.get("block") should be ("BLOCK1") + userInfo.get("cluster") should be ("CLUSTER1") + userInfo.get("schooludisecode") should be (null) + userInfo.get("schoolname") should be (null) + userInfo.get("usertype") should be ("teacher") + userInfo.get("usersubtype") should be ("") + userInfo.get("board") should be ("") + userInfo.get("rootorgid") should be ("0127738024883077121") + userInfo.get("orgname") should be ("Root Org2") + userInfo.get("subject") should be ("""[]""") + userInfo.get("language") should be ("") + userInfo.get("grade") should be ("""["Volunteers"]""") + userInfo.get("framework") should be ("") + userInfo.get("medium") should be ("""["English"]""") + userInfo.get("phone") should be ("LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==") + userInfo.get("email") should be ("LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==") + userInfo.get("profileusertypes") should be ("""\[{"type":"teacher"}\]""") + + /** + * UserId = user-5 + * EData state is "Created" + * User SignupType is "sso" + * It should able to insert The Map(usersignintype, Validated) + * + */ + userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix + "user-5") + userInfo.get("usersignintype") + + //user information: user-10 + userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix + "user-10") + userInfo.get("firstname") should be ("Utkarsha") + userInfo.get("lastname") should be ("Kapoor") + userInfo.get("state") should be ("Odisha") + userInfo.get("district") should be ("CUTTACK") + userInfo.get("block") should be ("""BLOCK1\]""") + userInfo.get("cluster") should be ("CLUSTER1") + userInfo.get("schooludisecode") should be ("3183211") + userInfo.get("schoolname") should be ("""\[RPMMAT M.S UDHADIH""") + userInfo.get("usertype") should be ("administrator") + userInfo.get("usersubtype") should be ("") + userInfo.get("board") should be ("IGOT-Health") + userInfo.get("rootorgid") should be ("0127738024883077121") + userInfo.get("orgname") should be ("Root Org2") + userInfo.get("subject") should be ("""["IRCS"]""") + userInfo.get("language") should be ("""["English"]""") + userInfo.get("grade") should be ("""["Volunteers"]""") + userInfo.get("framework") should be ("igot_health") + userInfo.get("medium") should be ("""["English"]""") + userInfo.get("profileusertypes") should be ("""\[{"type":"administrator"}\]""") + } + + "UserCacheUpdater" should "be able to add and update user record with different producer ids" in { + setupRestUtilData() + when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new LeanerInputSource) + + val task = new UserCacheUpdaterStreamTaskV2(userCacheConfig, mockKafkaUtil) + task.process() + + // select index: 12 + jedis.select(userCacheConfig.userStore) + + //user information: user-1 + var userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix + "user-1") + userInfo.get("firstname") should be ("Utkarsha") + userInfo.get("lastname") should be ("Kapoor") + userInfo.get("state") should be ("Odisha") + userInfo.get("district") should be ("CUTTACK") + userInfo.get("block") should be ("BLOCK1") + userInfo.get("cluster") should be ("CLUSTER1") + userInfo.getOrDefault(userCacheConfig.userLoginTypeKey, null) should be (null) + + when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new AppInputSource) + task.process() + + userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix + "user-1") + userInfo.get("firstname") should be ("Utkarsha") + userInfo.get("lastname") should be ("Kapoor") + userInfo.get("state") should be ("Odisha") + userInfo.get("district") should be ("CUTTACK") + userInfo.get("block") should be ("BLOCK1") + userInfo.get("cluster") should be ("CLUSTER1") + userInfo.getOrDefault(userCacheConfig.userLoginTypeKey, null) should be ("teacher") + } + + "UserCacheUpdater" should "throw exception" in intercept[Exception] { + setupRestUtilDataWithErrors + when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new InputSource) + + val task = new UserCacheUpdaterStreamTaskV2(userCacheConfig, mockKafkaUtil) + task.process() + + } +} + +abstract class InputSourceBase[T] extends SourceFunction[T] { + override def cancel() = {} +} + +class InputSource extends InputSourceBase[Event]{ + + override def run(ctx: SourceContext[Event]) { + val gson = new Gson() + EventFixture.telemetrEvents.foreach(f => { + val eventMap = gson.fromJson(f, new util.HashMap[String, Any]().getClass) + val event = new Event(eventMap) + event.kafkaKey() + ctx.collect(event) + }) + } +} + +class LeanerInputSource extends InputSourceBase[Event]{ + override def run(ctx: SourceContext[Event]): Unit = { + val gson = new Gson() + val eventMap = gson.fromJson(EventFixture.telemetryEventWithLearnerPid, new util.HashMap[String, Any]().getClass) + val event = new Event(eventMap) + event.kafkaKey() + ctx.collect(event) + } +} + +class AppInputSource extends InputSourceBase[Event]{ + override def run(ctx: SourceContext[Event]): Unit = { + val gson = new Gson() + val eventMap = gson.fromJson(EventFixture.telemetryEventWithAppPid, new util.HashMap[String, Any]().getClass) + val event = new Event(eventMap) + event.kafkaKey() + ctx.collect(event) + } +} + diff --git a/data-pipeline-flink/user-cache-updater/pom.xml b/data-pipeline-flink/user-cache-updater/pom.xml deleted file mode 100644 index 7d9d9efe43..0000000000 --- a/data-pipeline-flink/user-cache-updater/pom.xml +++ /dev/null @@ -1,225 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.sunbird.dp - pipeline-jobs - 1.0 - - - org.sunbird.dp.jobs - user-cache-updater - 1.0.0 - jar - UserCacheUpdater - - Updating user details into readis cache - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird.dp - dp-core - 1.0.0 - - - org.sunbird.dp - dp-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_2.12 - 1.10.0 - test - - - org.apache.flink - flink-runtime_2.12 - 1.10.0 - test - tests - - - it.ozimov - embedded-redis - 0.7.1 - test - - - com.google.guava - guava - - - - - org.cassandraunit - cassandra-unit - 3.1.1.0 - test - - - org.apache.flink - flink-streaming-java_2.12 - 1.10.0 - test - tests - - - org.scalatest - scalatest_2.12 - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - com.fiftyonred - mock-jedis - 0.4.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - false - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.dp.usercache.task.UserCacheUpdaterStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 3.2.2 - - 1.8 - 1.8 - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - usercache-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater/src/main/resources/log4j.properties b/data-pipeline-flink/user-cache-updater/src/main/resources/log4j.properties deleted file mode 100644 index 1be669fa75..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=user-cache-updater.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater/src/main/resources/user-cache-updater.conf b/data-pipeline-flink/user-cache-updater/src/main/resources/user-cache-updater.conf deleted file mode 100644 index 5e34574265..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/main/resources/user-cache-updater.conf +++ /dev/null @@ -1,30 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = ${job.env}".telemetry.audit" - groupId = ${job.env}"-user-cache-updater-group" -} - -task { - usercache.updater.parallelism = 1 -} - -# redis-metadata -redis-meta { - database { - userstore.id = 4 - } -} - -lms-cassandra { - keyspace = "sunbird" - table { - user = "user" - location = "location" - } -} - -user.self.signin.types = ["google","self"] -user.validated.types = ["sso"] -user.self.signin.key = "Self-Signed-In" -user.valid.key = "Validated" \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala b/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala deleted file mode 100644 index db9ac18f96..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala +++ /dev/null @@ -1,37 +0,0 @@ -package org.sunbird.dp.usercache.domain - -import java.util - -import org.sunbird.dp.core.domain.Events - -class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) { - - override def kafkaKey(): String = { - did() - } - - def getId: String = { - Option(objectType()).map({ t => if (t.equalsIgnoreCase("User")) objectID() else null - }).getOrElse(null) - } - - def getState: String = { - telemetry.read[String]("edata.state").getOrElse(null) - } - - def getContextDataId(cDataType: String): String = { - val cdata = telemetry.read[util.ArrayList[util.Map[String, AnyRef]]]("context.cdata").getOrElse(null) - var signInType: String = null - Option(cdata).map(data => { - data.forEach(cdataMap => { - if (cdataMap.get("type").asInstanceOf[String].equalsIgnoreCase(cDataType)) signInType = cdataMap.get("id").toString else signInType - }) - }).getOrElse(signInType) - signInType - } - - def userMetaData(): util.ArrayList[String] = { - telemetry.read[util.ArrayList[String]]("edata.props").getOrElse(new util.ArrayList[String]()) - } - -} diff --git a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunction.scala b/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunction.scala deleted file mode 100644 index 2a356cadec..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunction.scala +++ /dev/null @@ -1,161 +0,0 @@ -package org.sunbird.dp.usercache.functions - -import java.util - -import com.datastax.driver.core.Row -import com.datastax.driver.core.querybuilder.{Clause, QueryBuilder} -import com.google.gson.Gson -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.dp.core.cache.{DataCache, RedisConnect} -import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics} -import org.sunbird.dp.core.util.CassandraUtil -import org.sunbird.dp.usercache.domain.Event -import org.sunbird.dp.usercache.task.UserCacheUpdaterConfig - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -class UserCacheUpdaterFunction(config: UserCacheUpdaterConfig)(implicit val mapTypeInfo: TypeInformation[Event]) - extends BaseProcessFunction[Event, Event](config) { - - private[this] val logger = LoggerFactory.getLogger(classOf[UserCacheUpdaterFunction]) - private var dataCache: DataCache = _ - private var cassandraConnect: CassandraUtil = _ - - override def metricsList(): List[String] = { - List(config.dbReadSuccessCount, config.dbReadMissCount, config.userCacheHit, config.skipCount, config.successCount, config.totalEventsCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.userStore, config.userFields) - dataCache.init() - cassandraConnect = new CassandraUtil(config.cassandraHost, config.cassandraPort) - } - - override def close(): Unit = { - super.close() - dataCache.close() - } - - override def processElement(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - Option(event.getId).map(id => { - Option(event.getState).map(name => { - val userData: mutable.Map[String, AnyRef] = name.toUpperCase match { - case "CREATE" | "CREATED" => createAction(id, event, metrics) - case "UPDATE" | "UPDATED" => updateAction(id, event, metrics) - case _ => { - logger.info(s"Invalid event state name either it should be(Create/Created/Update/Updated) but found $name for ${event.mid()}") - metrics.incCounter(config.skipCount) - mutable.Map[String, AnyRef]() - } - } - if (!userData.isEmpty) { - dataCache.setWithRetry(id, new Gson().toJson(mapAsJavaMap(userData))) - metrics.incCounter(config.successCount) - metrics.incCounter(config.userCacheHit) - } else { - metrics.incCounter(config.skipCount) - } - }).getOrElse(metrics.incCounter(config.skipCount)) - }).getOrElse(metrics.incCounter(config.skipCount)) - } - - - /** - * When Edata.State is Create/Created then insert the user data into redis in string formate - */ - def createAction(userId: String, event: Event, metrics: Metrics): mutable.Map[String, AnyRef] = { - val userData: mutable.Map[String, AnyRef] = mutable.Map[String, AnyRef]() - Option(event.getContextDataId(cDataType = "SignupType")).map(signInType => { - if (config.userSelfSignedInTypeList.contains(signInType)) { - userData.put(config.userSignInTypeKey, config.userSelfSignedKey) - } - if (config.userValidatedTypeList.contains(signInType)) { - userData.put(config.userSignInTypeKey, config.userValidatedKey) - } - }).orNull - userData - } - - /** - * When edata.state is update/updated then update the user metadta - * information by reading from the cassandra - */ - - def updateAction(userId: String, event: Event, metrics: Metrics): mutable.Map[String, AnyRef] = { - val userCacheData: mutable.Map[String, AnyRef] = dataCache.getWithRetry(userId) - - Option(event.getContextDataId(cDataType = "UserRole")).map(loginType => { - userCacheData.put(config.userLoginTypeKey, loginType) - }) - val userMetaDataList = event.userMetaData() - if (!userMetaDataList.isEmpty && userCacheData.contains(config.userSignInTypeKey) && ("Anonymous" != userCacheData.get(config.userSignInTypeKey).toString)) { - - // Get the user details from the cassandra table - val userDetails: mutable.Map[String, AnyRef] = userCacheData.++(extractUserMetaData(readFromCassandra( - keyspace = config.keySpace, - table = config.userTable, - QueryBuilder.eq("id", userId), - metrics) - )) - - // Fetching the location details from the cassandra table - val updatedUserDetails: mutable.Map[String, AnyRef] = userDetails.++(extractLocationMetaData(readFromCassandra( - keyspace = config.keySpace, - table = config.locationTable, - clause = QueryBuilder.in("id", userDetails.getOrElse("locationids", new util.ArrayList()).asInstanceOf[util.ArrayList[String]]), - metrics) - )) - logger.info(s"User details ( $userId ) are fetched from the db's and updating the redis now.") - updatedUserDetails - } else { - logger.info(s"Skipping the event update from databases since event Does not have user properties or user sigin in type is Anonymous ") - userCacheData - } - } - - def readFromCassandra(keyspace: String, table: String, clause: Clause, metrics: Metrics): util.List[Row] = { - var rowSet: util.List[Row] = null - val query = QueryBuilder.select.all.from(keyspace, table).where(clause).toString - rowSet = cassandraConnect.find(query) - if (null != rowSet && !rowSet.isEmpty) { - metrics.incCounter(config.dbReadSuccessCount) - rowSet - } else { - metrics.incCounter(config.dbReadMissCount) - new util.ArrayList[Row]() - } - - } - - def extractUserMetaData(userDetails: util.List[Row]): mutable.Map[String, AnyRef] = { - val result: mutable.Map[String, AnyRef] = mutable.Map[String, AnyRef]() - if (null != userDetails && !userDetails.isEmpty) { - val row: Row = userDetails.get(0) - val columnDefinitions = row.getColumnDefinitions() - val columnCount = columnDefinitions.size - for (i <- 0 until columnCount) { - result.put(columnDefinitions.getName(i), row.getObject(i)) - } - } - result - } - - def extractLocationMetaData(locationDetails: util.List[Row]): mutable.Map[String, AnyRef] = { - val result: mutable.Map[String, AnyRef] = mutable.Map[String, AnyRef]() - locationDetails.forEach((record: Row) => { - record.getString("type").toLowerCase match { - case config.stateKey => result.put(config.stateKey, record.getString("name")) - case config.districtKey => result.put(config.districtKey, record.getString("name")) - case _ => // Do nothing - } - }) - result - } - -} diff --git a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfig.scala b/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfig.scala deleted file mode 100644 index 2f0e71ef4b..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfig.scala +++ /dev/null @@ -1,58 +0,0 @@ -package org.sunbird.dp.usercache.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.sunbird.dp.core.job.BaseJobConfig -import org.sunbird.dp.usercache.domain.Event - -import java.util.{List => JList} - -class UserCacheUpdaterConfig(override val config: Config) extends BaseJobConfig(config, "UserCacheUpdaterJob") { - - private val serialVersionUID = 2905979434303791379L - - implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - // Kafka Topics Configuration - val inputTopic: String = config.getString("kafka.input.topic") - val userFields = List("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype","locationids") - - // User cache updater job metrics - val userCacheHit = "user-cache-hit" - val skipCount = "skipped-message-count" - val successCount = "success-message-count" - val dbReadSuccessCount = "db-read-success-count" - val dbReadMissCount = "db-read-miss-count" - val totalEventsCount ="total-audit-events-count" - - val userSelfSignedInTypeList: JList[String] = config.getStringList("user.self.signin.types") - val userValidatedTypeList: JList[String] = config.getStringList("user.validated.types") - val userSelfSignedKey: String = config.getString("user.self.signin.key") - val userValidatedKey: String = config.getString("user.valid.key") - - // Redis - val userStore: Int = config.getInt("redis-meta.database.userstore.id") - - // lms-cassandra - val keySpace: String = config.getString("lms-cassandra.keyspace") - val locationTable: String = config.getString("lms-cassandra.table.location") - val userTable: String = config.getString("lms-cassandra.table.user") - val cassandraHost: String = config.getString("lms-cassandra.host") - val cassandraPort: Int = config.getInt("lms-cassandra.port") - - val userCacheParallelism: Int = config.getInt("task.usercache.updater.parallelism") - - // constants - val userSignInTypeKey = "usersignintype" - val userLoginTypeKey = "userlogintype" - val stateKey = "state" - val districtKey = "district" - - // Consumers - val userCacheConsumer = "user-cache-consumer" - - // Functions - val userCacheUpdaterFunction = "UserCacheUpdaterFunction" - -} diff --git a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTask.scala b/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTask.scala deleted file mode 100644 index dcb0606948..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTask.scala +++ /dev/null @@ -1,48 +0,0 @@ -package org.sunbird.dp.usercache.task - -import java.io.File - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment -import org.sunbird.dp.core.job.FlinkKafkaConnector -import org.sunbird.dp.core.util.FlinkUtil -import org.sunbird.dp.usercache.domain.Event -import org.sunbird.dp.usercache.functions.UserCacheUpdaterFunction - -class UserCacheUpdaterStreamTask(config: UserCacheUpdaterConfig, kafkaConnector: FlinkKafkaConnector) { - - private val serialVersionUID = -7729362727131516112L - - def process(): Unit = { - - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - val source = kafkaConnector.kafkaEventSource[Event](config.inputTopic) - env.addSource(source, config.userCacheConsumer).uid(config.userCacheConsumer).rebalance() - .process(new UserCacheUpdaterFunction(config)).setParallelism(config.userCacheParallelism) - .name(config.userCacheUpdaterFunction).uid(config.userCacheUpdaterFunction) - env.execute(config.jobName) - } - -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object UserCacheUpdaterStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("user-cache-updater.conf").withFallback(ConfigFactory.systemEnvironment())) - val userCacheUpdaterConfig = new UserCacheUpdaterConfig(config) - val kafkaUtil = new FlinkKafkaConnector(userCacheUpdaterConfig) - val task = new UserCacheUpdaterStreamTask(userCacheUpdaterConfig, kafkaUtil) - task.process() - } -} - -// $COVERAGE-ON$ diff --git a/data-pipeline-flink/user-cache-updater/src/test/resources/data.cql b/data-pipeline-flink/user-cache-updater/src/test/resources/data.cql deleted file mode 100644 index a98159edb6..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/test/resources/data.cql +++ /dev/null @@ -1,112 +0,0 @@ -CREATE KEYSPACE sunbird WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 3}; - -CREATE TABLE sunbird.user ( - id text PRIMARY KEY, - avatar text, - channel text, - countrycode text, - createdby text, - createddate text, - currentlogintime text, - dob text, - email text, - emailverified boolean, - firstname text, - flagsvalue int, - framework map>>, - gender text, - grade list, - isdeleted boolean, - language list, - lastlogintime text, - lastname text, - location text, - locationids list, - loginid text, - maskedemail text, - maskedphone text, - password text, - phone text, - phoneverified boolean, - prevusedemail text, - prevusedphone text, - profilesummary text, - profilevisibility map, - provider text, - recoveryemail text, - recoveryphone text, - registryid text, - roles list, - rootorgid text, - status int, - subject list, - tcstatus text, - tcupdateddate text, - temppassword text, - thumbnail text, - tncacceptedon timestamp, - tncacceptedversion text, - updatedby text, - updateddate text, - userid text, - username text, - usertype text, - webpages list>> -) WITH bloom_filter_fp_chance = 0.01 - AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} - AND comment = '' - AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} - AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} - AND crc_check_chance = 1.0 - AND dclocal_read_repair_chance = 0.1 - AND default_time_to_live = 0 - AND gc_grace_seconds = 864000 - AND max_index_interval = 2048 - AND memtable_flush_period_in_ms = 0 - AND min_index_interval = 128 - AND read_repair_chance = 0.0 - AND speculative_retry = '99PERCENTILE'; -CREATE INDEX inx_u_userid ON sunbird.user (userid); -CREATE INDEX inx_u_email ON sunbird.user (email); -CREATE INDEX inx_u_loginid ON sunbird.user (loginid); -CREATE INDEX inx_u_status ON sunbird.user (status); -CREATE INDEX inx_u_username ON sunbird.user (username); -CREATE INDEX inx_u_phone ON sunbird.user (phone); - -INSERT INTO sunbird.user ( - id, - avatar, - channel , - countrycode, - createdby, - createddate, - currentlogintime, - dob, - email, - emailverified, - firstname, - flagsvalue, - gender, - lastname, - location, - maskedemail, - maskedphone, - password, - phone, - phoneverified -) -VALUES ('user-3', 'VOS','sunbird', 'IN','MANJU','24-apr-2020','24-apr-2020','24-08-1993','manjunathd@ilimi.in',true,'Manjunath', 1, 'Male','Davanam','Banglore','********gmail.com','******181', '*******','4682647342',true); - -CREATE TABLE sunbird.location ( - id text PRIMARY KEY, - code text, - name text, - parentid text, - type text -); - -CREATE INDEX inx_loc_code ON sunbird.location (code); - -INSERT INTO sunbird.location(id,code,name,type) VALUES ('location-1','0f5f0702000f6', 'KARNATAKA', 'state'); -INSERT INTO sunbird.location(id,code,name,type) VALUES ('location-2','g0507020f00f6', 'TUMKUR', 'district'); -INSERT INTO sunbird.location(id,code,name,type) VALUES ('location-3','331211','MANVI','block'); \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater/src/test/resources/test.conf b/data-pipeline-flink/user-cache-updater/src/test/resources/test.conf deleted file mode 100644 index b66d4a4bd6..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/test/resources/test.conf +++ /dev/null @@ -1,29 +0,0 @@ -include "base-test.conf" - -kafka { - input.topic = "flink.telemetry.audit" - groupId = "flink-user-cache-updater-group" -} - -task { - usercache.updater.parallelism = 1 -} - -redis-meta { - database { - userstore.id = 4 - } -} - -lms-cassandra { - keyspace = "sunbird" - table { - user = "user" - location = "location" - } -} - -user.self.signin.types = ["google","self"] -user.validated.types = ["sso"] -user.self.signin.key = "Self-Signed-In" -user.valid.key = "Validated" \ No newline at end of file diff --git a/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala deleted file mode 100644 index 9d42ae92c2..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala +++ /dev/null @@ -1,125 +0,0 @@ -package org.sunbird.dp.fixture - -object EventFixture { - - val userCacheData3 = """{"usersignintype":"Validated","usertype":"TEACHER"}""" - val userCacheData4 = """{"channel":"KV123","phoneverified":false,"createdby":"c8e51123-61a3-454d-beb0-2202450b0096","subject":["English"],"email":"BJAguqy3GaJECrYqDUPjeducVxa5J9ZsW9A8qc7YHelkV7KbgkCKW10quCbhpgxbh2t4toXC8uXW\\ngiguS+8ucwzbmgPm7q7YSYz26SfpHnzBo/0Vh3TWqr2MOq9LlX6gT6a+wzaAmCWueMEdPmZuRg==","username":"I+CyiN6Bx0GCRm9lkA3xn5uNBm0AODhxeDwJebxxBfuGJ5V2v1R8v1PEQsP+V+y9sAFcM2WtaMLj\\n91hpzBq0PFcQTq6OSPQOm0sySPXTDzyLvm1cKaLwzvJ6fzLLs9nKT6a+wzaAmCWueMEdPmZuRg==","firstname":"A512","framework":{},"userid":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","usertype":"TEACHER","rootorgid":"0126978705345576967","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","language":[],"grade":[],"roles":["BOOK_REVIEWER"],"status":1,"webpages":[],"createddate":"2019-04-11 08:58:16:512+0000","emailverified":true,"isdeleted":false,"locationids":["location-1","location-2","location-3"],"maskedemail":"a5**@yopmail.com","profilevisibility":{},"loginid":"I+CyiN6Bx0GCRm9lkA3xnx2W8+QgN39Y0We3KjR98O8hD6YjyoCirIBDsWHGwRf65PY/Cx+pFFK1\\nIz1VinIaKgDnSQwkl7ajzQjjRTzQbKOyHsAXkJgo9I5l7ulEYVXRT6a+wzaAmCWueMEdPmZuRg==","usersignintype":"Self-Signed-In","userlogintype":"Student","state":"Telangana","district":"Hyderabad"}""" - - val telemetrEvents: List[String] = List( - - /** - * UserId = 89490534-126f-4f0b-82ac-3ff3e49f3468 - * EData state is "Created" - * User SignupType is "sso" - * It should able to insert The Map(usersignintype, Validated) - * - */ - """ - |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Created","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"user-1"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"}""".stripMargin, - - /** - * UserId = user-2 - * EData state is "create" - * User SignupType is "google" - * It should able to insert The Map(usersignintype, Self-Signed-In) - */ - - - """ - |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"create","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"user-2"},{"type":"SignupType","id":"google"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"user-2"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"}""".stripMargin, - - /** - * User-Id :user-3 - * Edata.state is update - * - */ - """ - |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-3", "type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} - | - |""".stripMargin, - - /** - * UseId-Id: user-4 - * Location id's are defined in the events and also loaded into redis db - */ - - """ - |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-4","type":"user"},"edata":{"state":"Updated","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy","locationIds"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} - | - |""".stripMargin, - - - /** - * User Id - user-5 - * Props are not defined - */ - """ - |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-5","type":"user"},"edata":{"state":"Updated"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} - | - |""".stripMargin, - - - /** - * AUDIT Event with object type as content. (not related to user) - * Props are not defined - Skip Count should incr (skipCount =+ 1) - */ - """ - |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"do_r8r8rew97we9r8","type":"content"},"edata":{"state":"Updated"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} - | - |""".stripMargin, - - /** - * AUDIT Event, But having invalid state(other than create/update) - * - */ - - """ - | - |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-4","type":"user"},"edata":{"state":"wrongState"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} - |""".stripMargin, - - - /** - * AUDIT Event, But having without state id null - * - */ - - """ - | - |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-4","type":"user"},"edata":{},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} - |""".stripMargin - - - - - // |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606dfdd6","actor":{"id":"627a431d-4f5c-4adc-812d-1f01c5588555","type":"User"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"2bcfc645e27e64625f7bad6ce282f9d0","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"627a431d-4f5c-4adc-812d-1f01c5588555","type":"User"},"edata":{"state":"Create","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} - // |""".stripMargin, - // - // """ - // |{"eid":"AUDIT","ets":1.573121861118E12,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"2bcfc645e27e64625f7bad6ce282f9d0","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"5609876543h2fd34h5678jf909876af54345678"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1.573121861125E12,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} - // |""".stripMargin, - // - // """ - // |{"eid":"AUDIT","ets":1.573121861118E12,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"2bcfc645e27e64625f7bad6ce282f9d0","rollup":{"l1":"01285019302823526477"}},"object":{"type":"User","id":"5609876543h2fd34h5678jf909876af54345678"},"edata":{"state":"Create","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1.573121861125E12,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"} - // |""".stripMargin, - // - // """ - // |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Create","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"} - // |""".stripMargin - // """ - // |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Created","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"} - // |""".stripMargin, - // """ - // |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Create","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"} - // |""".stripMargin, - // """ - // |{"actor":{"type":"System","id":"3b46b4c9-3a10-439a-a2cb-feb5435b3a0d"},"eid":"AUDIT","edata":{"state":"Update","props":["medium","board","grade","syllabus","gradeValue"]},"ver":"3.0","ets":1561739240727,"context":{"pdata":{"pid":"sunbird.app","ver":"2.1.92","id":"prod.diksha.app"},"channel":"505c7c48ac6dc1edc9b08f21db5a571d","env":"sdk","did":"010612971a80a7677d0a3e849ab35cb4a83157de","cdata":[{"type":"UserRole","id":"student"}],"sid":"ea68a05e-0843-4c06-9a84-9b98cd974724"},"mid":"0268860e-76b0-4b4e-b99b-ebf543e7a9d8","object":{"id":"3b46b4c9-3a10-439a-a2cb-feb5435b3a0d","type":"User","version":"","rollup":{}},"syncts":1561739245463,"@timestamp":"2019-06-28T16:27:25.463Z","flags":{"tv_processed":true},"type":"events"} - // |""".stripMargin, - // """ - // |{"eid":"AUDIT","ets":1571297660511,"ver":"3.0","mid":"1571297660511.32f5024a-aa30-4c82-abd8-bb8d8914ed2d","actor":{"id":"ef70da5a-bb99-4785-b970-1d6d6ee75aad","type":"User"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.4.0"},"env":"User","did":"bfaf115f65a2086b062735885f4d2f1a","cdata":[{"id":"1b1392bc-39d8-6e47-d7d6-5781a2f1481a","type":"Request"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"52226956-61d8-4c1b-b115-c660111866d3","type":"User"},"edata":{"state":"Update","props":["firstName","userId","id","externalIds","locationIds","updatedDate","updatedBy"]},"syncts":1571297660521,"@timestamp":"2019-10-17T07:34:20.521Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-10-17T07:34:20.511+0000"} - // |""".stripMargin - - ) - - -} diff --git a/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpec.scala b/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpec.scala deleted file mode 100644 index 6745cf9e88..0000000000 --- a/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpec.scala +++ /dev/null @@ -1,175 +0,0 @@ -package org.sunbird.dp.spec - -import java.util - -import com.google.gson.Gson -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.sunbird.dp.core.cache.RedisConnect -import org.sunbird.dp.core.job.FlinkKafkaConnector -import org.sunbird.dp.core.util.CassandraUtil -import org.sunbird.dp.fixture.EventFixture -import org.sunbird.dp.usercache.domain.Event -import org.sunbird.dp.usercache.task.{UserCacheUpdaterConfig, UserCacheUpdaterStreamTask} -import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec} -import redis.clients.jedis.Jedis -import redis.embedded.RedisServer - -class UserCacheUpdatetStreamTaskSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - - var redisServer: RedisServer = _ - val config: Config = ConfigFactory.load("test.conf") - val userCacheConfig: UserCacheUpdaterConfig = new UserCacheUpdaterConfig(config) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val gson = new Gson() - var jedis: Jedis = _ - - override protected def beforeAll(): Unit = { - super.beforeAll() - println("******Starting the Embedded Cassandra*******") - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - val cassandraUtil = new CassandraUtil(userCacheConfig.cassandraHost, userCacheConfig.cassandraPort) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/data.cql").getPath, true, true)); - testCassandraUtil(cassandraUtil) - redisServer = new RedisServer(6340) - redisServer.start() - BaseMetricsReporter.gaugeMetrics.clear() - val redisConnect = new RedisConnect(userCacheConfig.metaRedisHost, userCacheConfig.metaRedisPort, userCacheConfig) - - jedis = redisConnect.getConnection(userCacheConfig.userStore) - setupRedisTestData(jedis) - flinkCluster.before() - } - - override protected def afterAll(): Unit = { - super.afterAll() - redisServer.stop() - flinkCluster.after() - } - - def setupRedisTestData(jedis: Jedis) { - - - // Insert user test data - jedis.set("user-3", EventFixture.userCacheData3) - jedis.set("user-4", EventFixture.userCacheData4) - jedis.close() - } - - "UserCache Updater pipeline" should "Should able to add user data into cache" in { - - when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new InputSource) - - val task = new UserCacheUpdaterStreamTask(userCacheConfig, mockKafkaUtil) - task.process() - - /** - * Metrics Assertions - */ - BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.userCacheHit}").getValue() should be(5) - BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.totalEventsCount}").getValue() should be(8) - BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.dbReadSuccessCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.dbReadMissCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.skipCount}").getValue() should be(4) - BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.successCount}").getValue() should be(5) - - /** - * UserId = 89490534-126f-4f0b-82ac-3ff3e49f3468 - * EData state is "Created" - * User SignupType is "sso" - * It should able to insert The Map(usersignintype, Validated) - */ - jedis.select(userCacheConfig.userStore) - - val ssoUser = jedis.get("user-1") - ssoUser should not be null - val ssoUserMap: util.Map[String, AnyRef] = gson.fromJson(ssoUser, new util.LinkedHashMap[String, AnyRef]().getClass) - ssoUserMap.get("usersignintype") should be("Validated") - - /** - * UserId = user-2 - * EData state is "Created" - * User SignupType is "google" - * It should able to insert The Map(usersignintype, Self-Signed-In) - */ - val googleUser = jedis.get("user-2") - googleUser should not be null - val googleUserMap: util.Map[String, AnyRef] = gson.fromJson(googleUser, new util.LinkedHashMap[String, AnyRef]().getClass) - googleUserMap.get("usersignintype") should be("Self-Signed-In") - - - // When action is Update and location id's are empty - - val userInfo = jedis.get("user-3") - val userInfoMap: util.Map[String, AnyRef] = gson.fromJson(userInfo, new util.LinkedHashMap[String, AnyRef]().getClass) - userInfoMap.get("createdby") should be("MANJU") - userInfoMap.get("location") should be("Banglore") - userInfoMap.get("maskedphone") should be("******181") - - // When action is Updated and location ids are present - val locationInfo = jedis.get("user-4") - val locationInfoMap: util.Map[String, AnyRef] = gson.fromJson(locationInfo, new util.LinkedHashMap[String, AnyRef]().getClass) - - // Initially, in the redis for the user-4 is loaded with location details - // State - Telangana & District - Hyderabad - // Now it should be bellow assertions - locationInfoMap.get("state") should be("KARNATAKA") - locationInfoMap.get("district") should be("TUMKUR") - - - val emptyProps = jedis.get("user-5") - val emptyPropsMap: util.Map[String, AnyRef] = gson.fromJson(emptyProps, new util.LinkedHashMap[String, AnyRef]().getClass) - emptyPropsMap.get(userCacheConfig.userLoginTypeKey) should be("25cb0530-7c52-ecb1-cff2-6a14faab7910") - - } - - def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { - cassandraUtil.reconnect() - val response = cassandraUtil.findOne("SELECT * FROM sunbird.location;") - response should not be null - val upsert = cassandraUtil.upsert("SELECT * FROM sunbird.location;") - upsert should be(true) - cassandraUtil.getUDTType("sunbird", "test") should not be not - cassandraUtil.close() - val onSessionClose = cassandraUtil.find("SELECT * FROM sunbird.location;") - onSessionClose should not be null - - } - -} - -class InputSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - val gson = new Gson() - EventFixture.telemetrEvents.foreach(f => { - val eventMap = gson.fromJson(f, new util.HashMap[String, Any]().getClass) - val event = new Event(eventMap) - event.kafkaKey() - ctx.collect(event) - }) - } - - override def cancel() = {} -} - diff --git a/data-pipeline/.gitignore b/data-pipeline/.gitignore deleted file mode 100644 index 0435c14b33..0000000000 --- a/data-pipeline/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*.class -*.war -*.ear -target/ -.classpath -.project -.vagrant -.settings/ -.idea/ -.idea_modules/ -*.iml -*.ipr -*.iws -*/.cache -deploy -*.swp diff --git a/data-pipeline/README.md b/data-pipeline/README.md deleted file mode 100644 index e0cd6c88b7..0000000000 --- a/data-pipeline/README.md +++ /dev/null @@ -1,26 +0,0 @@ -To create a working kafka-samza pipeline, do the following: -- bin/grid bootstrap -- mvn clean package -- mkdir -p deploy/samza -- tar -xvf ./target/ekstep-samza-0.0.1-dist.tar.gz -C deploy/samza - -Assuming that kafka is running -- deploy/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic telemetry_events - -Starting a producer -- deploy/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic telemetry_events - -Starting a consumer (to see if everything is working) -- deploy/kafka/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic events_with_location - -Note: If job cannot get location of an event, the event will be posted to another topic called events_failed_location - -Sample events for producer -- {"did":"bc811958-b4b7-4873-a43a-03718edba45b","edata":{"eks":{"loc":"12.9310593,77.6238299","ueksid":"sharan"}},"eid":"GE_SESSION_START","gdata":{"id":"genie.android","ver":"2.2.18"},"sid":"6d5d6eeb-4f1b-4eed-8641-ec9e1884a218","ts":"2015-07-14T12:43:47+05:30","uid":"31e1cbf2b23a01ea035ee3323fe2ab95950c8284","ver":"1.0"} -- {"did":"bc811958-b4b7-4873-a43a-03718edba45b","edata":{"eks":{"err":"","gid":"org.ekstep.math.pp","length":9,"tmsize":0}},"eid":"GE_GAME_END","gdata":{"id":"genie.android","ver":"2.2.18"},"sid":"6d5d6eeb-4f1b-4eed-8641-ec9e1884a218","ts":"2015-07-14T12:43:59+05:30","uid":"31e1cbf2b23a01ea035ee3323fe2ab95950c8284","ver":"1.0"} - -Deploying job on YARN -- deploy/samza/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file://$PWD/deploy/samza/config/reverseSearch.properties - -So, simple setup will do the following -test11 -> reverse_search -> test13 diff --git a/data-pipeline/assessment-aggregator/pom.xml b/data-pipeline/assessment-aggregator/pom.xml deleted file mode 100644 index a0ee8418a8..0000000000 --- a/data-pipeline/assessment-aggregator/pom.xml +++ /dev/null @@ -1,164 +0,0 @@ - - - 4.0.0 - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - org.ekstep.ecosystem.jobs - assessment-aggregator - 0.0.4 - AssessmentAggregator - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - com.google.code.gson - gson - 2.4 - - - com.google.guava - guava - 18.0 - - - com.datastax.cassandra - cassandra-driver-core - 3.1.0 - - - org.cassandraunit - cassandra-unit - 3.1.1.0 - test - - - org.jetbrains - annotations - RELEASE - compile - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - \ No newline at end of file diff --git a/data-pipeline/assessment-aggregator/src/main/assembly/src.xml b/data-pipeline/assessment-aggregator/src/main/assembly/src.xml deleted file mode 100644 index 70b36631b8..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/assessment-aggregator.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-kv-rocksdb_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:assessment-aggregator - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/assessment-aggregator/src/main/config/assessment-aggregator.properties b/data-pipeline/assessment-aggregator/src/main/config/assessment-aggregator.properties deleted file mode 100644 index facaca80f6..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/config/assessment-aggregator.properties +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -####Environment - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.AssessmentAggregator - -# YARN -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__assessment_aggregator_yarn_container_count__ - - -# Task -task.class=org.ekstep.ep.samza.task.AssessmentAggregatorTask -task.inputs=kafka.__env__.telemetry.assess -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot,jmx -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics -metrics.reporter.jmx.class=org.apache.samza.metrics.reporter.JmxReporterFactory - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.msg.serde=string -systems.kafka.samza.key.serde=string -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__ingestion_zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__ingestion_kafka_brokers__ -systems.kafka.consumer.fetch.message.max.bytes=__assessment_aggregator_consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__assessment_aggregator_consumer_fetch_max_bytes__ -systems.kafka.samza.fetch.threshold=__assessment_aggregator_messages_fetch_threshold__ - - -## Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -output.failed.topic.name=__env__.telemetry.assess.failed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=db-hit-count,db-insert-count,db-update-count - - -# sunbird middleware cassandra config -middleware.cassandra.host=__middleware_cassandra_host__ -middleware.cassandra.port=__middleware_cassandra_port__ -middleware.cassandra.courses_keyspace=__middleware_cassandra__courses_keyspace__ -middleware.cassandra.aggregator_table=__middleware_cassandra_assessment_aggregator_table__ -middleware.cassandra.question_type=__middleware_cassandra_assessment_question_type__ \ No newline at end of file diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Aggregate.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Aggregate.java deleted file mode 100644 index ebd808cc5c..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Aggregate.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.datastax.driver.core.UDTValue; - -import java.text.DecimalFormat; -import java.util.List; - - -public class Aggregate { - - private double totalScore; - private double totalMaxScore; - private List questionsList; - private DecimalFormat df = new DecimalFormat("0.0#"); - - public Aggregate(double totalScore, double totalMaxScore, List questionsList) { - - this.totalScore = totalScore; - this.totalMaxScore = totalMaxScore; - this.questionsList = questionsList; - } - - public double getTotalScore() { - return totalScore; - } - - - public double getTotalMaxScore() { - return totalMaxScore; - } - - public String getGrandTotal() { - return String.format("%s/%s", df.format(totalScore), df.format(totalMaxScore)); - } - - public List getQuestionsList() { - return questionsList; - } - - -} diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/BatchEvent.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/BatchEvent.java deleted file mode 100644 index 5cbcec432c..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/BatchEvent.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; -import org.ekstep.ep.samza.reader.NullableValue; -import org.ekstep.ep.samza.reader.Telemetry; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class BatchEvent { - private final Telemetry telemetry; - - public BatchEvent(Map map) { - this.telemetry = new Telemetry(map); - - } - - public String getJson() { - Gson gson = new Gson(); - return gson.toJson(telemetry.getMap()); - } - - public Long assessmentEts() { - NullableValue ets = telemetry.read("assessmentTs"); - if (ets.value().getClass().equals(Double.class)) { - return ((Double) ets.value()).longValue(); - } - return ((Long) ets.value()); - } - - - public String courseId() { - NullableValue courseid = telemetry.read("courseId"); - return courseid.value(); - } - - public String contentId() { - NullableValue worksheetId = telemetry.read("contentId"); - return worksheetId.value(); - } - - public String batchId() { - NullableValue batchId = telemetry.read("batchId"); - return batchId.value(); - } - - public String attemptId() { - NullableValue attemptId = telemetry.read("attemptId"); - return attemptId.value(); - } - - public String userId() { - NullableValue userId = telemetry.read("userId"); - return userId.value(); - } - - public List> assessEvents() { - NullableValue>> assessEvents = telemetry.read("events"); - return assessEvents.value(); - } - - public void markSkipped(BatchEvent batchEvent) { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.addFieldIfAbsent("metadata", new HashMap()); - telemetry.add("flags.batch_aggregate_assess_skipped", true); - telemetry.add("metadata.aggregate_assess", batchEvent.attemptId() + - " : skipping batch Event older than last batch assessment time"); - } - - - @Override - public String toString() { - return "BatchEvent{" + - "telemetry=" + telemetry + - '}'; - } -} - diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Question.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Question.java deleted file mode 100644 index b9fb62dd68..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Question.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; - -import java.util.List; -import java.util.ListIterator; -import java.util.Map; - -public class Question { - private String id; - private double maxscore; - private List> params; - private String title; - private String type; - private String desc; - - public Question(String id, double maxscore, List> params, String title, String type, String desc) { - this.id = id; - this.maxscore = maxscore; - this.params = params; - this.title = title; - this.type = type; - this.desc = desc; - } - - public String getId() { - return id; - } - - public double getMaxScore() { - return maxscore; - } - - - public List> getParams() { - ListIterator> - iterator = params.listIterator(); - while (iterator.hasNext()) { - Map values = iterator.next(); - for (Map.Entry entry : values.entrySet()) { - if (null!= entry.getValue() && (!(entry.getValue() instanceof String))) - entry.setValue(new Gson().toJson(entry.getValue())); - } - } - return params; - } - - public String getTitle() { - return title; - } - - - public String getType() { - return type; - } - - - public String getDesc() { - return desc; - } - - -} diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/QuestionData.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/QuestionData.java deleted file mode 100644 index 99e679b707..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/QuestionData.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; - -import java.util.List; -import java.util.ListIterator; -import java.util.Map; - -public class QuestionData { - - - private List> resvalues; - private double duration; - private double score; - private Question item; - private long assessts; - - public QuestionData(List> resvalues, double duration, double score, Question item) { - - this.resvalues = resvalues; - this.duration = duration; - this.score = score; - this.item = item; - } - - - public List> getResvalues() { - ListIterator> - iterator = resvalues.listIterator(); - while (iterator.hasNext()) { - Map values = iterator.next(); - for (Map.Entry entry : values.entrySet()) { - if (null!= entry.getValue() && (!(entry.getValue() instanceof String))) - entry.setValue(new Gson().toJson(entry.getValue())); - } - } - return resvalues; - } - - public double getDuration() { - return duration; - } - - public double getScore() { - return score; - } - - public Question getItem() { - return item; - } - - public long getEts() { - return assessts; - } - - public void setEts(long ets) { - this.assessts = ets; - } - -} - - diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/service/AssessmentAggregatorService.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/service/AssessmentAggregatorService.java deleted file mode 100644 index 9fa13140a6..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/service/AssessmentAggregatorService.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.datastax.driver.core.Row; -import com.datastax.driver.core.UDTValue; -import com.datastax.driver.core.exceptions.DriverException; -import com.google.gson.Gson; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Aggregate; -import org.ekstep.ep.samza.domain.BatchEvent; -import org.ekstep.ep.samza.domain.QuestionData; -import org.ekstep.ep.samza.task.AssessmentAggregatorConfig; -import org.ekstep.ep.samza.task.AssessmentAggregatorSink; -import org.ekstep.ep.samza.task.AssessmentAggregatorSource; -import org.ekstep.ep.samza.util.CassandraConnect; -import org.ekstep.ep.samza.util.DBUtil; -import org.joda.time.DateTime; - -import java.util.*; - -public class AssessmentAggregatorService { - - private static Logger LOGGER = new Logger(AssessmentAggregatorService.class); - private DBUtil dbUtil; - private Comparator byEts = (QuestionData o1, QuestionData o2) -> Long.compare(o2.getEts(), o1.getEts()); - - public AssessmentAggregatorService(CassandraConnect cassandraConnect, AssessmentAggregatorConfig config) { - this.dbUtil = new DBUtil(cassandraConnect, config); - } - - public void process(AssessmentAggregatorSource source, AssessmentAggregatorSink sink) throws Exception { - try { - BatchEvent batchEvent = source.getEvent(); - Row assessment = dbUtil.getAssessmentFromDB(batchEvent); - if (null != assessment) { - sink.incDBHits(); - } - if (null != assessment) { - Long last_attempted_on = assessment.getTimestamp("last_attempted_on").getTime(); - if (batchEvent.assessmentEts() > last_attempted_on) { - saveAssessment(assessment, batchEvent); - sink.incDBUpdateCount(); - sink.batchSuccess(); - } else { - LOGGER.info(batchEvent.attemptId(), ": Batch Event older than last assessment time, skipping"); - sink.skip(batchEvent); - } - } else { - saveAssessment(assessment, batchEvent); - sink.incDBInsertCount(); - sink.batchSuccess(); - } - - } catch (DriverException ex) { - ex.printStackTrace(); - LOGGER.error("", "Exception while fetching from db : " + ex); - throw new DriverException(ex); - } catch (Exception ex) { - ex.printStackTrace(); - LOGGER.error("", "Failed to parse the batchEvent: ", ex); - sink.fail(source.getMessage().toString()); - } - } - - public void saveAssessment(Row assessment, BatchEvent batchEvent) { - Long createdOn = null != assessment ? assessment.getTimestamp("created_on").getTime() : new DateTime().getMillis(); - Aggregate assess = getAggregateData(batchEvent, createdOn); - dbUtil.updateAssessmentToDB(batchEvent, assess, createdOn); - LOGGER.info("", " Successfully Aggregated the batch event - batchid: " + batchEvent.batchId() - + " ,userid: " + batchEvent.userId() + " ,couserid: " + batchEvent.courseId() - + " ,contentid: " + batchEvent.contentId()); - } - - public Aggregate getAggregateData(BatchEvent batchEvent, Long createdOn) { - double totalMaxScore = 0; - double totalScore = 0; - List questionsList = new ArrayList<>(); - TreeSet questionSet = new TreeSet(byEts); - HashMap checkDuplicate = new HashMap(); - for (Map event : batchEvent.assessEvents()) { - if (event.containsKey("edata")) { - QuestionData questionData = new Gson().fromJson(new Gson().toJson(event.get("edata")), QuestionData.class); - questionData.setEts(((Number) event.get("ets")).longValue()); - questionSet.add(questionData); - } - } - for (QuestionData questionData : questionSet) { - if (!checkDuplicate.containsKey(questionData.getItem().getId())) { - totalScore += questionData.getScore(); - totalMaxScore += questionData.getItem().getMaxScore(); - questionsList.add(dbUtil.getQuestion(questionData)); - checkDuplicate.put(questionData.getItem().getId(), ""); - } - } - return new Aggregate(totalScore, totalMaxScore, questionsList); - } - -} diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorConfig.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorConfig.java deleted file mode 100644 index b3157feac6..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; - -public class AssessmentAggregatorConfig { - - private final String JOB_NAME = "AssessmentAggregator"; - private String failedTopic; - private final String coursesKeyspace; - private final String assessementTable; - private final String assessmentQuestionUDT; - private final String cassandraHost; - private final int cassandraPort; - - public AssessmentAggregatorConfig(Config config) { - - coursesKeyspace = config.get("middleware.cassandra.courses_keyspace", "sunbird_courses"); - assessementTable = config.get("middleware.cassandra.aggregator_table", "assessment_aggregator"); - assessmentQuestionUDT = config.get("middleware.cassandra.question_type", "question"); - failedTopic = config.get("output.failed.topic.name", "telemetry.assess.failed"); - cassandraHost = config.get("middleware.cassandra.host", "127.0.0.1"); - cassandraPort = config.getInt("middleware.cassandra.port", 9042); - } - - public String jobName() { - return JOB_NAME; - } - - public String failedTopic() { - return failedTopic; - } - - public String getCoursesKeyspace() { - return coursesKeyspace; - } - - public String getAssessementTable() { - return assessementTable; - } - - public String getAssessmentQuestionUDT() { - return assessmentQuestionUDT; - } - - public String getCassandraHost() { return cassandraHost; } - - public int getCassandraPort() { return cassandraPort; } - -} diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSink.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSink.java deleted file mode 100644 index 939e94a262..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSink.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.BatchEvent; - -public class AssessmentAggregatorSink extends BaseSink { - - private AssessmentAggregatorConfig config; - - public AssessmentAggregatorSink(MessageCollector collector, JobMetrics metrics, - AssessmentAggregatorConfig config) { - super(collector, metrics); - this.config = config; - } - - public void batchSuccess() { - metrics.incBatchSuccessCounter(); - } - - public void success() { - metrics.incSuccessCounter(); - } - - public void skip(BatchEvent batchEvent) { - batchEvent.markSkipped(batchEvent); - toTopic(config.failedTopic(), batchEvent.attemptId(), batchEvent.getJson()); - metrics.incSkippedCounter(); - } - - public void fail(String message) { - toTopic(config.failedTopic(), null, message); - metrics.incFailedCounter(); - } - - public void incDBHits() { - metrics.incDBHitCount(); - } - - public void incDBInsertCount() { - metrics.incDBInsertCount(); - } - - public void incDBUpdateCount() { - metrics.incDBUpdateCount(); - } -} - diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSource.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSource.java deleted file mode 100644 index 8e0cf70671..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSource.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.gson.Gson; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.ekstep.ep.samza.domain.BatchEvent; - -import java.util.Map; - -public class AssessmentAggregatorSource { - private IncomingMessageEnvelope envelope; - - public AssessmentAggregatorSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Object getMessage() - { - return envelope.getMessage(); - } - - @SuppressWarnings("unchecked") - public BatchEvent getEvent() { - String message = (String) envelope.getMessage(); - Map jsonMap = (Map) new Gson().fromJson(message, Map.class); - return new BatchEvent(jsonMap); - } - -} - diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorTask.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorTask.java deleted file mode 100644 index f7770d3be6..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorTask.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.*; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.AssessmentAggregatorService; -import org.ekstep.ep.samza.util.CassandraConnect; - -public class AssessmentAggregatorTask extends BaseSamzaTask { - - private JobMetrics metrics; - private AssessmentAggregatorService service; - private AssessmentAggregatorConfig config; - - public AssessmentAggregatorTask(Config config, TaskContext taskContext, CassandraConnect cassandraConnect) { - init(config, taskContext, cassandraConnect); - } - - public AssessmentAggregatorTask() { - - } - - @Override - public void init(Config config, TaskContext context) { - init(config, context, null); - } - - private void init(Config config, TaskContext context, CassandraConnect cassandraConnect) { - - this.config = new AssessmentAggregatorConfig(config); - this.metrics = new JobMetrics(context, this.config.jobName()); - cassandraConnect = null != cassandraConnect ? cassandraConnect : new CassandraConnect(this.config.getCassandraHost(), this.config.getCassandraPort()); - this.service = new AssessmentAggregatorService(cassandraConnect, this.config); - this.initTask(config, this.metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, - TaskCoordinator taskCoordinator) throws Exception { - AssessmentAggregatorSource source = new AssessmentAggregatorSource(envelope); - AssessmentAggregatorSink sink = new AssessmentAggregatorSink(collector, metrics, config); - service.process(source, sink); - } - - public JobMetrics getJobMetrics() { - return this.metrics; - } -} diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/util/DBUtil.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/util/DBUtil.java deleted file mode 100644 index 49ff425ad3..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/util/DBUtil.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.ekstep.ep.samza.util; - -import com.datastax.driver.core.Row; -import com.datastax.driver.core.UDTValue; -import com.datastax.driver.core.UserType; -import com.datastax.driver.core.querybuilder.Insert; -import com.datastax.driver.core.querybuilder.QueryBuilder; -import org.ekstep.ep.samza.domain.Aggregate; -import org.ekstep.ep.samza.domain.BatchEvent; -import org.ekstep.ep.samza.domain.QuestionData; -import org.ekstep.ep.samza.task.AssessmentAggregatorConfig; -import org.joda.time.DateTime; - -import java.math.BigDecimal; -import java.sql.Timestamp; - - -public class DBUtil { - private CassandraConnect cassandraConnect; - private AssessmentAggregatorConfig config; - private UserType questionType; - - public DBUtil(CassandraConnect cassandraConnect, AssessmentAggregatorConfig config) { - this.cassandraConnect = cassandraConnect; - this.config = config; - questionType = cassandraConnect.getUDTType(config.getCoursesKeyspace(), config.getAssessmentQuestionUDT()); - } - - public UDTValue getQuestion(QuestionData questionData) { - return questionType.newValue().setString("id", questionData.getItem().getId()) - .setDouble("max_score", questionData.getItem().getMaxScore()) - .setDouble("score", questionData.getScore()) - .setString("type", questionData.getItem().getType()) - .setString("title", questionData.getItem().getTitle()) - .setList("resvalues", questionData.getResvalues()) - .setList("params", questionData.getItem().getParams()) - .setString("description", questionData.getItem().getDesc()) - .setDecimal("duration", BigDecimal.valueOf(questionData.getDuration())) - .setTimestamp("assess_ts", new Timestamp(questionData.getEts())); - - } - - public void updateAssessmentToDB(BatchEvent batchEvent, Aggregate aggregate, Long createdOn) { - - Insert query = QueryBuilder.insertInto(config.getCoursesKeyspace(), config.getAssessementTable()) - .value("course_id", batchEvent.courseId()) - .value("batch_id", batchEvent.batchId()).value("user_id", batchEvent.userId()) - .value("content_id", batchEvent.contentId()).value("attempt_id", batchEvent.attemptId()) - .value("updated_on", new DateTime().getMillis()).value("created_on", createdOn) - .value("last_attempted_on", batchEvent.assessmentEts()).value("total_score", aggregate.getTotalScore()) - .value("total_max_score", aggregate.getTotalMaxScore()).value("question", aggregate.getQuestionsList()) - .value("grand_total", aggregate.getGrandTotal()); - - cassandraConnect.upsert(query); - } - - - public Row getAssessmentFromDB(BatchEvent event) { - String query = QueryBuilder.select("last_attempted_on", "created_on").from(config.getCoursesKeyspace(), config.getAssessementTable()) - .where(QueryBuilder.eq("attempt_id", event.attemptId())).and(QueryBuilder.eq("batch_id", event.batchId())) - .and(QueryBuilder.eq("user_id", event.userId())).and(QueryBuilder.eq("content_id", event.contentId())) - .and(QueryBuilder.eq("course_id", event.courseId())).toString(); - return cassandraConnect.findOne(query); - } -} diff --git a/data-pipeline/assessment-aggregator/src/main/resources/log4j.xml b/data-pipeline/assessment-aggregator/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/assessment-aggregator/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index 25d08ac64f..0000000000 --- a/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,1525 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; - -import java.util.Map; - -public class EventFixture { - - public static final String BATCH_ASSESS_EVENT = "{\n" + - " \"assessmentTs\": 1568891729576,\n" + - " \"batchId\": \"012846671379595264119\",\n" + - " \"courseId\": \"do_2128415652377067521127\",\n" + - " \"userId\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" + - " \"attemptId\": \"8cd87e24df268ad09a8b0060c0a40271\",\n" + - " \"contentId\": \"do_212686723743318016173\",\n" + - " \"events\": [\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1568891735461,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:db00a858fec1b8796c62f224874c7edf\",\n" + - " \"actor\": {\n" + - " \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124784842112040965\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" + - " \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128415652377067521127\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"012846671379595264119\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124784842112040965\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_212686723743318016173\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128415652377067521127\",\n" + - " \"l2\": \"do_2128415660716359681128\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124784842112040965\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"801ae93c-8807-4be5-8853-dd49362d8776\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"What is the Full form of WHO..?\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [],\n" + - " \"duration\": 2\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1568891738245,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:135815023ec32a430632ba5d7f84fe18\",\n" + - " \"actor\": {\n" + - " \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124784842112040965\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" + - " \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128415652377067521127\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"012846671379595264119\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124784842112040965\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_212686723743318016173\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128415652377067521127\",\n" + - " \"l2\": \"do_2128415660716359681128\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124784842112040965\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"801ae93c-8807-4be5-8853-dd49362d8776\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"What is the Full form of WHO..?\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 4\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1568891747395,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:6ba5953669ea86e8f85759d3e7f5998b\",\n" + - " \"actor\": {\n" + - " \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124784842112040965\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" + - " \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128415652377067521127\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"012846671379595264119\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124784842112040965\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_212686723743318016173\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128415652377067521127\",\n" + - " \"l2\": \"do_2128415660716359681128\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124784842112040965\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"801ae93c-8807-4be5-8853-dd49362d8776\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"What is the Full form of WHO..?\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"Yes\",\n" + - " \"score\": 1,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 14\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1568891772964,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:018f01bf99288474860b630b513b9d0c\",\n" + - " \"actor\": {\n" + - " \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124784842112040965\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" + - " \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128415652377067521127\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"012846671379595264119\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124784842112040965\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_212686723743318016173\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128415652377067521127\",\n" + - " \"l2\": \"do_2128415660716359681128\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124784842112040965\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"2bc922e7-985e-486a-ae23-4ba9a1c67edc\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mtf\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"lhs\": \"[{\\\"1\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"1\\\\\\\"}\\\"},{\\\"2\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"2\\\\\\\"}\\\"},{\\\"3\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"3\\\\\\\"}\\\"}]\"\n" + - " },\n" + - " {\n" + - " \"rhs\": \"[{\\\"1\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"2\\\\\\\"}\\\"},{\\\"2\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"3\\\\\\\"}\\\"},{\\\"3\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"1\\\\\\\"}\\\"}]\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"lhs\\\":[\\\"1\\\",\\\"2\\\",\\\"3\\\"],\\\"rhs\\\":[\\\"3\\\",\\\"1\\\",\\\"2\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"MTF 3\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 2,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0.33,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"lhs\": \"[{\\\"1\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"1\\\\\\\"}\\\"},{\\\"2\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"2\\\\\\\"}\\\"},{\\\"3\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"3\\\\\\\"}\\\"}]\"\n" + - " },\n" + - " {\n" + - " \"rhs\": \"[{\\\"1\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"3\\\\\\\"}\\\"},{\\\"2\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"2\\\\\\\"}\\\"},{\\\"3\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"1\\\\\\\"}\\\"}]\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 24\n" + - " }\n" + - " }\n" + - " ]\n" + - "}"; - - public static final String BATCH_ASSESS__OLDER_EVENT = "{\n" + - " \"courseId\": \"do_312712196780204032110117\",\n" + - " \"batchId\": \"01271220181664563270\",\n" + - " \"contentId\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"userId\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"assessmentTs\": 1567073236195,\n" + - " \"attemptId\": \"attempt1\",\n" + - " \"events\": [\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1567073236195,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"ASSESS:48d754770446994b98e64577683ada25\",\n" + - " \"actor\": {\n" + - " \"id\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.portal\",\n" + - " \"ver\": \"2.2.1\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"310f123e-2764-c784-803a-0ca871f2b651\",\n" + - " \"did\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"6b8617ccf0b4c35f3fe17ccdb71af908\",\n" + - " \"type\": \"ContentSession\"\n" + - " },\n" + - " {\n" + - " \"id\": \"01271220181664563270\",\n" + - " \"type\": \"batch\"\n" + - " },\n" + - " {\n" + - " \"id\": \"do_312712196780204032110117\",\n" + - " \"type\": \"course\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312592741863645184122936\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"3\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_3126430145280819201402\"\n" + - " }\n" + - " },\n" + - " \"tags\": [],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"do_312468066279276544217372\",\n" + - " \"maxscore\": 10,\n" + - " \"exlength\": 0,\n" + - " \"params\": [],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"TNXMATHS-STATISTICS 5\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"For any collection of n items ∑(χ - χ̅ )=\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"Yes\",\n" + - " \"score\": 5,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"∑χ\": \"true\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 9\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1567073236195,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"ASSESS:48d754770446994b98e64577683ada25\",\n" + - " \"actor\": {\n" + - " \"id\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.portal\",\n" + - " \"ver\": \"2.2.1\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"310f123e-2764-c784-803a-0ca871f2b651\",\n" + - " \"did\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"6b8617ccf0b4c35f3fe17ccdb71af908\",\n" + - " \"type\": \"ContentSession\"\n" + - " },\n" + - " {\n" + - " \"id\": \"01271220181664563270\",\n" + - " \"type\": \"batch\"\n" + - " },\n" + - " {\n" + - " \"id\": \"do_312712196780204032110117\",\n" + - " \"type\": \"course\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312592741863645184122936\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"3\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_3126430145280819201402\"\n" + - " }\n" + - " },\n" + - " \"tags\": [],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"do_312468066279276544217372\",\n" + - " \"maxscore\": 1,\n" + - " \"exlength\": 0,\n" + - " \"params\": [],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"TNXMATHS-STATISTICS 5\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"For any collection of n items ∑(χ - χ̅ )=\"\n" + - " },\n" + - " \"index\": 2,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"∑χ\": \"true\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 9\n" + - " }\n" + - " }\n" + - " ]\n" + - "}"; - - public static final String BATCH_ASSESS_FAIL_EVENT = "{\n" + - " \"courseId\": \"do_312712196780204032110117\",\n" + - " \"batchId\": \"01271220181664563270\",\n" + - " \"contentId\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"userId\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"assessmentTs\": 1567073236195,\n" + - " \"attemptId\": \"attempt1\",\n" + - " \"events\": [\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1567687351000,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"ASSESS:48d754770446994b98e64577683ada25\",\n" + - " \"actor\": {\n" + - " \"id\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"type\": \"User\"\n" + - " }\n" + - " \"context\": {\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.portal\",\n" + - " \"ver\": \"2.2.1\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"310f123e-2764-c784-803a-0ca871f2b651\",\n" + - " \"did\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"6b8617ccf0b4c35f3fe17ccdb71af908\",\n" + - " \"type\": \"ContentSession\"\n" + - " },\n" + - " {\n" + - " \"id\": \"01271220181664563270\",\n" + - " \"type\": \"batch\"\n" + - " },\n" + - " {\n" + - " \"id\": \"do_312712196780204032110117\",\n" + - " \"type\": \"course\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312592741863645184122936\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"3\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_3126430145280819201402\"\n" + - " }\n" + - " },\n" + - " \"tags\": [],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"do_312468066279276544217372\",\n" + - " \"maxscore\": 10,\n" + - " \"exlength\": 0,\n" + - " \"params\": [],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"TNXMATHS-STATISTICS 5\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"For any collection of n items ∑(χ - χ̅ )=\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"Yes\",\n" + - " \"score\": 5,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"∑χ\": \"true\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 9\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1567073236195,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"ASSESS:48d754770446994b98e64577683ada25\",\n" + - " \"actor\": {\n" + - " \"id\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.portal\",\n" + - " \"ver\": \"2.2.1\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"310f123e-2764-c784-803a-0ca871f2b651\",\n" + - " \"did\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"6b8617ccf0b4c35f3fe17ccdb71af908\",\n" + - " \"type\": \"ContentSession\"\n" + - " },\n" + - " {\n" + - " \"id\": \"01271220181664563270\",\n" + - " \"type\": \"batch\"\n" + - " },\n" + - " {\n" + - " \"id\": \"do_312712196780204032110117\",\n" + - " \"type\": \"course\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312592741863645184122936\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"3\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_3126430145280819201402\"\n" + - " }\n" + - " },\n" + - " \"tags\": [],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"do_312468066279276544217372\",\n" + - " \"maxscore\": 1,\n" + - " \"exlength\": 0,\n" + - " \"params\": [],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"TNXMATHS-STATISTICS 5\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"For any collection of n items ∑(χ - χ̅ )=\"\n" + - " },\n" + - " \"index\": 2,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"∑χ\": \"true\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 9\n" + - " }\n" + - " }\n" + - " ]\n" + - "}"; - - public static final String BATCH_DUPLICATE_QUESTION_EVENT ="{\n" + - " \"assessmentTs\": 1569304757079,\n" + - " \"batchId\": \"01284169026368307244\",\n" + - " \"courseId\": \"do_2128410273679114241112\",\n" + - " \"userId\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" + - " \"attemptId\": \"90e1a0d12542806389a1a52aaf1fc622\",\n" + - " \"contentId\": \"do_2128373396098744321673\",\n" + - " \"events\": [\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1569304758743,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:5b2e689446886f3cee13de44fec8c02f\",\n" + - " \"actor\": {\n" + - " \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124511394914140160\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" + - " \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128410273679114241112\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"01284169026368307244\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124511394914140160\",\n" + - " \"l2\": \"01245115225042944040\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_2128373396098744321673\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128410273679114241112\",\n" + - " \"l2\": \"do_2128410274404106241113\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124511394914140160\",\n" + - " \"01245115225042944040\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"b7886294-95dd-4c83-91e3-7b67e82aaab2\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"3\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 2\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1569304761105,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:0cba50019b880fc064a343a6d01d3d1a\",\n" + - " \"actor\": {\n" + - " \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124511394914140160\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" + - " \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128410273679114241112\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"01284169026368307244\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124511394914140160\",\n" + - " \"l2\": \"01245115225042944040\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_2128373396098744321673\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128410273679114241112\",\n" + - " \"l2\": \"do_2128410274404106241113\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124511394914140160\",\n" + - " \"01245115225042944040\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"b7886294-95dd-4c83-91e3-7b67e82aaab2\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"3\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 4\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1569304763323,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:ace9f977fdb8254e709180777ff81ba6\",\n" + - " \"actor\": {\n" + - " \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124511394914140160\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" + - " \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128410273679114241112\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"01284169026368307244\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124511394914140160\",\n" + - " \"l2\": \"01245115225042944040\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_2128373396098744321673\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128410273679114241112\",\n" + - " \"l2\": \"do_2128410274404106241113\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124511394914140160\",\n" + - " \"01245115225042944040\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"b7886294-95dd-4c83-91e3-7b67e82aaab2\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"3\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 6\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1569304765515,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:b4b19e35aed5b32cf240650aa09ec558\",\n" + - " \"actor\": {\n" + - " \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124511394914140160\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" + - " \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128410273679114241112\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"01284169026368307244\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124511394914140160\",\n" + - " \"l2\": \"01245115225042944040\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_2128373396098744321673\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128410273679114241112\",\n" + - " \"l2\": \"do_2128410274404106241113\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124511394914140160\",\n" + - " \"01245115225042944040\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"b7886294-95dd-4c83-91e3-7b67e82aaab2\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"3\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"Yes\",\n" + - " \"score\": 1,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 8\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1569304767576,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:366e140cf5fddf850d1e548644a35729\",\n" + - " \"actor\": {\n" + - " \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124511394914140160\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" + - " \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128410273679114241112\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"01284169026368307244\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124511394914140160\",\n" + - " \"l2\": \"01245115225042944040\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_2128373396098744321673\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128410273679114241112\",\n" + - " \"l2\": \"do_2128410274404106241113\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124511394914140160\",\n" + - " \"01245115225042944040\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"759b8138-4c47-4724-8cd6-cae6a7d1ad18\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"descriptive statistic\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"probability function\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"variance\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"random variable\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"4\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"A numerical description of the outcome of an experiment is called a\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 2,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"variance\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 1\n" + - " }\n" + - " },\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1569304769433,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:834b83c5205b657a208e552d5964a872\",\n" + - " \"actor\": {\n" + - " \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124511394914140160\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" + - " \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128410273679114241112\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"01284169026368307244\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124511394914140160\",\n" + - " \"l2\": \"01245115225042944040\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_2128373396098744321673\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128410273679114241112\",\n" + - " \"l2\": \"do_2128410274404106241113\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124511394914140160\",\n" + - " \"01245115225042944040\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"759b8138-4c47-4724-8cd6-cae6a7d1ad18\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"descriptive statistic\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"probability function\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"variance\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"random variable\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"4\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"A numerical description of the outcome of an experiment is called a\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 2,\n" + - " \"pass\": \"Yes\",\n" + - " \"score\": 1,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"random variable\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 3\n" + - " }\n" + - " }\n" + - " ]\n" + - "}"; - public static final String QUESTION_EVENT_RES_VALUES = "{\n" + - " \"assessmentTs\": 1578663044164,\n" + - " \"userId\": \"50a9e3fc-d047-4fa5-a37b-67501b8933db\",\n" + - " \"contentId\": \"do_3129323935897108481169\",\n" + - " \"courseId\": \"do_3129323995959541761169\",\n" + - " \"batchId\": \"0129324118211215362\",\n" + - " \"attemptId\": \"702ae8b81e37c94448d1fa117678d68c\",\n" + - " \"events\": [\n" + - " {\n" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1578663044164,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"ASSESS:0b5c1d06920337df9d662b69d53f48bf\",\n" + - " \"actor\": {\n" + - " \"id\": \"50a9e3fc-d047-4fa5-a37b-67501b8933db\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0126684405014528002\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.app\",\n" + - " \"ver\": \"2.6.203\",\n" + - " \"pid\": \"sunbird.app.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"11059dcc-0a69-4d92-90b3-c1eb97f5f93b\",\n" + - " \"did\": \"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"83f8c374dc0e4d91ec4c48d6e5d10b4c\",\n" + - " \"type\": \"AttemptId\"\n" + - " },\n" + - " {\n" + - " \"id\": \"streaming\",\n" + - " \"type\": \"PlayerLaunch\"\n" + - " },\n" + - " {\n" + - " \"id\": \"6a7809b4eccf75ae62b18c224a4cfeb5\",\n" + - " \"type\": \"ContentSession\"\n" + - " },\n" + - " {\n" + - " \"id\": \"48e8e20e8c6fbc660f4ca317497fa7a6\",\n" + - " \"type\": \"PlaySession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0126684405014528002\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_3129323935897108481169\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_3129323995959541761169\",\n" + - " \"l2\": \"do_31293239986759270411495\"\n" + - " }\n" + - " },\n" + - " \"tags\": [],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"QMCQ02115\",\n" + - " \"maxscore\": 1,\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"item\": {\n" + - " \"keywords\": [\n" + - " \"mdd\"\n" + - " ],\n" + - " \"subject\": \"Mathematics\",\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"language\": [\n" + - " \"English\"\n" + - " ],\n" + - " \"type\": \"ftb\",\n" + - " \"editorState\": null,\n" + - " \"body\": null,\n" + - " \"question_audio\": \"\",\n" + - " \"gradeLevel\": [\n" + - " \"Class 4\"\n" + - " ],\n" + - " \"appId\": \"prod.diksha.app\",\n" + - " \"used_for\": \"worksheet\",\n" + - " \"model\": {\n" + - " \"hintMsg\": \"Observe the key value and find the number of aircrafts which flies in the morning.\",\n" + - " \"numericLangId\": \"en\",\n" + - " \"langId\": \"en\",\n" + - " \"variables\": {\n" + - " \"$aircraft1\": \"random(100,100)\",\n" + - " \"$aircraft2\": \"random(70,70)\",\n" + - " \"$aircraft3\": \"random(80,80)\",\n" + - " \"$aircraft4\": \"random(30,30)\"\n" + - " },\n" + - " \"dropDowns\": [\n" + - " {\n" + - " \"identifier\": 1,\n" + - " \"options\": [\n" + - " {\n" + - " \"text\": \"OPT_1_0\",\n" + - " \"mh\": null,\n" + - " \"mmc\": [\n" + - " \"DH277\",\n" + - " \"DH278\"\n" + - " ],\n" + - " \"answer\": true,\n" + - " \"image\": null\n" + - " },\n" + - " {\n" + - " \"text\": \"OPT_1_1\",\n" + - " \"mh\": \"MH_1_1\",\n" + - " \"mmc\": [\n" + - " \"DH277\",\n" + - " \"DH278\"\n" + - " ],\n" + - " \"answer\": false,\n" + - " \"image\": null\n" + - " },\n" + - " {\n" + - " \"text\": \"OPT_1_2\",\n" + - " \"mh\": \"MH_1_2\",\n" + - " \"mmc\": [\n" + - " \"DH277\",\n" + - " \"DH278\"\n" + - " ],\n" + - " \"answer\": false,\n" + - " \"image\": null\n" + - " },\n" + - " {\n" + - " \"text\": \"OPT_1_3\",\n" + - " \"mh\": \"MH_1_3\",\n" + - " \"mmc\": [\n" + - " \"DH277\",\n" + - " \"DH278\"\n" + - " ],\n" + - " \"answer\": false,\n" + - " \"image\": null\n" + - " }\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"identifier\": 2,\n" + - " \"options\": []\n" + - " },\n" + - " {\n" + - " \"identifier\": 3,\n" + - " \"options\": []\n" + - " },\n" + - " {\n" + - " \"identifier\": 4,\n" + - " \"options\": []\n" + - " }\n" + - " ],\n" + - " \"variablesProcessed\": true\n" + - " },\n" + - " \"state\": \"Verified\",\n" + - " \"identifier\": \"QMCQ02115\",\n" + - " \"level\": 2,\n" + - " \"author\": \"funtoot\",\n" + - " \"consumerId\": \"1762141d-8681-458b-9bd1-c6af98c0d989\",\n" + - " \"solutions\": null,\n" + - " \"portalOwner\": \"562\",\n" + - " \"version\": 1,\n" + - " \"i18n\": {\n" + - " \"en\": {\n" + - " \"HINT\": \"Hint\",\n" + - " \"MICROHINT\": \"Micro Hint\",\n" + - " \"SOLUTION\": \"Solution\",\n" + - " \"HELP\": \"Help\",\n" + - " \"REDUCE_FRACTION\": \"Check the solution again. Remove the common factors in the fraction and reduce it to its lowest term.\",\n" + - " \"MIXED_FRACTION\": \"Convert the mixed fraction to an improper fraction.\",\n" + - " \"IMPROPER_FRACTION\": \"Convert the improper fraction to an mixed fraction.\",\n" + - " \"LIKEFRACTIONADDITION_NUMERATOR\": \"Add the numerator of the given fractions correctly.\",\n" + - " \"LIKEFRACTIONADDITION_DENOMINATOR\": \"For addition of like fractions the denominator remains the same.\",\n" + - " \"LIKEFRACTIONADDITION_FULL\": \"Check the addition of the numerator. The denominator remains the same.\",\n" + - " \"UNLIKEFRACTIONADDITION_NUMERATOR\": \"First convert the unlike fractions to like fractions. Then add the numerator of the like fractions.\",\n" + - " \"UNLIKEFRACTIONADDITION_DENOMINATOR\": \"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.\",\n" + - " \"UNLIKEFRACTIONADDITION_FULL\": \"Convert the given fractions to like fractions and then carefully do the addition.\",\n" + - " \"LIKEFRACTIONSUBTRACTION_NUMERATOR\": \"Subtract the numerator of the given fractions correctly.\",\n" + - " \"LIKEFRACTIONSUBTRACTION_DENOMINATOR\": \"For subtraction of like fractions the denominator remains the same as that of the question.\",\n" + - " \"LIKEFRACTIONSUBTRACTION_FULL\": \"Subtract the numerator of the fractions. Denominator remains the same.\",\n" + - " \"UNLIKEFRACTIONSUBTRACTION_NUMERATOR\": \"First convert the unlike fractions to like fractions. Then subtract the numerator of the like fractions.\",\n" + - " \"UNLIKEFRACTIONSUBTRACTION_DENOMINATOR\": \"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.\",\n" + - " \"UNLIKEFRACTIONSUBTRACTION_FULL\": \"Convert the given fractions to like fractions and then carefully do the subtraction.\",\n" + - " \"LIKEMIXEDFRACTIONADDITION_WHOLE\": \"Add the whole number part of both the fractions carefully. Check for the conversion of the improper fraction to proper fractions in the solution.\",\n" + - " \"LIKEMIXEDFRACTIONADDITION_NUMERATOR\": \"Add the numerator of the like fractions. If the fractional part is an improper fraction, then convert it to a proper fraction.\",\n" + - " \"LIKEMIXEDFRACTIONADDITION_DENOMINATOR\": \"The denominator of the solution fraction does not change for like fractions.\",\n" + - " \"LIKEMIXEDFRACTIONADDITION_FULL\": \"Convert the mixed fractions to improper fractions and then add them. The final solution should again be as a mixed fraction.\",\n" + - " \"LIKEMIXEDFRACTIONSUBTRACTION_WHOLE\": \"Subtract the whole number part of both the fractions carefully.\",\n" + - " \"LIKEMIXEDFRACTIONSUBTRACTION_NUMERATOR\": \"Subtract the numerator of the like fractions.\",\n" + - " \"LIKEMIXEDFRACTIONSUBTRACTION_DENOMINATOR\": \"The denominator of the solution fraction does not change for like fractions.\",\n" + - " \"LIKEMIXEDFRACTIONSUBTRACTION_FULL\": \"Convert the mixed fractions to improper fractions and then subtract them. The final solution should again be as a mixed fraction.\",\n" + - " \"WHOLEANDPROPERFRACTIONADDITION_WHOLE\": \"The integer added becomes the whole number part of the mixed fraction.\",\n" + - " \"WHOLEANDPROPERFRACTIONADDITION_NUMERATOR\": \"The numerator of the solution fraction remains the same. \",\n" + - " \"WHOLEANDPROPERFRACTIONADDITION_DENOMINATOR\": \"The denominator of the solution fraction remains the same as given in the problem.\",\n" + - " \"WHOLEANDPROPERFRACTIONADDITION_FULL\": \"The integer and the proper fraction when added becomes the mixed fraction.\",\n" + - " \"WHOLEANDPROPERFRACTIONSUBTRACTION_WHOLE\": \"Convert the whole number to an equivalent fraction. Then find the difference between the like fractions.\",\n" + - " \"WHOLEANDPROPERFRACTIONSUBTRACTION_NUMERATOR\": \"First find the least common factor and convert them to difference between like fractions. Then subtract the numerators carefully.\",\n" + - " \"WHOLEANDPROPERFRACTIONSUBTRACTION_DENOMINATOR\": \"The denominator of the solution fraction remains the same as given in the problem. \",\n" + - " \"WHOLEANDPROPERFRACTIONSUBTRACTION_FULL\": \"When the proper fraction is subtracted from the integer becomes the mixed fraction.\",\n" + - " \"WHOLEANDFRACTIONMULTIPLICATION_NUMERATOR\": \"Multiply the numerator and the whole number carefully.\",\n" + - " \"WHOLEANDFRACTIONMULTIPLICATION_DENOMINATOR\": \"Multiply the denominators of the fractions carefully. \",\n" + - " \"WHOLEANDFRACTIONMULTIPLICATION_FULL\": \"Multiply the integer with the fraction.\",\n" + - " \"FRACTIONMULTIPLICATION_NUMERATOR\": \"Multiply the numerators of the fractions carefully.\",\n" + - " \"FRACTIONMULTIPLICATION_DENOMINATOR\": \"Multiply the denominators of the fractions carefully. \",\n" + - " \"FRACTIONMULTIPLICATION_FULL\": \"Multiply the numerators and denominators of the fractions.\",\n" + - " \"FRACTIONDIVISION\": \"First find the reciprocal of the divisor. Then multiply the fraction with the reciprocal of the divisor to get the solution.\",\n" + - " \"SELECT\": \"Select\",\n" + - " \"EXPRESSIONS\": \"$aircraft1=random(100,100)\\n$aircraft2=random(70,70)\\n$aircraft3=random(80,80)\\n$aircraft4=random(30,30)\",\n" + - " \"NO_HINT\": \"There is no hint for this question.\",\n" + - " \"NO_ANSWER\": \"Please answer.\",\n" + - " \"SOLUTION_ID\": \"a. The tallest bar shows the subject which is most liked by the students i.e., Social Studies. 40 students like Social Studiesb. There are 5 subjects mentioned in the graph, namely English, Mathematics, Hindi, Social Studies, and General Science.c. The shortest bar represents the subject which is least liked by the students i.e., Mathematics. Only 10 students like Mathematics.d. Number of students who like Hindi = 20Number of students who like Mathematics = 10We can clearly see that the number of students who like Hindi is double that of Mathematics. Hence the statement is true.\",\n" + - " \"HINT_ID\": \"Observe the key value and find the number of aircrafts which flies in the morning.\",\n" + - " \"QUESTION_TEXT\": \"The pictograph represents the number of aircrafts flying from India to foreign countries at different timings. Airlines decided to fly $aircraft1 aircrafts in the morning. How many aircrafts should they fly more? __1__\",\n" + - " \"MH_1_3\": \"Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.\",\n" + - " \"OPT_1_3\": \"$aircraft1\",\n" + - " \"MH_1_2\": \"According to the pictograph the number of aircrafts which flies in the afternoon is $aircraft3. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.\",\n" + - " \"OPT_1_2\": \"$aircraft3\",\n" + - " \"MH_1_1\": \"According to the pictograph the number of aircrafts which flies in the morning is $aircraft2. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.\",\n" + - " \"OPT_1_1\": \"$aircraft2\",\n" + - " \"OPT_1_0\": \"$aircraft4\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"mdd\"\n" + - " ],\n" + - " \"concepts\": [\n" + - " \"C421\"\n" + - " ],\n" + - " \"grade\": [\n" + - " \"4\"\n" + - " ],\n" + - " \"domain\": \"Numeracy\",\n" + - " \"name\": \"QMCQ02115\",\n" + - " \"bloomsTaxonomyLevel\": \"Understand\",\n" + - " \"status\": \"Live\",\n" + - " \"itemType\": \"UNIT\",\n" + - " \"code\": \"QMCQ02115\",\n" + - " \"qtype\": \"mdd\",\n" + - " \"qlevel\": \"MEDIUM\",\n" + - " \"flags\": \"{\\\"isZoomable\\\":true}\",\n" + - " \"questionImage\": \"org.ekstep.funtoot.QMCQ02115.image.9074280266192281\",\n" + - " \"media\": [\n" + - " {\n" + - " \"id\": \"org.ekstep.funtoot.QMCQ02115.image.9074280266192281\",\n" + - " \"src\": \"https://ekstep-public-prod.s3-ap-south-1.amazonaws.com/content/org.ekstep.funtoot.qmcq02115.image.9886103695997586/artifact/org.ekstep.funtoot.qmcq02115.image.9886103695997586_1514092203316.png\",\n" + - " \"type\": \"image\"\n" + - " }\n" + - " ],\n" + - " \"title\": \"\",\n" + - " \"qid\": \"QMCQ02115\",\n" + - " \"createdOn\": \"2017-12-24T05:10:03.956+0000\",\n" + - " \"qindex\": 1,\n" + - " \"lastUpdatedOn\": \"2018-08-16T14:08:14.323+0000\",\n" + - " \"subLevel\": \"\",\n" + - " \"question\": \"QUESTION_TEXT\",\n" + - " \"versionKey\": \"1534428494323\",\n" + - " \"framework\": \"NCF\",\n" + - " \"answer\": {},\n" + - " \"max_score\": 5,\n" + - " \"sublevel\": 3,\n" + - " \"template_id\": \"org.ekstep.funtoot.template.01\",\n" + - " \"category\": \"MCQ\",\n" + - " \"isSelected\": true,\n" + - " \"$$hashKey\": \"object:1540\",\n" + - " \"template\": \"funtoot.template.01\",\n" + - " \"maxAttempts\": 2,\n" + - " \"curAttempt\": 0\n" + - " },\n" + - " \"config\": {\n" + - " \"count\": 1,\n" + - " \"selectedConfig\": {},\n" + - " \"title\": \"\",\n" + - " \"type\": \"items\",\n" + - " \"var\": \"item\"\n" + - " }\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [\n" + - " null\n" + - " ],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"Yes\",\n" + - " \"score\": 1,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"1\": \"random(30,30)\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 6\n" + - " }\n" + - " }\n" + - " ]\n" + - "}"; - - public static Map getMap(String message) { - return (Map) new Gson().fromJson(message, Map.class); - } -} \ No newline at end of file diff --git a/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/task/AssessmentAggregatorTest.java b/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/task/AssessmentAggregatorTest.java deleted file mode 100644 index a47e2334e8..0000000000 --- a/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/task/AssessmentAggregatorTest.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.datastax.driver.core.Row; -import com.datastax.driver.core.UDTValue; -import com.datastax.driver.core.UserType; -import com.google.gson.Gson; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Aggregate; -import org.ekstep.ep.samza.domain.BatchEvent; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.service.AssessmentAggregatorService; -import org.ekstep.ep.samza.util.CassandraConnect; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; - -import java.util.Date; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.*; - -public class AssessmentAggregatorTest { - - private IncomingMessageEnvelope envelope; - private MessageCollector collector; - private AssessmentAggregatorTask assessmentAggregatorTask; - private TaskCoordinator taskCoordinator; - private TaskContext taskContext; - private CassandraConnect cassandraConnect; - private AssessmentAggregatorConfig config; - private Counter counter; - - @Before - public void setUp() { - Config configMock = mock(Config.class); - envelope = mock(IncomingMessageEnvelope.class); - collector = mock(MessageCollector.class); - taskCoordinator = mock(TaskCoordinator.class); - taskContext = mock(TaskContext.class); - MetricsRegistry metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = Mockito.mock(Counter.class); - stub(metricsRegistry.newCounter(anyString(), anyString())) - .toReturn(counter); - stub(taskContext.getMetricsRegistry()).toReturn(metricsRegistry); - stub(envelope.getOffset()).toReturn("2"); - stub(envelope.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1))); - stub(configMock.get("middleware.cassandra.host", "127.0.0.1")).toReturn("127.0.0.1"); - stub(configMock.get("middleware.cassandra.port", "9042")).toReturn("9042"); - stub(configMock.get("middleware.cassandra.courses_keyspace", "sunbird_courses")).toReturn("sunbird_courses"); - stub(configMock.get("middleware.cassandra.aggregator_table", "assessment_aggregator")).toReturn("assessment_aggregator"); - stub(configMock.get("middleware.cassandra.question_type", "question")).toReturn("question"); - stub(configMock.get("output.failed.topic.name", "telemetry.assess.failed")).toReturn("telemetry.assess.failed"); - UserType userType = mock(UserType.class); - UDTValue udtValue = mock(UDTValue.class); - when(udtValue.setString(anyString(), anyString())).thenReturn(udtValue); - when(udtValue.setString(any(), any())).thenReturn(udtValue); - when(udtValue.setDouble(anyString(), anyDouble())).thenReturn(udtValue); - when(udtValue.setDecimal(anyString(), any())).thenReturn(udtValue); - when(udtValue.setList(anyString(), anyList())).thenReturn(udtValue); - when(udtValue.setTimestamp(anyString(), any())).thenReturn(udtValue); - when(userType.newValue()).thenReturn(udtValue); - cassandraConnect = mock(CassandraConnect.class); - when(cassandraConnect.getUDTType("sunbird_courses", "question")).thenReturn(userType); - config = new AssessmentAggregatorConfig(configMock); - assessmentAggregatorTask = new AssessmentAggregatorTask(configMock, taskContext, cassandraConnect); - - } - - @Test - public void shouldUpdateCassandra() throws Exception { - - String event = EventFixture.BATCH_ASSESS_EVENT; - BatchEvent batchEvent = new BatchEvent((Map) new Gson().fromJson(event, Map.class)); - stub(envelope.getMessage()).toReturn(event); - assessmentAggregatorTask.process(envelope, collector, taskCoordinator); - AssessmentAggregatorService assessmentAggregatorService = new AssessmentAggregatorService - (cassandraConnect, config); - AssessmentAggregatorSink sink = new AssessmentAggregatorSink(collector, - new JobMetrics(taskContext, "AssessmentAggregator"), config); - Aggregate assess = assessmentAggregatorService.getAggregateData(batchEvent, 1565198476000L); - assertEquals(2.0, assess.getTotalMaxScore(), 0.001); - assertEquals(1.33, assess.getTotalScore(), 0.001); - assertEquals("1.33/2.0", assess.getGrandTotal()); - } - - @Test - public void shouldSkipBatchIfAssessmentDateIsOlder() throws Exception { - - String event = EventFixture.BATCH_ASSESS__OLDER_EVENT; - stub(envelope.getMessage()).toReturn(event); - Row row = mock(Row.class); - stub(row.getTimestamp("last_attempted_on")).toReturn(new Date(1567444876000L)); - when(cassandraConnect.findOne(anyString())).thenReturn(row); - assessmentAggregatorTask.process(envelope, collector, taskCoordinator); - verify(collector).send(argThat(validateOutputTopic(envelope.getMessage(), "telemetry.assess.failed"))); - verify(counter, times(2)).inc(); - } - - @Test - public void shouldSendDuplicateBatchEventToFailedTopic() throws Exception { - stub(envelope.getMessage()).toReturn(EventFixture.BATCH_ASSESS_FAIL_EVENT); - AssessmentAggregatorService assessmentAggregatorService = new AssessmentAggregatorService(cassandraConnect, config); - AssessmentAggregatorSource assessmentAggregatorSource = new AssessmentAggregatorSource(envelope); - AssessmentAggregatorSink sink = new AssessmentAggregatorSink(collector, mock(JobMetrics.class), config); - assessmentAggregatorService.process(assessmentAggregatorSource, sink); - verify(collector).send(argThat(validateOutputTopic(envelope.getMessage(), "telemetry.assess.failed"))); - } - - @Test - public void shouldTakeLatestAssessEventInBatchWhenDuplicates() throws Exception { - String event = EventFixture.BATCH_DUPLICATE_QUESTION_EVENT; - BatchEvent batchEvent = new BatchEvent((Map) new Gson().fromJson(event, Map.class)); - stub(envelope.getMessage()).toReturn(event); - AssessmentAggregatorService assessmentAggregatorService = new AssessmentAggregatorService - (cassandraConnect, config); - Aggregate assess = assessmentAggregatorService.getAggregateData(batchEvent, 1565198476000L); - assertEquals(2, assess.getQuestionsList().size()); - assertEquals(2.0, assess.getTotalMaxScore(), 0.001); - assertEquals(2.0, assess.getTotalScore(), 0.001); - - } - - @Test - public void shouldUpdateCassandraWithArrayValues() throws Exception { - - String event = EventFixture.QUESTION_EVENT_RES_VALUES; - BatchEvent batchEvent = new BatchEvent((Map) new Gson().fromJson(event, Map.class)); - stub(envelope.getMessage()).toReturn(event); - assessmentAggregatorTask.process(envelope, collector, taskCoordinator); - AssessmentAggregatorService assessmentAggregatorService = new AssessmentAggregatorService - (cassandraConnect, config); - AssessmentAggregatorSink sink = new AssessmentAggregatorSink(collector, - new JobMetrics(taskContext, "AssessmentAggregator"), config); - Aggregate assess = assessmentAggregatorService.getAggregateData(batchEvent, new Date().getTime()); - assertEquals(1.0, assess.getTotalMaxScore(), 0.001); - assertEquals(1.0, assess.getTotalScore(), 0.001); - assertEquals("1.0/1.0", assess.getGrandTotal()); - } - - - private ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - return true; - } - }; - } - -} \ No newline at end of file diff --git a/data-pipeline/assessment-aggregator/src/test/resources/test.cql b/data-pipeline/assessment-aggregator/src/test/resources/test.cql deleted file mode 100644 index 80208aa495..0000000000 --- a/data-pipeline/assessment-aggregator/src/test/resources/test.cql +++ /dev/null @@ -1,35 +0,0 @@ -CREATE KEYSPACE sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1}; - - -CREATE TYPE IF NOT EXISTS sunbird_courses.question ( -id text, -assess_ts timestamp, -max_score double, -score double, -type text, -title text, -resvalues list>>, -params list>>, -description text, -duration decimal -); - - -CREATE TABLE IF NOT EXISTS sunbird_courses.assessment_aggregator ( -course_id text, -batch_id text, -content_id text, -attempt_id text, -user_id text, -created_on timestamp, -last_attempted_on timestamp, -total_max_score double, -grand_total text, -question list>, -total_score double, -updated_on timestamp, -PRIMARY KEY (course_id, batch_id, content_id, attempt_id, user_id) -); - -CREATE INDEX assessment_aggregator_last_attempted_on_idx ON sunbird_courses.assessment_aggregator (last_attempted_on); - diff --git a/data-pipeline/content-cache-updater/pom.xml b/data-pipeline/content-cache-updater/pom.xml deleted file mode 100644 index 06e93f359c..0000000000 --- a/data-pipeline/content-cache-updater/pom.xml +++ /dev/null @@ -1,174 +0,0 @@ - - - - jobs - org.ekstep.ecosystem - 0.0.1 - - - 4.0.0 - - org.ekstep.ecosystem.jobs - content-cache-updater - 0.0.4 - ContentCacheUpdater - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - redis.clients - jedis - 2.9.0 - - - com.google.code.gson - gson - 2.4 - - - com.fiftyonred - mock-jedis - 0.4.0 - jar - test - - - com.google.guava - guava - 18.0 - - - org.jetbrains - annotations - RELEASE - compile - - - com.squareup.okhttp3 - okhttp - 4.4.0 - test - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - \ No newline at end of file diff --git a/data-pipeline/content-cache-updater/src/main/assembly/src.xml b/data-pipeline/content-cache-updater/src/main/assembly/src.xml deleted file mode 100644 index ca9b9f29f9..0000000000 --- a/data-pipeline/content-cache-updater/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/content-cache-updater.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:content-cache-updater - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - redis.clients:2.9.0 - - true - - - diff --git a/data-pipeline/content-cache-updater/src/main/config/content-cache-updater.properties b/data-pipeline/content-cache-updater/src/main/config/content-cache-updater.properties deleted file mode 100644 index f3f51ffd00..0000000000 --- a/data-pipeline/content-cache-updater/src/main/config/content-cache-updater.properties +++ /dev/null @@ -1,89 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.ContentCacheUpdater - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__redis_updater_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.ContentCacheUpdaterTask -task.inputs=kafka.__env__.learning.graph.events -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__ingestion_zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__ingestion_kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,skipped-message-count - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - - -# redis -# redis.host=localhost -redis.host=__metadata_redis_host__ -# redis.port=6379 -redis.port=__metadata_redis_port__ -redis.connection.max=3 -redis.connection.idle.max=3 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.database.contentStore.id=5 -redis.database.dialCodeStore.id=6 - -# content model fields to convert to List from String -contentModel.fields.listType=gradeLevel,subject,medium,language -date.fields.listType=lastStatusChangedOn,lastUpdatedOn,createdOn - -dialcode.api.host = __dialcode_host__ -dialcode.api.endpoint = __dialcode_api_endpoint__ -dialcode.api.authorizationkey =__dialcode_authorization_key__ - diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/service/ContentCacheUpdaterService.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/service/ContentCacheUpdaterService.java deleted file mode 100644 index 503dffd87c..0000000000 --- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/service/ContentCacheUpdaterService.java +++ /dev/null @@ -1,120 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.ekstep.ep.samza.core.BaseCacheUpdaterService; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.task.ContentCacheConfig; -import org.ekstep.ep.samza.task.ContentCacheUpdaterSink; -import org.ekstep.ep.samza.task.ContentCacheUpdaterSource; -import org.ekstep.ep.samza.util.ContentData; -import org.ekstep.ep.samza.util.RedisConnect; -import org.ekstep.ep.samza.util.RestUtil; - -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class ContentCacheUpdaterService extends BaseCacheUpdaterService { - - private static Logger LOGGER = new Logger(ContentCacheUpdaterService.class); - private JobMetrics metrics; - private ContentData contentData; - private ContentCacheConfig contentCacheConfig; - private RestUtil restUtil; - int storeId; - private Gson gson = new Gson(); - Type mapType = new TypeToken>() { - }.getType(); - - public ContentCacheUpdaterService(ContentCacheConfig config, RedisConnect redisConnect, JobMetrics metrics, RestUtil restUtil) { - super(redisConnect); - this.metrics = metrics; - this.contentCacheConfig = config; - this.restUtil = restUtil; - } - - public void process(ContentCacheUpdaterSource source, ContentCacheUpdaterSink sink) { - Map message = source.getMap(); - String nodeUniqueId = (String) message.get("nodeUniqueId"); - String objectType = (String) message.get("objectType"); - Map parsedData = null; - if (nodeUniqueId == null || objectType == null || nodeUniqueId.isEmpty() || objectType.isEmpty()) { - sink.markSkipped(); - return; - } - LOGGER.info("", "processing event for nodeUniqueId: " + nodeUniqueId); - contentData = new ContentData(); - if (!nodeUniqueId.isEmpty()) { - parsedData = getCacheData(message); - } - if (null != parsedData) { - addToCache(nodeUniqueId, gson.toJson(parsedData), storeId); - LOGGER.info(nodeUniqueId, "Updated in cache for type " + objectType); - sink.success(); - } - } - - public Map getCacheData(Map message) { - String nodeUniqueId = (String) message.get("nodeUniqueId"); - Map parsedData = null; - Map newProperties = contentData.extractProperties(message); - /** - * Responsibility: - * 1. Take dialCode property form the transaction event. - * 2. Check whether dialCode metadata is present in the cache or not. - * 3. If not present in the redis - * 1. Get the dialCode metadata using dialCode api and update to redis cache - */ - updateDialCodeToCache((List) newProperties.get("dialcodes"), contentCacheConfig.getdialCodeStoreDb(), contentCacheConfig.getDialCodeAPIUrl()); - updateDialCodeToCache(getDialCodesAsList(gson.fromJson((String) newProperties.get("reservedDialcodes"), Map.class)), contentCacheConfig.getdialCodeStoreDb(), contentCacheConfig.getDialCodeAPIUrl()); - String contentNode = null; - storeId = contentCacheConfig.getContentStoreDb(); - contentNode = readFromCache(nodeUniqueId, storeId); - Map listTypeFields = contentData.convertType(newProperties, contentCacheConfig.getContentModelListTypeFields(), contentCacheConfig.getDateFields()); - if (!listTypeFields.isEmpty()) { - newProperties.putAll(listTypeFields); - } - if (contentNode != null) { - parsedData = gson.fromJson(contentNode, mapType); - } else { - parsedData = new HashMap<>(); - } - return contentData.getParsedData(parsedData, newProperties); - } - - public void updateDialCodeToCache(List dialCodeList, int storeId, String apiUrl) { - if (null != dialCodeList) { - dialCodeList.forEach(dialCode -> { - if (!dialCode.isEmpty()) { - metrics.incDialCodesCount(); - String metaData = readFromCache(dialCode, storeId); - if (null == metaData || metaData.isEmpty()) { - LOGGER.info("", String.format("Invoking DialCode API to fetch the metadata for this dialCode( %s )", dialCode)); - Object dialCodeMetadata = contentData.getMetadata(apiUrl.concat(dialCode), this.restUtil, contentCacheConfig.getAuthorizationKey(), "dialcode"); - if (null != dialCodeMetadata) { - metrics.incDialCodesFromApiCount(); - LOGGER.info("", String.format("Inserting dialCode( %s ) metadata to cache system", dialCode)); - addToCache(dialCode, gson.toJson(dialCodeMetadata), storeId); - } else { - LOGGER.info("", String.format("API did not fetched any dialCode metadata!!! %s", dialCode)); - } - } else { - metrics.incDialCodesFromCacheCount(); - } - } - }); - } - } - - public List getDialCodesAsList(Map dialCodesMap) { - if (null != dialCodesMap) { - return new ArrayList<>(dialCodesMap.keySet()); - } else { - return new ArrayList<>(); - } - } -} \ No newline at end of file diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheConfig.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheConfig.java deleted file mode 100644 index 6a1a8cdff7..0000000000 --- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheConfig.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; - -import java.util.ArrayList; -import java.util.List; - -public class ContentCacheConfig { - - private final String JOB_NAME = "ContentCacheUpdater"; - - private int contentStoreDb; - private int dialCodeStoreDb; - private List contentModelListTypeFields; - private List dateFields; - private String apiHost; - private String dialCodeAPiEndPoint; - private String authorizationKey = ""; - - public ContentCacheConfig(Config config) { - this.contentStoreDb = config.getInt("redis.database.contentStore.id", 5); - this.dialCodeStoreDb = config.getInt("redis.database.dialCodeStore.id", 6); - this.contentModelListTypeFields = config.getList("contentModel.fields.listType", new ArrayList<>()); - this.dateFields = config.getList("date.fields.listType", new ArrayList<>()); - apiHost = config.get("dialcode.api.host","https://localhost"); - dialCodeAPiEndPoint = config.get("dialcode.api.endpoint","/api/dialcode/v3/read/"); - authorizationKey = config.get("dialcode.api.authorizationkey",""); - } - - public int getContentStoreDb() { return this.contentStoreDb; } - - public int getdialCodeStoreDb() { return this.dialCodeStoreDb; } - - public List getContentModelListTypeFields() { return this.contentModelListTypeFields; } - - public List getDateFields() { return this.dateFields; } - - public String JOB_NAME() { return JOB_NAME; } - - public String getDialCodeAPIUrl(){ - return apiHost.concat(dialCodeAPiEndPoint); - } - public String getAuthorizationKey(){ - return this.authorizationKey; - } -} diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSink.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSink.java deleted file mode 100644 index 2ca306a503..0000000000 --- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSink.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; - -public class ContentCacheUpdaterSink extends BaseSink { - - public ContentCacheUpdaterSink(MessageCollector collector, JobMetrics metrics) { - super(collector, metrics); - } - - public void success() { - metrics.incSuccessCounter(); - } - - public void markSkipped() { metrics.incSkippedCounter(); } -} diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSource.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSource.java deleted file mode 100644 index ac9032f6fe..0000000000 --- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSource.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.gson.Gson; -import org.apache.samza.system.IncomingMessageEnvelope; - -import java.util.Map; - -public class ContentCacheUpdaterSource { - - private IncomingMessageEnvelope envelope; - - public ContentCacheUpdaterSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Map getMap() { - String message = (String) envelope.getMessage(); - return (Map) new Gson().fromJson(message, Map.class); - } -} diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterTask.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterTask.java deleted file mode 100644 index c34e072e02..0000000000 --- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterTask.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.ContentCacheUpdaterService; -import org.ekstep.ep.samza.util.RedisConnect; -import org.ekstep.ep.samza.util.RestUtil; - -public class ContentCacheUpdaterTask extends BaseSamzaTask { - - private JobMetrics metrics; - private ContentCacheUpdaterService service; - private ContentCacheConfig config; - private RestUtil restUtil; - - - public ContentCacheUpdaterTask(Config config, TaskContext context, RedisConnect redisConnect, RestUtil restUtil) { - init(config, context, redisConnect, restUtil); - } - - public ContentCacheUpdaterTask() { - } - - @Override - public void init(Config config, TaskContext context) { - init(config, context, null, restUtil); - } - - public void init(Config config, TaskContext context, RedisConnect redisConnect, RestUtil restUtil) { - this.config = new ContentCacheConfig(config); - metrics = new JobMetrics(context, this.config.JOB_NAME()); - redisConnect = null != redisConnect? redisConnect: new RedisConnect(config); - this.restUtil = null != restUtil ? restUtil : new RestUtil(); - service = new ContentCacheUpdaterService(this.config, redisConnect, metrics, this.restUtil); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, - TaskCoordinator taskCoordinator) throws Exception { - - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelope); - ContentCacheUpdaterSink sink = new ContentCacheUpdaterSink(collector, metrics); - service.process(source, sink); - } - -} diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/util/ContentData.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/util/ContentData.java deleted file mode 100644 index abe3a96516..0000000000 --- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/util/ContentData.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.ekstep.ep.samza.util; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import kong.unirest.UnirestException; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.task.ContentCacheConfig; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; - -public class ContentData { - static Logger LOGGER = new Logger(ContentData.class); - - public Map getParsedData(Map parsedData, Map newProperties) { - parsedData.putAll(newProperties); - if (parsedData.size() > 0) { - parsedData.values().removeAll(Collections.singleton(null)); - parsedData.values().removeAll(Collections.singleton("")); - } - return parsedData; - } - - public Map extractProperties(Map message) { - Map properties = new HashMap<>(); - Map transactionData = (Map) message.get("transactionData"); - if (transactionData != null) { - Map addedProperties = (Map) transactionData.get("properties"); - if (addedProperties != null && !addedProperties.isEmpty()) { - for (Map.Entry propertyMap : addedProperties.entrySet()) { - if (propertyMap != null && propertyMap.getKey() != null && null!=((Map) propertyMap.getValue()).get("nv") && !((Map) propertyMap.getValue()).get("nv").toString().isEmpty()) { - String propertyName = propertyMap.getKey(); - Object propertyNewValue = ((Map) propertyMap.getValue()).get("nv"); - properties.put(propertyName, propertyNewValue); - } - } - } - } - return properties; - } - - public Map convertType(Map newProperties, List contentModelListTypeFields, List dateFields) { - Map result = new HashMap(); - for(String entry: contentModelListTypeFields){ - if(newProperties.containsKey(entry) - && newProperties.get(entry) instanceof String) { - String str = (String) newProperties.get(entry); - if(dateFields.contains(entry)) { - try{ - Date date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").parse(str); - result.put(entry,date.getTime()); - }catch (ParseException ex){ ex.printStackTrace();} - } - else { - List value = Arrays.asList(str); - result.put(entry, value); - } - } - } - return result; - } - - public Object getMetadata(String apiUrl, RestUtil restUtil, String authKey, String fieldName) { - Map headers = new HashMap<>(); - headers.put("Authorization", "Bearer " + authKey); - try { - String responseBody = restUtil.get(apiUrl, headers); - Gson gson = new Gson(); - Map response = gson.fromJson( - responseBody, new TypeToken>() { - }.getType() - ); - return gson.fromJson(gson.toJson(response.get("result")), Map.class).get(fieldName); - } catch (NullPointerException e) { - LOGGER.error(null, "Exception while fetching metadata" + e); - e.printStackTrace(); - return null; - } - } -} diff --git a/data-pipeline/content-cache-updater/src/main/resources/log4j.xml b/data-pipeline/content-cache-updater/src/main/resources/log4j.xml deleted file mode 100644 index 7e0a688c9b..0000000000 --- a/data-pipeline/content-cache-updater/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/ContentCacheUpdaterServiceTest.java b/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/ContentCacheUpdaterServiceTest.java deleted file mode 100644 index 19fcc65a37..0000000000 --- a/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/ContentCacheUpdaterServiceTest.java +++ /dev/null @@ -1,374 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.fiftyonred.mock_jedis.MockJedis; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import okhttp3.*; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.BaseCacheUpdaterService; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.Fixtures.EventFixture; -import org.ekstep.ep.samza.task.ContentCacheConfig; -import org.ekstep.ep.samza.task.ContentCacheUpdaterSink; -import org.ekstep.ep.samza.task.ContentCacheUpdaterSource; -import org.ekstep.ep.samza.task.ContentCacheUpdaterTask; -import org.ekstep.ep.samza.util.RedisConnect; -import org.ekstep.ep.samza.util.RestUtil; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import redis.clients.jedis.Jedis; - -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.*; - -public class ContentCacheUpdaterServiceTest { - - private RedisConnect redisConnectMock; - private IncomingMessageEnvelope envelopeMock; - private JobMetrics jobMetricsMock; - private Jedis jedisMock = new MockJedis("test"); - private ContentCacheUpdaterService contentCacheUpdaterService; - private ContentCacheUpdaterSink contentCacheUpdaterSinkMock; - private ContentCacheUpdaterTask contentCacheUpdaterTask; - private ContentCacheConfig contentCacheConfig; - private TaskContext contextMock; - private TaskCoordinator taskCoordinator; - private MessageCollector messageCollector; - private MetricsRegistry metricsRegistry; - private Counter counter; - private Config configMock; - private Integer contentStoreId = 5; - private Integer dialCodeStoreId = 6; - private static final String DIAL_CODE_API_ENDPOINT = "/api/dialcode/v3/read/"; - private static final String DIAL_CODE_HOST = "https://localhost"; - private static final String DIAL_CODE_API_KEY = ""; - private RestUtil restUtilMock; - - - @Before - public void setUp() throws IOException { - redisConnectMock = mock(RedisConnect.class); - contentCacheUpdaterSinkMock = mock(ContentCacheUpdaterSink.class); - configMock = mock(Config.class); - contextMock = mock(TaskContext.class); - taskCoordinator = mock(TaskCoordinator.class); - messageCollector = mock(MessageCollector.class); - jobMetricsMock = mock(JobMetrics.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = mock(Counter.class); - restUtilMock = mock(RestUtil.class); - - stub(redisConnectMock.getConnection(contentStoreId)).toReturn(jedisMock); - stub(redisConnectMock.getConnection(dialCodeStoreId)).toReturn(jedisMock); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - envelopeMock = mock(IncomingMessageEnvelope.class); - stub(configMock.getInt("redis.database.contentStore.id", contentStoreId)).toReturn(contentStoreId); - stub(configMock.getInt("redis.database.dialCodeStore.id", dialCodeStoreId)).toReturn(dialCodeStoreId); - stub(configMock.getInt("location.db.redis.key.expiry.seconds", 86400)).toReturn(86400); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "learning.graph.events", new Partition(0))); - stub(configMock.get("dialcode.api.host", DIAL_CODE_HOST)).toReturn(DIAL_CODE_HOST); - stub(configMock.get("dialcode.api.endpoint", DIAL_CODE_API_ENDPOINT)).toReturn(DIAL_CODE_API_ENDPOINT); - stub(configMock.get("dialcode.api.authorizationkey", DIAL_CODE_API_KEY)).toReturn(DIAL_CODE_API_KEY); - List defaultListValues = new ArrayList<>(); - defaultListValues.add("gradeLevel"); - defaultListValues.add("subject"); - defaultListValues.add("medium"); - defaultListValues.add("language"); - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - stub(configMock.getList("contentModel.fields.listType", new ArrayList<>())) - .toReturn(defaultListValues); - contentCacheConfig = new ContentCacheConfig(configMock); - new BaseCacheUpdaterService(redisConnectMock); - String validDialCodeUrl = "https://localhost/api/dialcode/v3/read/E1L8W5"; - String inValidDialCodeUrl_Case1 = "https://localhost/api/dialcode/v3/read/test"; - String inValidDialCodeUrl_Case2 = "https://localhost/api/dialcode/v3/read/4328W56"; - String validDialCodeUrl_case1 = "https://localhost/api/dialcode/v3/read/Z5A8W1"; - String validDialCodeUrl_case2 = "https://localhost/api/dialcode/v3/read/H4D5Q9"; - String validDialCodeUrl_case3 = "https://localhost/api/dialcode/v3/read/C1M7J9"; - createStub(validDialCodeUrl, createTestResponse(validDialCodeUrl, EventFixture.VALID_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey()); - createStub(validDialCodeUrl_case1, createTestResponse(validDialCodeUrl_case1, EventFixture.VALID_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey()); - createStub(validDialCodeUrl_case2, createTestResponse(validDialCodeUrl_case2, EventFixture.VALID_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey()); - createStub(validDialCodeUrl_case3, createTestResponse(validDialCodeUrl_case3, EventFixture.VALID_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey()); - createStub(inValidDialCodeUrl_Case1, createTestResponse(inValidDialCodeUrl_Case1, EventFixture.INVALID_DIAL_CODE_RESPONSE, 404, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey()); - createStub(inValidDialCodeUrl_Case2, createTestResponse(inValidDialCodeUrl_Case2, EventFixture.EMPTY_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey()); - contentCacheUpdaterService = new ContentCacheUpdaterService(contentCacheConfig, redisConnectMock, jobMetricsMock, restUtilMock); - jedisMock.flushAll(); - } - - - public String createTestResponse(String apiUrl, String response, int status, String authKey) throws IOException { - Request mockRequest = new Request.Builder() - .url(apiUrl) - .header("Authorization", authKey) - .build(); - return Objects.requireNonNull(new Response.Builder() - .request(mockRequest) - .protocol(Protocol.HTTP_2) - .code(status) // status code - .message("") - .body(ResponseBody.create( - MediaType.get("application/json; charset=utf-8"), - response - )) - .build().body()).string(); - } - - public void createStub(String apiUrl, String response, String authKey) { - Map headers = new HashMap(); - headers.put("Authorization", authKey); - try { - when(restUtilMock.get(apiUrl, headers)).thenReturn(response); - } catch (Exception e) { - System.out.println("Exception is" + e); - } - } - - @Test - @SuppressWarnings("unchecked") - public void updateContentsToContentStoreCache() throws Exception { - jedisMock.flushAll(); - stub(envelopeMock.getMessage()).toReturn(EventFixture.OBJECT_TYPE_CONTENT_EVENT_1); - Gson gson = new Gson(); - Map event = gson.fromJson(EventFixture.OBJECT_TYPE_CONTENT_EVENT_1, Map.class); - String contentId = (String) event.get("nodeUniqueId"); - - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - Map contentData = contentCacheUpdaterService.getCacheData(source.getMap()); - jedisMock.set(contentId, contentData.toString()); - - String cachedData = jedisMock.get(contentId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals(5, parsedData.size()); - assertEquals("in.ekstep", parsedData.get("channel")); - assertEquals("TestCollection", parsedData.get("description")); - assertEquals("testbook1", parsedData.get("code")); - assertEquals("Draft", parsedData.get("status")); - assertEquals(parsedData.get("ownershipType") instanceof List, true); - assertEquals(Arrays.asList("createdBy"), parsedData.get("ownershipType")); - } - - @Test - @SuppressWarnings("unchecked") - public void handleStringToList() throws Exception { - jedisMock.flushAll(); - - stub(envelopeMock.getMessage()).toReturn(EventFixture.INCORRECT_LIST_TYPE_FIELDS); - Gson gson = new Gson(); - Map event = gson.fromJson(EventFixture.INCORRECT_LIST_TYPE_FIELDS, Map.class); - jedisMock.select(contentStoreId); - String contentId = (String) event.get("nodeUniqueId"); - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - Map contentData = contentCacheUpdaterService.getCacheData(source.getMap()); - jedisMock.set(contentId, contentData.toString()); - - String cachedData = jedisMock.get(contentId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals(4, parsedData.size()); - assertEquals(parsedData.get("language") instanceof List, true); - assertEquals(Arrays.asList("Spanish"), parsedData.get("language")); - assertEquals(parsedData.get("subject") instanceof List, true); - assertEquals(Arrays.asList("CS"), parsedData.get("subject")); - assertEquals(22.0, parsedData.get("ageGroup")); - assertEquals(parsedData.get("ownershipType") instanceof List, true); - assertEquals(Arrays.asList("createdBy"), parsedData.get("ownershipType")); - verify(contentCacheUpdaterSinkMock, times(1)).success(); - } - - @Test - @SuppressWarnings("unchecked") - public void shouldUpdateCacheIfDataExist() throws Exception { - jedisMock.flushAll(); - - // Update Redis Cache with First Event - when(envelopeMock.getMessage()).thenReturn(EventFixture.OBJECT_TYPE_CONTENT_EVENT_1, EventFixture.OBJECT_TYPE_CONTENT_EVENT_1_UPDATED); - contentCacheUpdaterService.process(new ContentCacheUpdaterSource(envelopeMock), contentCacheUpdaterSinkMock); - - // Update the redis cache with Second event with updated fields: channel, status - // removed fields: ownershipType - // new fields: visibility - Gson gson = new Gson(); - Map event = gson.fromJson(EventFixture.OBJECT_TYPE_CONTENT_EVENT_1_UPDATED, Map.class); - String contentId = (String) event.get("nodeUniqueId"); - - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - Map contentData = contentCacheUpdaterService.getCacheData(source.getMap()); - jedisMock.set(contentId, contentData.toString()); - String cachedData = jedisMock.get(contentId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals(6, parsedData.size()); - assertEquals("testbook1", parsedData.get("code")); - assertEquals("sunbird.portal", parsedData.get("channel")); - assertEquals("Live", parsedData.get("status")); - assertEquals("Default", parsedData.get("visibility")); - assertEquals("TestCollection", parsedData.get("description")); - assertEquals(Arrays.asList("createdBy"), parsedData.get("ownershipType")); - } - - @Test - public void shouldNotAddToCacheIfKeyValueIsNullOrEmpty() throws Exception { - jedisMock.flushAll(); - - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - when(envelopeMock.getMessage()).thenReturn(EventFixture.CONTENT_EVENT_EMPTY_NODE_UNIQUEID, EventFixture.CONTENT_EVENT_EMPTY_PROPERTIES); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - assertEquals(0, jedisMock.keys("*").size()); - } - - @Test - @SuppressWarnings("unchecked") - public void shouldNotUpdateEmptyMessageProperties() throws Exception { - jedisMock.flushAll(); - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - - // Update Redis Cache with First Event - when(envelopeMock.getMessage()).thenReturn(EventFixture.CONTENT_EVENT_EMPTY_PROPERTIES); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - - // Second event with empty properties - Gson gson = new Gson(); - Map event = gson.fromJson(EventFixture.CONTENT_EVENT_EMPTY_PROPERTIES, Map.class); - String contentId = (String) event.get("nodeUniqueId"); - - String cachedData = jedisMock.get(contentId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals(0, parsedData.size()); - } - - @Test - @SuppressWarnings("unchecked") - public void shouldAddOtherObjectTypeToCache() throws Exception { - jedisMock.flushAll(); - stub(envelopeMock.getMessage()).toReturn(EventFixture.OBJECT_TYPE_CONCEPT_EVENT); - - Gson gson = new Gson(); - Map event = gson.fromJson(EventFixture.OBJECT_TYPE_CONCEPT_EVENT, Map.class); - String conceptId = (String) event.get("nodeUniqueId"); - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - Map contentData = contentCacheUpdaterService.getCacheData(source.getMap()); - jedisMock.set(conceptId, contentData.toString()); - - String cachedData = jedisMock.get(conceptId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals(1, parsedData.size()); - ArrayList str = new ArrayList<>(); - str.add("English"); - assertEquals(str, parsedData.get("language")); - } - - @Test - public void shouldMarkEventSkippedForNonodeUniqueId() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.CONTENT_EVENT_EMPTY_NODE_UNIQUEID); - contentCacheUpdaterTask = new ContentCacheUpdaterTask(configMock, contextMock, redisConnectMock, restUtilMock); - contentCacheUpdaterTask.process(envelopeMock, messageCollector, taskCoordinator); - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - - verify(contentCacheUpdaterSinkMock, times(1)).markSkipped(); - } - - @Test - @SuppressWarnings("unchecked") - public void shouldNotAddEventIfItIsEmpty() throws Exception { - jedisMock.flushAll(); - stub(envelopeMock.getMessage()).toReturn(EventFixture.EMPTY_LANGUAGE_FIELD_EVENT); - - Gson gson = new Gson(); - Map event = gson.fromJson(EventFixture.EMPTY_LANGUAGE_FIELD_EVENT, Map.class); - String conceptId = (String) event.get("nodeUniqueId"); - - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - Map contentData = contentCacheUpdaterService.getCacheData(source.getMap()); - jedisMock.set(conceptId, contentData.toString()); - - String cachedData = jedisMock.get(conceptId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals(0, parsedData.size()); - assertEquals(null, parsedData.get("language")); - } - - @Test - @SuppressWarnings("unchecked") - public void shouldInvokeDialCodeAPICall() throws Exception { - jedisMock.flushAll(); - stub(envelopeMock.getMessage()).toReturn(EventFixture.DIALCODE_LINKED_TEXTBOOK); - Gson gson = new Gson(); - Map event = gson.fromJson(EventFixture.DIALCODE_LINKED_TEXTBOOK, Map.class); - String conceptId = (String) event.get("nodeUniqueId"); - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - } - - @Test - @SuppressWarnings("unchecked") - public void WhenDialCodeIsNotPresetInTheExternalSystemAPIShouldReturnResourceNotFound() throws Exception { - jedisMock.flushAll(); - stub(envelopeMock.getMessage()).toReturn(EventFixture.INVALID_DIALCODE_LINKED_TEXTBOOK); - Gson gson = new Gson(); - Map event = gson.fromJson(EventFixture.INVALID_DIALCODE_LINKED_TEXTBOOK, Map.class); - String conceptId = (String) event.get("nodeUniqueId"); - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - } - - @Test - @SuppressWarnings("unchecked") - public void WhenEmptyDialCodeAPIResponse() throws Exception { - jedisMock.flushAll(); - stub(envelopeMock.getMessage()).toReturn(EventFixture.EMPTY_DIALCODE_LINKED_TEXTBOOK); - Gson gson = new Gson(); - Map event = gson.fromJson(EventFixture.EMPTY_DIALCODE_LINKED_TEXTBOOK, Map.class); - String conceptId = (String) event.get("nodeUniqueId"); - ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock); - contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock); - } -} diff --git a/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java b/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java deleted file mode 100644 index edf7caccfb..0000000000 --- a/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.ekstep.ep.samza.service.Fixtures; - -public class EventFixture { - public static final String OBJECT_TYPE_CONTENT_EVENT_1 = "{\"ets\":1548780816826,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"ownershipType\":{\"ov\":null,\"nv\":[\"createdBy\"]},\"code\":{\"ov\":null,\"nv\":\"testbook1\"},\"channel\":{\"ov\":null,\"nv\":\"in.ekstep\"},\"description\":{\"ov\":null,\"nv\":\"TestCollection\"},\"status\":{\"ov\":null,\"nv\":\"Draft\"}}},\"label\":\"TestBook1\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-29T22:23:36.826+0530\",\"objectType\":\"Content\",\"nodeUniqueId\":\"do_11268761245100441611\",\"requestId\":null,\"operationType\":\"CREATE\",\"nodeGraphId\":703654,\"graphId\":\"domain\"}"; - public static final String OBJECT_TYPE_CONTENT_EVENT_1_UPDATED = "{\"ets\":1548780816826,\"channel\":\"sunbird.portal\",\"transactionData\":{\"properties\":{\"ownershipType\":{\"ov\":null,\"nv\":null},\"code\":{\"ov\":null,\"nv\":\"testbook1\"},\"channel\":{\"ov\":null,\"nv\":\"sunbird.portal\"},\"status\":{\"ov\":null,\"nv\":\"Live\"},\"visibility\":{\"ov\":null,\"nv\":\"Default\"}}},\"label\":\"TestBook1\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-29T22:23:36.826+0530\",\"objectType\":\"Content\",\"nodeUniqueId\":\"do_11268761245100441611\",\"requestId\":null,\"operationType\":\"CREATE\",\"nodeGraphId\":703654,\"graphId\":\"domain\"}"; - public static final String OBJECT_TYPE_DIAL_CODE_1 = "{\"nodeUniqueId\":\"YC9EP8\",\"ets\":1549345256766,\"requestId\":null,\"audit\":false,\"transactionData\":{\"properties\":{\"dialcode_index\":{\"ov\":null,\"nv\":261351.0},\"identifier\":{\"ov\":null,\"nv\":\"YC9EP8\"},\"channel\":{\"ov\":null,\"nv\":\"b00bc992ef25f1a9a8d63291e20efc8d\"},\"batchcode\":{\"ov\":null,\"nv\":\"do_112692236142379008136\"},\"publisher\":{\"ov\":null,\"nv\":null},\"generated_on\":{\"ov\":null,\"nv\":\"2019-02-05T05:40:56.762\"},\"status\":{\"ov\":null,\"nv\":\"Draft\"}}},\"channel\":\"in.ekstep\",\"index\":true,\"operationType\":\"CREATE\",\"nodeType\":\"EXTERNAL\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-02-05T05:40:56.766+0000\",\"objectType\":\"DialCode\"}"; - public static final String CONTENT_EVENT_EMPTY_PROPERTIES = "{\"ets\":1548780816826,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{}},\"label\":\"TestBook1\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-29T22:23:36.826+0530\",\"objectType\":\"Content\",\"nodeUniqueId\":\"do_11268761245100441611\",\"requestId\":null,\"operationType\":\"CREATE\",\"nodeGraphId\":703654,\"graphId\":\"domain\"}"; - public static final String CONTENT_EVENT_EMPTY_NODE_UNIQUEID = "{\"ets\":1548780816826,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"ownershipType\":{\"ov\":null,\"nv\":[\"createdBy\"]},\"code\":{\"ov\":null,\"nv\":\"testbook1\"},\"channel\":{\"ov\":null,\"nv\":\"in.ekstep\"},\"description\":{\"ov\":null,\"nv\":\"TestCollection\"},\"status\":{\"ov\":null,\"nv\":\"Draft\"}}},\"label\":\"TestBook1\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-29T22:23:36.826+0530\",\"objectType\":\"Content\",\"nodeUniqueId\":\"\",\"requestId\":null,\"operationType\":\"CREATE\",\"nodeGraphId\":703654,\"graphId\":\"domain\"}"; - public static final String OBJECT_TYPE_CONCEPT_EVENT = "{\"ets\":1547644874028,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"language\":{\"ov\":null,\"nv\":\"English\"}}},\"label\":\"Properties of Light\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-16T18:51:14.028+0530\",\"objectType\":\"Concept\",\"nodeUniqueId\":\"SMC4\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":101409,\"graphId\":\"domain\"}"; - public static final String EMPTY_LANGUAGE_FIELD_EVENT= "{\"ets\":1547644874028,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"language\":{\"ov\":null,\"nv\":\"\"}}},\"label\":\"Properties of Light\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-16T18:51:14.028+0530\",\"objectType\":\"Concept\",\"nodeUniqueId\":\"SMC4\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":101409,\"graphId\":\"domain\"}"; - public static final String LIST_LANGUAGE_FIELD_EVENT= "{\"ets\":1547644874028,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"language\":{\"ov\":null,\"nv\":[\"Kannada\"]}}},\"label\":\"Properties of Light\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-16T18:51:14.028+0530\",\"objectType\":\"Concept\",\"nodeUniqueId\":\"SMC4\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":101409,\"graphId\":\"domain\"}"; - public static final String INCORRECT_LIST_TYPE_FIELDS= "{\"ets\":1559670759,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"language\":{\"ov\":null,\"nv\":\"Spanish\"},\"subject\":{\"ov\":null,\"nv\":\"CS\"},\"ageGroup\":{\"ov\":null,\"nv\":\"22\"},\"ownershipType\":{\"ov\":null,\"nv\":[\"createdBy\"]}}},\"label\":\"Properties of Light\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-06-07T18:51:14.028+0530\",\"objectType\":\"Concept\",\"nodeUniqueId\":\"SMC4\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":98765,\"graphId\":\"domain\"}"; - public static final String DIALCODE_LINKED_TEXTBOOK="{\"ets\":1577734412619,\"channel\":\"channel-01\",\"transactionData\":{\"properties\":{\"dialcodes\":{\"ov\":null,\"nv\":[\"E1L8W5\"]},\"reservedDialcodes\":{\"ov\":null,\"nv\":\"{\\\"Z5A8W1\\\":2,\\\"H4D5Q9\\\":0,\\\"C1M7J9\\\":1}\"},\"SYS_INTERNAL_LAST_UPDATED_ON\":{\"ov\":\"2019-12-30T19:33:32.504+0000\",\"nv\":\"2019-12-30T19:33:32.580+0000\"}}},\"mid\":\"3c99290d-037e-47c6-98f8-ddbd26452fb8\",\"label\":\"KP Integration Test Collection Content\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-12-30T19:33:32.619+0000\",\"objectType\":\"Content\",\"nodeUniqueId\":\"KP_FT_1577734411271\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":373563,\"graphId\":\"domain\"}"; - public static final String INVALID_DIALCODE_LINKED_TEXTBOOK="{\"ets\":1577734412619,\"channel\":\"channel-01\",\"transactionData\":{\"properties\":{\"dialcodes\":{\"ov\":null,\"nv\":[\"test\"]},\"SYS_INTERNAL_LAST_UPDATED_ON\":{\"ov\":\"2019-12-30T19:33:32.504+0000\",\"nv\":\"2019-12-30T19:33:32.580+0000\"}}},\"mid\":\"3c99290d-037e-47c6-98f8-ddbd26452fb8\",\"label\":\"KP Integration Test Collection Content\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-12-30T19:33:32.619+0000\",\"objectType\":\"Content\",\"nodeUniqueId\":\"KP_FT_1577734411271\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":373563,\"graphId\":\"domain\"}"; - public static final String VALID_DIAL_CODE_RESPONSE = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-03-03T09:09:50ZZ\",\"params\":{\"resmsgid\":\"fef4be15-b87e-421e-9e31-077bab44f0f2\",\"msgid\":null,\"err\":null,\"status\":\"successful\",\"errmsg\":null},\"responseCode\":\"OK\",\"result\":{\"dialcode\":{\"identifier\":\"977D3I\",\"channel\":\"0123221617357783046602\",\"publisher\":\"MHPUBLISHER\",\"batchCode\":\"MHPUBLISHER.20180402T112421\",\"status\":\"Draft\",\"generatedOn\":\"2018-04-02T11:24:22.366\",\"publishedOn\":null,\"metadata\":null}}}"; - public static final String EMPTY_DIAL_CODE_RESPONSE = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-03-03T09:09:50ZZ\",\"params\":{\"resmsgid\":\"fef4be15-b87e-421e-9e31-077bab44f0f2\",\"msgid\":null,\"err\":null,\"status\":\"successful\",\"errmsg\":null},\"responseCode\":\"OK\",\"result\":null}"; - public static final String INVALID_DIAL_CODE_RESPONSE = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-03-04T10:00:18ZZ\",\"params\":{\"resmsgid\":\"df4a320b-2cc9-4bc8-9273-dc4f82ee4b43\",\"msgid\":null,\"err\":\"ERR_DIALCODE_INFO\",\"status\":\"failed\",\"errmsg\":\"Dial Code Not Found With Id: S6B4X11\"},\"responseCode\":\"RESOURCE_NOT_FOUND\",\"result\":{}}"; - public static final String EMPTY_DIALCODE_LINKED_TEXTBOOK="{\"ets\":1577734412619,\"channel\":\"channel-01\",\"transactionData\":{\"properties\":{\"dialcodes\":{\"ov\":null,\"nv\":[\"4328W56\",\"\"]},\"SYS_INTERNAL_LAST_UPDATED_ON\":{\"ov\":\"2019-12-30T19:33:32.504+0000\",\"nv\":\"2019-12-30T19:33:32.580+0000\"}}},\"mid\":\"3c99290d-037e-47c6-98f8-ddbd26452fb8\",\"label\":\"KP Integration Test Collection Content\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-12-30T19:33:32.619+0000\",\"objectType\":\"Content\",\"nodeUniqueId\":\"KP_FT_1577734411271\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":373563,\"graphId\":\"domain\"}"; -} diff --git a/data-pipeline/de-duplication/pom.xml b/data-pipeline/de-duplication/pom.xml deleted file mode 100644 index 61028dc4a9..0000000000 --- a/data-pipeline/de-duplication/pom.xml +++ /dev/null @@ -1,179 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - - org.ekstep.ecosystem.jobs - de-duplication - 0.5.1 - jar - Deduplication - - Identifying Duplicate Events - - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - com.fiftyonred - mock-jedis - 0.4.0 - test - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - - diff --git a/data-pipeline/de-duplication/src/main/assembly/src.xml b/data-pipeline/de-duplication/src/main/assembly/src.xml deleted file mode 100644 index 437104ac07..0000000000 --- a/data-pipeline/de-duplication/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/de-duplication.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-kv-rocksdb_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:de-duplication - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/de-duplication/src/main/config/de-duplication.properties b/data-pipeline/de-duplication/src/main/config/de-duplication.properties deleted file mode 100644 index 436ef9e702..0000000000 --- a/data-pipeline/de-duplication/src/main/config/de-duplication.properties +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.DeDuplication - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__telemetry_de_duplication_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.DeDuplicationTask -task.inputs=kafka.__env__.telemetry.valid -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -# YARN container configuration -# yarn.am.container.memory.mb=__yarn_am_container_mb__ -# yarn.container.memory.mb=__dedup_job_yarn_container_memory_mb__ - -output.success.topic.name=__env__.telemetry.unique -output.failed.topic.name=__env__.telemetry.failed -output.duplicate.topic.name=__env__.telemetry.duplicate -output.malformed.topic.name=__env__.telemetry.malformed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,failed-message-count,error-message-count,duplicate-event-count -default.channel=__default_channel__ - - -# redis -#redis.host=localhost -redis.host=__redis_host__ -#redis.port=6379 -redis.port=__redis_port__ -redis.connection.max=2 -redis.connection.idle.max=2 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.database.duplicationstore.id=7 -redis.database.key.expiry.seconds=86400 - - -dedup.producer.include.ids=__dedup_include_producer_ids__ - - diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index 0d40545d34..0000000000 --- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.ekstep.ep.samza.domain; - - -import org.apache.commons.lang.StringUtils; -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.task.DeDuplicationConfig; - -import java.util.HashMap; -import java.util.Map; - -public class Event extends Events { - - public Event(Map map) { - - super(map); - } - - public void markSkipped() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.dd_processed", false); - telemetry.add("flags.dd_checksum_present", false); - } - - public void markDuplicate() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.dd_processed", false); - telemetry.add("flags.dd_duplicate_event", true); - } - - public void markSuccess() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.dd_processed", true); - telemetry.add("type", "events"); - } - - public void markRedisFailure() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.dd_processed", false); - telemetry.add("flags.dd_redis_failure", true); - telemetry.add("type", "events"); - } - - public void updateDefaults(DeDuplicationConfig config) { - String channelString = telemetry.read("context.channel").value(); - String channel = StringUtils.deleteWhitespace(channelString); - if (channel == null || channel.isEmpty()) { - telemetry.addFieldIfAbsent("context", new HashMap()); - telemetry.add("context.channel", config.defaultChannel()); - } - } -} - diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java deleted file mode 100644 index 1cc36c771a..0000000000 --- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.JsonSyntaxException; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.DeDuplicationConfig; -import org.ekstep.ep.samza.task.DeDuplicationSink; -import org.ekstep.ep.samza.task.DeDuplicationSource; -import org.ekstep.ep.samza.util.DeDupEngine; - -import redis.clients.jedis.exceptions.JedisException; - -public class DeDuplicationService { - private static Logger LOGGER = new Logger(DeDuplicationService.class); - private final DeDupEngine deDupEngine; - private final DeDuplicationConfig config; - - public DeDuplicationService(DeDupEngine deDupEngine, DeDuplicationConfig config) { - this.deDupEngine = deDupEngine; - this.config = config; - } - - public void process(DeDuplicationSource source, DeDuplicationSink sink) throws Exception { - Event event = null; - - try { - event = source.getEvent(); - String checksum = event.getChecksum(); - if (checksum == null) { - LOGGER.info(event.id(), "EVENT WITHOUT CHECKSUM & MID, PASSING THROUGH : {}", event); - event.markSkipped(); - event.updateDefaults(config); - sink.toSuccessTopic(event); - return; - } - - if (isDupCheckRequired(event)) { - if (!deDupEngine.isUniqueEvent(checksum)) { - LOGGER.info(event.id(), "DUPLICATE EVENT, CHECKSUM: {}", checksum); - event.markDuplicate(); - sink.toDuplicateTopic(event); - return; - } - - LOGGER.info(event.id(), "ADDING EVENT CHECKSUM TO STORE"); - - deDupEngine.storeChecksum(checksum); - } - event.updateDefaults(config); - event.markSuccess(); - sink.toSuccessTopic(event); - - } catch (JedisException e) { - e.printStackTrace(); - LOGGER.error(null, "Exception when retrieving data from redis: ", e); - deDupEngine.getRedisConnection().close(); - throw e; - } catch (JsonSyntaxException e) { - LOGGER.error(null, "INVALID EVENT: " + source.getMessage()); - sink.toMalformedEventsTopic(source.getMessage()); - } - } - - public boolean isDupCheckRequired(Event event) { - return (config.inclusiveProducerIds().isEmpty() - || (null != event.producerId() && config.inclusiveProducerIds().contains(event.producerId()))); - } -} diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java deleted file mode 100644 index efca4a5e40..0000000000 --- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.ekstep.ep.samza.task; - - -import org.apache.samza.config.Config; - -import java.util.ArrayList; -import java.util.List; - -public class DeDuplicationConfig { - - private final String JOB_NAME = "DeDuplication"; - private String successTopic; - private String failedTopic; - private String duplicateTopic; - private String malformedTopic; - private String defaultChannel; - private final int dupStore; - private int expirySeconds; - private List includedProducerIds = new ArrayList<>(); - - - public DeDuplicationConfig(Config config) { - successTopic = config.get("output.success.topic.name", "telemetry.valid"); - failedTopic = config.get("output.failed.topic.name", "telemetry.failed"); - duplicateTopic = config.get("output.duplicate.topic.name", "telemetry.duplicate"); - malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed"); - defaultChannel = config.get("default.channel", "org.sunbird"); - dupStore = config.getInt("redis.database.duplicationstore.id", 7); - expirySeconds = config.getInt("redis.database.key.expiry.seconds", 86400); - if (!config.get("dedup.producer.include.ids", "").isEmpty()) { - includedProducerIds = config.getList("dedup.producer.include.ids", new ArrayList<>()); - } - - } - - public String successTopic() { - return successTopic; - } - - public String failedTopic() { - return failedTopic; - } - - public String duplicateTopic() { - return duplicateTopic; - } - - public String malformedTopic() { - return malformedTopic; - } - - public String defaultChannel() { - return defaultChannel; - } - - public String jobName() { - return JOB_NAME; - } - - public int dupStore() { - return dupStore; - } - - public int expirySeconds() { - return expirySeconds; - } - - public List inclusiveProducerIds() { - return includedProducerIds; - } - -} \ No newline at end of file diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java deleted file mode 100644 index a100e4be50..0000000000 --- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - -public class DeDuplicationSink extends BaseSink { - - private DeDuplicationConfig config; - - public DeDuplicationSink(MessageCollector collector, JobMetrics metrics, DeDuplicationConfig config) { - super(collector, metrics); - this.config = config; - } - - public void toSuccessTopic(Event event) { - toTopic(config.successTopic(), event.did(), event.getJson()); - metrics.incSuccessCounter(); - } - - public void toSuccessTopicIfRedisException(Event event) { - toTopic(config.successTopic(), event.did(), event.getJson()); - metrics.incCacheErrorCounter(); - } - - public void toDuplicateTopic(Event event) { - toTopic(config.duplicateTopic(), event.did(), event.getJson()); - metrics.incDuplicateCounter(); - } - - public void toMalformedEventsTopic(String message) { - toTopic(config.malformedTopic(), null, message); - metrics.incFailedCounter(); - } - - public void toErrorTopic(Event event) { - toTopic(config.failedTopic(), event.did(), event.getJson()); - metrics.incErrorCounter(); - } - -} diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java deleted file mode 100644 index 533787e194..0000000000 --- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.ekstep.ep.samza.task; - -import java.util.Map; - -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; - -import com.google.gson.Gson; - -public class DeDuplicationSource { - static Logger LOGGER = new Logger(DeDuplicationSource.class); - - private IncomingMessageEnvelope envelope; - - public DeDuplicationSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - String message = (String) envelope.getMessage(); - @SuppressWarnings("unchecked") - Map jsonMap = (Map) new Gson().fromJson(message, Map.class); - return new Event(jsonMap); - } - - public String getMessage() { - return envelope.toString(); - } - -} diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java deleted file mode 100644 index b168b19292..0000000000 --- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.DeDuplicationService; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.ekstep.ep.samza.util.RedisConnect; - -public class DeDuplicationTask extends BaseSamzaTask { - - private DeDuplicationConfig config; - private JobMetrics metrics; - private DeDuplicationService service; - - public DeDuplicationTask(Config config, TaskContext context, - DeDupEngine deDupEngine) { - init(config, context, deDupEngine); - } - - public DeDuplicationTask() { - - } - - @Override - public void init(Config config, TaskContext context) { - init(config, context, null); - } - - private void init(Config config, TaskContext context, - DeDupEngine deDupEngine) { - this.config = new DeDuplicationConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - deDupEngine = deDupEngine == null ? - new DeDupEngine(new RedisConnect(config), this.config.dupStore(), this.config.expirySeconds()) : deDupEngine; - service = new DeDuplicationService(deDupEngine, this.config); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) throws Exception { - DeDuplicationSource source = new DeDuplicationSource(envelope); - DeDuplicationSink sink = new DeDuplicationSink(collector, metrics, config); - service.process(source, sink); - } - -} diff --git a/data-pipeline/de-duplication/src/main/resources/log4j.xml b/data-pipeline/de-duplication/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/de-duplication/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java b/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java deleted file mode 100644 index 82294daabe..0000000000 --- a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Map; - -public class EventTest { - - @Test - public void shouldMarkTheRedisFailure() { - String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}"; - Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class)); - event.markRedisFailure(); - Map flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class); - Object type = event.getMap().get("type"); - Assert.assertEquals(type, "events"); - Assert.assertNotNull(flagData); - Assert.assertEquals(flagData.get("dd_processed"), false); - Assert.assertEquals(flagData.get("dd_redis_failure"), true); - Assert.assertNotNull(type); - - } - - @Test - public void shouldMarkSkipped() { - String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}"; - Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class)); - event.markSkipped(); - Map flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class); - Assert.assertNotNull(flagData); - Assert.assertEquals(flagData.get("dd_processed"), false); - Assert.assertEquals(flagData.get("dd_checksum_present"), false); - - } - - -} diff --git a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index ce61e49754..0000000000 --- a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,166 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import java.util.Map; - -public class EventFixture { - private static final String EVENT_WITH_CHECKSUM = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"9a78dacddfcdfae7a9679b143e348158\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"id\": \"do_312764599773110272147\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"pageid\": \"ContentApp-EndScreen\",\n" + - " \"subtype\": \"ContentID\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1561620619274,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.14.5\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\",\n" + - " \"id\": \"dev.diksha.portal\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"contentplayer\",\n" + - " \"did\": \"9a78dacddfcdfae7a9679b143e348158\",\n" + - " \"sid\": \"8c416b30-4e47-a695-b1d2-9cd030eadeff\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"DialCode\",\n" + - " \"id\": \"DNILBR\"\n" + - " },\n" + - " {\n" + - " \"type\": \"ContentSession\",\n" + - " \"id\": \"88de9e1dfa6f237c076278a02cc4b305\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"INTERACT:fd63ff6bd0eedf05cf57c0058588ae69\",\n" + - " \"object\": {\n" + - " \"ver\": \"1\",\n" + - " \"id\": \"do_312764599773110272147\",\n" + - " \"type\": \"Content\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_31276594374681395211909\"\n" + - " }\n" + - " },\n" + - " \"tags\": [],\n" + - " \"syncts\": 1561620619795,\n" + - " \"@timestamp\": \"2019-06-27T07:30:19.795Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - private static final String EVENT_WITH_MID = "{\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"loc\": \"\",\n" + - " \"uid\": \"6\",\n" + - " \"age\": 5,\n" + - " \"handle\": \"Jijesh\",\n" + - " \"standard\": -1,\n" + - " \"language\": \"ML\"\n" + - " }\n" + - " },\n" + - " \"eid\": \"GE_CREATE_PROFILE\",\n" + - " \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genieservice.android\",\n" + - " \"ver\": \"1.0.local-qa-debug\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"tags\": [],\n" + - " \"ts\": \"2015-09-27T13:03:43-04:00\",\n" + - " \"uid\": \"6\",\n" + - " \"ver\": \"1.0\",\n" + - " \"mid\": \"22e1430f2e5f339230dbf9595b060008\"" + - "}"; - - private static final String EVENT_WITHOUT_CHECKSUM = "{\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"loc\": \"\",\n" + - " \"uid\": \"6\",\n" + - " \"age\": 5,\n" + - " \"handle\": \"Jijesh\",\n" + - " \"standard\": -1,\n" + - " \"language\": \"ML\"\n" + - " }\n" + - " },\n" + - " \"eid\": \"GE_CREATE_PROFILE\",\n" + - " \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genieservice.android\",\n" + - " \"ver\": \"1.0.local-qa-debug\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"tags\": [],\n" + - " \"ts\": \"2015-09-27T13:03:43-04:00\",\n" + - " \"uid\": \"6\",\n" + - " \"ver\": \"1.0\"" + - "}"; - - private static final String EVENT_WITH_EMTPY_CHANNEL = "{\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"loc\": \"\",\n" + - " \"uid\": \"6\",\n" + - " \"age\": 5,\n" + - " \"handle\": \"Jijesh\",\n" + - " \"standard\": -1,\n" + - " \"language\": \"ML\"\n" + - " }\n" + - " },\n" + - " \"eid\": \"GE_CREATE_PROFILE\",\n" + - " \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genieservice.android\",\n" + - " \"ver\": \"1.0.local-qa-debug\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"context\": {\n" + - " \"channel\": \"\"\n" + - " },\n" + - " \"tags\": [],\n" + - " \"ts\": \"2015-09-27T13:03:43-04:00\",\n" + - " \"uid\": \"6\",\n" + - " \"ver\": \"1.0\"" + - "}"; - - public static Map EventWithChecksumMap() { - return new Gson().fromJson(EVENT_WITH_CHECKSUM, new TypeToken>() { - }.getType()); - } - - public static Map EventWithMidMap() { - return new Gson().fromJson(EVENT_WITH_MID, new TypeToken>() { - }.getType()); - } - - public static Map EventWithoutChecksumFieldMap() { - return new Gson().fromJson(EVENT_WITHOUT_CHECKSUM, new TypeToken>() { - }.getType()); - } - - public static Map EventWithEmptyChannel() { - return new Gson().fromJson(EVENT_WITH_EMTPY_CHANNEL, new TypeToken>() { - }.getType()); - } - - public static String EventWithChecksumJson() { - return EVENT_WITH_CHECKSUM; - } - -} - diff --git a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java deleted file mode 100644 index d29ab1f347..0000000000 --- a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.ekstep.ep.samza.system; - - -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.task.DeDuplicationConfig; -import org.junit.Assert; -import org.junit.Test; -import java.util.Map; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class EventTest { - @Test - public void shouldReturnChecksumIfPresent(){ - - Event event = new Event(EventFixture.EventWithChecksumMap()); - Assert.assertEquals("INTERACT:fd63ff6bd0eedf05cf57c0058588ae69", (String) event.getChecksum()); - } - - @Test - public void shouldReturnMidIfPresent(){ - - Event event = new Event(EventFixture.EventWithMidMap()); - Assert.assertEquals("22e1430f2e5f339230dbf9595b060008", (String) event.getChecksum()); - } - - @Test - public void shouldReturnNullIfChecksumAndMidAreAbsent(){ - - Event event = new Event(EventFixture.EventWithoutChecksumFieldMap()); - Assert.assertEquals(null, (String) event.getChecksum()); - } - - @Test - public void shouldAddChannelIfChannelIsAbsentNullOrEmpty(){ - DeDuplicationConfig conf = mock(DeDuplicationConfig.class); - when(conf.defaultChannel()).thenReturn("in.ekstep"); - - Event event = new Event(EventFixture.EventWithMidMap()); - event.updateDefaults(conf); - Assert.assertEquals("in.ekstep", ( (Map) event.getMap().get("context")).get("channel") ); - - - Event event2 = new Event(EventFixture.EventWithEmptyChannel()); - event2.updateDefaults(conf); - Assert.assertEquals("in.ekstep", ( (Map) event.getMap().get("context")).get("channel")); - } -} - diff --git a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java b/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java deleted file mode 100644 index 20e842eaab..0000000000 --- a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.fiftyonred.mock_jedis.MockJedis; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; -import redis.clients.jedis.Jedis; - -import java.util.ArrayList; -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -public class DeDuplicationTaskTest { - - private static final String SUCCESS_TOPIC = "telemetry.valid"; - private static final String FAILED_TOPIC = "telemetry.unique.fail"; - private static final String DUPLICATE_TOPIC = "telemetry.duplicate"; - private static final String MALFORMED_TOPIC = "telemetry.malformed"; - private MessageCollector collectorMock; - private TaskContext contextMock; - private MetricsRegistry metricsRegistry; - private Counter counter; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private Config configMock; - private DeDuplicationTask deDuplicationTask; - private DeDupEngine deDupEngineMock; - private Jedis jedisMock = new MockJedis("duplicationtest"); - private int dupStoreId = 1; - - @Before - public void setUp() { - collectorMock = mock(MessageCollector.class); - contextMock = Mockito.mock(TaskContext.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = Mockito.mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - configMock = Mockito.mock(Config.class); - RedisConnect redisConnectMock = mock(RedisConnect.class); - deDupEngineMock = mock(DeDupEngine.class); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC); - stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC); - stub(configMock.get("output.duplicate.topic.name", DUPLICATE_TOPIC)).toReturn(DUPLICATE_TOPIC); - stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC); - stub(metricsRegistry.newCounter(anyString(), anyString())) - .toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - stub(envelopeMock.getOffset()).toReturn("2"); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(0))); - stub(configMock.getInt("redis.database.duplicationstore.id", dupStoreId)).toReturn(dupStoreId); - when(configMock.get("dedup.producer.include.ids", "")).thenReturn("dev.diskha.portal"); - stub(configMock.getList("dedup.producer.include.ids", new ArrayList<>())).toReturn(Arrays.asList("dev.diksha.portal")); - - - deDuplicationTask = new DeDuplicationTask(configMock, contextMock, deDupEngineMock); - } - - @Test - public void ShouldSendEventToSuccessTopicIfEventIsUnique() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.EventWithChecksumJson()); - - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - } - - @Test - public void ShouldSendEventsToDuplicateTopicIfEventIsDuplicate() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.EventWithChecksumJson()); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(false); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), DUPLICATE_TOPIC))); - } - - @Test - public void ShouldSendEventsToMalformedTopicIfEventIsMalformed() throws Exception { - - when(envelopeMock.getMessage()).thenReturn("{'metadata':{'checksum':'sajksajska'}"); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - @Test(expected = Exception.class) - public void ShouldSendEventsToFailedTopicForAnyUnhandledException() throws Exception { - - when(envelopeMock.getMessage()).thenReturn(EventFixture.EventWithChecksumJson()); - when(deDupEngineMock.isUniqueEvent(anyString())).thenThrow(new Exception()); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - } - - @Test - public void ShouldStoreChecksumIfEventIsUnique() throws Exception { - - when(envelopeMock.getMessage()).thenReturn(EventFixture.EventWithChecksumJson()); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(deDupEngineMock, times(1)).isUniqueEvent(anyString()); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - } - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - return true; - } - }; - } -} diff --git a/data-pipeline/de-normalization/pom.xml b/data-pipeline/de-normalization/pom.xml deleted file mode 100644 index 201f8201c3..0000000000 --- a/data-pipeline/de-normalization/pom.xml +++ /dev/null @@ -1,188 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - - org.ekstep.ecosystem.jobs - de-normalization - 0.1.7 - jar - De-normalizer - - Denormalise device, user and content data - - - - - org.apache.samza - samza-api - ${samza.version} - - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - com.google.guava - guava - 18.0 - - - redis.clients - jedis - 2.9.0 - - - com.datastax.cassandra - cassandra-driver-core - 3.1.0 - - - com.fiftyonred - mock-jedis - 0.4.0 - jar - test - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - diff --git a/data-pipeline/de-normalization/src/main/assembly/src.xml b/data-pipeline/de-normalization/src/main/assembly/src.xml deleted file mode 100644 index c01210c393..0000000000 --- a/data-pipeline/de-normalization/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/de-normalization.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-kv-rocksdb_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:de-normalization - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/de-normalization/src/main/config/de-normalization.properties b/data-pipeline/de-normalization/src/main/config/de-normalization.properties deleted file mode 100644 index b5e995aebd..0000000000 --- a/data-pipeline/de-normalization/src/main/config/de-normalization.properties +++ /dev/null @@ -1,101 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.DeNormalization - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__denormalization_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.DeNormalizationTask -task.inputs=kafka.__env__.telemetry.with_location -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -# YARN container configuration -# yarn.am.container.memory.mb=__yarn_am_container_mb__ -# yarn.container.memory.mb=__dedup_job_yarn_container_memory_mb__ - -output.success.topic.name=__env__.telemetry.denorm -output.failed.topic.name=__env__.telemetry.failed -output.malformed.topic.name=__env__.telemetry.malformed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,failed-message-count,skipped-message-count,error-message-count,cache-hit-count,cache-empty-values-count,user-cache-hit-count,expired-event-count - -# redis -# redis.host=localhost -redis.host=__metadata_redis_host__ -# redis.port=6379 -redis.port=__metadata_redis_port__ -redis.connection.max=2 -redis.connection.idle.max=2 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -location.db.redis.key.expiry.seconds=__location_db_redis_key_expiry_seconds__ - -# redis database index -redis.userDB.index=4 -redis.contentDB.index=5 -redis.dialcodeDB.index=6 - -# metadata fields -content.metadata.fields=__content_metadata_fields__ -user.metadata.fields=__user_metadata_fields__ -dialcode.metadata.fields=__dialcode_metadata_fields__ - -user.signin.type.default = __user_sigin_type_default__ -user.login.type.defalut = __user_login_type_default__ - -telemetry.ignore.period.months=6 -summary.filter.events=ME_WORKFLOW_SUMMARY \ No newline at end of file diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/AbstractFactory.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/AbstractFactory.java deleted file mode 100644 index b0b7fe0fd2..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/AbstractFactory.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.ekstep.ep.samza.domain; - -public interface AbstractFactory { - public T getInstance(String type); -} - diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/CollectionDataUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/CollectionDataUpdater.java deleted file mode 100644 index 4568aadd0b..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/CollectionDataUpdater.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.util.ContentDataCache; - -public class CollectionDataUpdater extends IEventUpdater { - - CollectionDataUpdater(ContentDataCache contentCache) { - this.dataCache = contentCache; - this.cacheType = "collection"; - } - - public void update(Event event){ - } -} \ No newline at end of file diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/ContentDataUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/ContentDataUpdater.java deleted file mode 100644 index dde475a52a..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/ContentDataUpdater.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.util.ContentDataCache; - -public class ContentDataUpdater extends IEventUpdater { - - ContentDataUpdater(ContentDataCache contentCache) { - this.dataCache = contentCache; - this.cacheType = "content"; - } - - public void update(Event event) { - } - -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/DialcodeDataUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/DialcodeDataUpdater.java deleted file mode 100644 index 52f9c30c86..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/DialcodeDataUpdater.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.util.DialCodeDataCache; - -public class DialcodeDataUpdater extends IEventUpdater { - - DialcodeDataUpdater(DialCodeDataCache dialcodeCache) { - this.dataCache = dialcodeCache; - this.cacheType = "dialcode"; - } - - public void update(Event event) { - } -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index 9b4d4e415a..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,191 +0,0 @@ -package org.ekstep.ep.samza.domain; - - -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.reader.NullableValue; -import org.ekstep.ep.samza.task.DeNormalizationConfig; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -import java.math.BigDecimal; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.*; - -public class Event extends Events { - private DateTimeFormatter dateFormat = DateTimeFormat.forPattern("yyyy-MM-dd").withZoneUTC(); - private DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - - public Event(Map map) { - super(map); - } - - public Long getEndTimestampOfDay(String date) { - Long ets = dateFormat.parseDateTime(date).plusHours(23).plusMinutes(59).plusSeconds(59).getMillis(); - return ets; - } - - public Long compareAndAlterEts() { - Long eventEts = ets(); - Long currentMillis = new Date().getTime(); - String currentDate = formatter.format(currentMillis); - Long endTsOfCurrentDate = getEndTimestampOfDay(currentDate); - if (eventEts > endTsOfCurrentDate) telemetry.add("ets", currentMillis); - return ets(); - } - - public Boolean isOlder(Integer periodInMonths) { - Long eventEts = ets(); - Long periodInMillis = new DateTime().minusMonths(periodInMonths).getMillis(); - if (eventEts < periodInMillis) return true; - else return false; - } - - - public String objectRollUpl1ID() { - if (objectFieldsPresent() && objectRollUpl1FieldsPresent()) { - return telemetry.read("object.rollup.l1").value(); - } else return null; - } - - - public boolean objectRollUpl1FieldsPresent() { - if (objectFieldsPresent()) { - String objectrollUpl1 = telemetry.read("object.rollup.l1").value(); - return null != objectrollUpl1 && !objectrollUpl1.isEmpty(); - } else return false; - } - - public boolean checkObjectIdNotEqualsRollUpl1Id() { - if (objectRollUpl1FieldsPresent() && !(objectID().equals(objectRollUpl1ID()))) - return true; - else - return false; - } - - public String getKey(String type) { - switch (type) { - case "device": - return did(); - case "user": - if ("system".equalsIgnoreCase(actorType())) { - return ""; - } else { - return actorId(); - } - case "content": - return objectID(); - case "collection": - return objectRollUpl1ID(); - default: - return ""; - } - } - - public void addMetaData(String type, Map newData) { - switch (type) { - case "device": - addDeviceData(newData); - break; - case "user": - addUserData(newData); - break; - case "content": - addContentData(newData); - break; - case "dialcode": - addDialCodeData(newData); - break; - case "collection": - addCollectionData(newData); - default: - break; - - } - } - - public Map addISOStateCodeToDeviceData(Map deviceData) { - // add new statecode field - String statecode = deviceData.get("statecode").toString(); - if (statecode != null && !statecode.isEmpty()) { - String iso3166statecode = "IN-" + statecode; - deviceData.put("iso3166statecode", iso3166statecode); - return deviceData; - } else return deviceData; - } - - public void addDeviceData(Map newData) { - NullableValue> previousData = telemetry.read(path.deviceData()); - Map deviceData = previousData.isNull() ? new HashMap<>() : previousData.value(); - // add new statecode field - if (!deviceData.isEmpty()) { - deviceData = addISOStateCodeToDeviceData(deviceData); - } - deviceData.putAll(newData); - telemetry.add(path.deviceData(), deviceData); - setFlag(DeNormalizationConfig.getDeviceLocationJobFlag(), true); - } - - public void addUserData(Map newData) { - NullableValue> previousData = telemetry.read(path.userData()); - Map userdata = previousData.isNull() ? new HashMap<>() : previousData.value(); - userdata.putAll(newData); - telemetry.add(path.userData(), userdata); - if (newData.size() > 2) - setFlag(DeNormalizationConfig.getUserLocationJobFlag(), true); - else - setFlag(DeNormalizationConfig.getUserLocationJobFlag(), false); - } - - public void addContentData(Map newData) { - NullableValue> previousData = telemetry.read(path.contentData()); - Map contentData = previousData.isNull() ? new HashMap<>() : previousData.value(); - contentData.putAll(newData); - telemetry.add(path.contentData(), contentData); - setFlag(DeNormalizationConfig.getContentLocationJobFlag(), true); - } - - public void addCollectionData(Map newData) { - NullableValue> previousData = telemetry.read(path.collectionData()); - Map collectionData = previousData.isNull() ? new HashMap<>() : previousData.value(); - collectionData.putAll(newData); - telemetry.add(path.collectionData(), collectionData); - setFlag(DeNormalizationConfig.getCollectionLocationJobFlag(), true); - } - - public void addDialCodeData(Map newData) { - telemetry.add(path.dialCodeData(), newData); - setFlag(DeNormalizationConfig.getDialCodeLocationJobFlag(), true); - } - - public void setFlag(String key, Object value) { - NullableValue> telemetryFlag = telemetry.read(path.flags()); - Map flags = telemetryFlag.isNull() ? new HashMap<>() : telemetryFlag.value(); - if (!key.isEmpty()) flags.put(key, value); - telemetry.add(path.flags(), flags); - } - - public Map getFlag() { - NullableValue> telemetryFlag = telemetry.read(path.flags()); - Map flags = telemetryFlag.isNull() ? new HashMap<>() : telemetryFlag.value(); - return flags; - } - - public String getUpgradedVersion() { - BigDecimal updatedVer = BigDecimal.valueOf(Double.parseDouble(telemetry.read(path.ver()).value().toString())).add(BigDecimal.valueOf(0.1)); - return updatedVer.toString(); - } - - public void markFailure(String error, DeNormalizationConfig config) { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.denorm_processed", false); - - telemetry.addFieldIfAbsent("metadata", new HashMap()); - telemetry.add("metadata.denorm_error", error); - telemetry.add("metadata.src", config.jobName()); - } - - -} - diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/EventUpdaterFactory.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/EventUpdaterFactory.java deleted file mode 100644 index 300e3311c5..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/EventUpdaterFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.util.ContentDataCache; -import org.ekstep.ep.samza.util.DialCodeDataCache; -import org.ekstep.ep.samza.util.UserDataCache; - - -public class EventUpdaterFactory implements AbstractFactory { - - private ContentDataUpdater contentDataUpdater; - private UserDataUpdater userDataUpdater; - private DialcodeDataUpdater dialCodeDataUpdater; - private CollectionDataUpdater collectionDataUpdater; - - public EventUpdaterFactory(ContentDataCache contentDataCache, - UserDataCache userDataCache, - DialCodeDataCache dialCodeDataCache) { - - this.contentDataUpdater = new ContentDataUpdater(contentDataCache); - this.userDataUpdater = new UserDataUpdater(userDataCache); - this.dialCodeDataUpdater = new DialcodeDataUpdater(dialCodeDataCache); - this.collectionDataUpdater = new CollectionDataUpdater(contentDataCache); - } - - public IEventUpdater getInstance(String type) { - - switch (type) { - case "content-data-updater": - return this.contentDataUpdater; - case "user-data-updater": - return this.userDataUpdater; - case "dialcode-data-updater": - return this.dialCodeDataUpdater; - case "collection-data-updater": - return this.collectionDataUpdater; - default: - return null; - } - } -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/IEventUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/IEventUpdater.java deleted file mode 100644 index 26a7f7bb3c..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/IEventUpdater.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.task.DeNormalizationConfig; -import org.ekstep.ep.samza.util.DataCache; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -import java.util.Map; - -public abstract class IEventUpdater { - - static Logger LOGGER = new Logger(IEventUpdater.class); - DataCache dataCache; - String cacheType; - DateTimeFormatter df = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").withZoneUTC(); - DateTimeFormatter df1 = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS").withZoneUTC(); - - public abstract void update(Event event); - - public void update(Event event, String key) { - - if (key != null && !key.isEmpty()) { - Map data = dataCache.getData(key); - if (data != null && !data.isEmpty()) { - event.addMetaData(cacheType, getConvertedData(data)); - } else { - event.setFlag(DeNormalizationConfig.getJobFlag(cacheType), false); - } - } - } - - private Map getConvertedData(Map data) { - if("content".equals(cacheType) || "collection".equals((cacheType))) - return getEpochConvertedContentDataMap(data); - else if("dialcode".equals(cacheType)) - return getEpochConvertedDialcodeDataMap(data); - else - return data; - - } - - private Long getTimestamp(String ts, DateTimeFormatter df) { - try { - return df.parseDateTime(ts).getMillis(); - } catch (Exception ex) { - return 0L; - } - } - - private Long getConvertedTimestamp(String ts) { - Long epochTs = getTimestamp(ts, df); - if (epochTs == 0) { - epochTs = getTimestamp(ts, df1); - } - return epochTs; - } - - - private Map getEpochConvertedContentDataMap(Map data) { - - Object lastSubmittedOn = data.get("lastsubmittedon"); - Object lastUpdatedOn = data.get("lastupdatedon"); - Object lastPublishedOn = data.get("lastpublishedon"); - if (lastSubmittedOn instanceof String) { - lastSubmittedOn = getConvertedTimestamp(lastSubmittedOn.toString()); - } - if (lastUpdatedOn instanceof String) { - lastUpdatedOn = getConvertedTimestamp(lastUpdatedOn.toString()); - } - if (lastPublishedOn instanceof String) { - lastPublishedOn = getConvertedTimestamp(lastPublishedOn.toString()); - } - data.put("lastsubmittedon", lastSubmittedOn); - data.put("lastupdatedon", lastUpdatedOn); - data.put("lastpublishedon", lastPublishedOn); - return data; - } - - - private Map getEpochConvertedDialcodeDataMap(Map data) { - - Object generatedOn = data.get("generatedon"); - Object publishedOn = data.get("publishedon"); - if (generatedOn instanceof String) { - generatedOn = getConvertedTimestamp(generatedOn.toString()); - } - if (publishedOn instanceof String) { - publishedOn = getConvertedTimestamp(publishedOn.toString()); - } - data.put("generatedon", generatedOn); - data.put("publishedon", publishedOn); - return data; - } -} - - diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/UserDataUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/UserDataUpdater.java deleted file mode 100644 index a2e64276db..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/UserDataUpdater.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.task.DeNormalizationConfig; -import org.ekstep.ep.samza.util.UserDataCache; - -import java.util.Map; - -public class UserDataUpdater extends IEventUpdater { - - private UserDataCache userDataCache; - - UserDataUpdater(UserDataCache userDataCache) { - this.userDataCache = userDataCache; - this.cacheType = "user"; - } - - public void update(Event event) { - Map userCacheData; - String userId = event.actorId(); - if (userId != null && !userId.isEmpty() && !"system".equalsIgnoreCase(event.actorType())) { - userCacheData = userDataCache.getUserData(userId); - if (userCacheData != null && !userCacheData.isEmpty()) { - event.addUserData(userCacheData); - } else { - event.setFlag(DeNormalizationConfig.getUserLocationJobFlag(), false); - } - } - } - -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/service/DeNormalizationService.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/service/DeNormalizationService.java deleted file mode 100644 index a2cd931e1e..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/service/DeNormalizationService.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.JsonSyntaxException; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.domain.EventUpdaterFactory; -import org.ekstep.ep.samza.task.DeNormalizationConfig; -import org.ekstep.ep.samza.task.DeNormalizationSink; -import org.ekstep.ep.samza.task.DeNormalizationSource; - -import java.util.List; - -public class DeNormalizationService { - - private static Logger LOGGER = new Logger(DeNormalizationService.class); - private final DeNormalizationConfig config; - private final EventUpdaterFactory eventUpdaterFactory; - - public DeNormalizationService(DeNormalizationConfig config, EventUpdaterFactory eventUpdaterFactory) { - this.config = config; - this.eventUpdaterFactory = eventUpdaterFactory; - } - - public void process(DeNormalizationSource source, DeNormalizationSink sink) { - try { - Event event = source.getEvent(); - String eid = event.eid(); - List summaryRouteEventPrefix = this.config.getSummaryFilterEvents(); - - if("ERROR".equals(eid)){ - LOGGER.debug(null,"Skipping denormalization as eid is ERROR"); - sink.toSuccessTopic(event); - return; - } - - if (summaryRouteEventPrefix.contains(eid)) { - denormEvent(event); - sink.toSuccessTopic(event); - } else if (eid.startsWith("ME_")) { - LOGGER.debug(null, "Ignoring as eid is other than WFS"); - sink.incrementSkippedCount(event); - } else { - // ignore past data (older than last X months) - if (event.isOlder(config.ignorePeriodInMonths())) { - sink.incExpiredEventCount(); - LOGGER.debug(null, "Ignoring as ets is older than N months"); - sink.toFailedTopic(event, "older than " + config.ignorePeriodInMonths() + " months"); - } else { - denormEvent(event); - sink.toSuccessTopic(event); - } - } - } catch(JsonSyntaxException e){ - e.printStackTrace(); - LOGGER.error(null, "INVALID EVENT: " + source.getMessage()); - sink.toMalformedTopic(source.getMessage()); - } - } - - private void denormEvent(Event event) { - - // correct future dates - event.compareAndAlterEts(); - - if ("dialcode".equalsIgnoreCase(event.objectType()) || "qr".equalsIgnoreCase(event.objectType())) { - // add dialcode details to the event where object.type = dialcode/qr - eventUpdaterFactory.getInstance("dialcode-data-updater").update(event, event.getKey("content")); - } else if ("user".equalsIgnoreCase(event.objectType())) { - // skipping since it is same as user-denorm - } - else { - // add content details to the event - eventUpdaterFactory.getInstance("content-data-updater").update(event, event.getKey("content")); - if(event.checkObjectIdNotEqualsRollUpl1Id()) { - eventUpdaterFactory.getInstance("collection-data-updater").update(event, event.getKey("collection")); - } - } - // add user details to the event - eventUpdaterFactory.getInstance("user-data-updater").update(event); - - // add device details to the event -// eventUpdaterFactory.getInstance("device-data-updater").update(event); - } -} \ No newline at end of file diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationConfig.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationConfig.java deleted file mode 100644 index 2372629928..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationConfig.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.ekstep.ep.samza.task; - - -import org.apache.samza.config.Config; - -import java.util.ArrayList; -import java.util.List; - -public class DeNormalizationConfig { - - private final String JOB_NAME = "DeNormalization"; - private static final String deviceDataJobFlag = "device_data_retrieved"; - private static final String userDataJobFlag = "user_data_retrieved"; - private static final String contentDataJobFlag = "content_data_retrieved"; - private static final String dialCodeDataJobFlag = "dialcode_data_retrieved"; - private static final String collectionDataJobFlag = "collection_data_retrieved"; - - private String successTopic; - private String failedTopic; - private String malformedTopic; - private Integer ignorePeriodInMonths; - private List summaryFilterEvents; - - - public DeNormalizationConfig(Config config) { - successTopic = config.get("output.success.topic.name", "telemetry.denorm"); - failedTopic = config.get("output.failed.topic.name", "telemetry.failed"); - malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed"); - ignorePeriodInMonths = config.getInt("telemetry.ignore.period.months", 6); - List defaultSummaryEvents = new ArrayList(); - defaultSummaryEvents.add("ME_WORKFLOW_SUMMARY"); - summaryFilterEvents = config.getList("summary.filter.events", defaultSummaryEvents); - } - - public String successTopic() { - return successTopic; - } - - public String failedTopic() { - return failedTopic; - } - - public String malformedTopic() { - return malformedTopic; - } - - public Integer ignorePeriodInMonths() { - return ignorePeriodInMonths; - } - - public String jobName() { - return JOB_NAME; - } - - public static String getJobFlag(String type) { - switch (type) { - case "device": - return deviceDataJobFlag; - case "user": - return userDataJobFlag; - case "content": - return contentDataJobFlag; - case "dialcode": - return dialCodeDataJobFlag; - case "collection": - return collectionDataJobFlag; - default: - return ""; - } - } - - public static String getDeviceLocationJobFlag() { - return deviceDataJobFlag; - } - - public static String getUserLocationJobFlag() { - return userDataJobFlag; - } - - public static String getContentLocationJobFlag() { - return contentDataJobFlag; - } - - public static String getCollectionLocationJobFlag() { return collectionDataJobFlag; } - public static String getDialCodeLocationJobFlag() { - return dialCodeDataJobFlag; - } - - public List getSummaryFilterEvents() { - return this.summaryFilterEvents; - } - -} \ No newline at end of file diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSink.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSink.java deleted file mode 100644 index b385c8ff7f..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSink.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - -public class DeNormalizationSink extends BaseSink { - - private DeNormalizationConfig config; - - public DeNormalizationSink(MessageCollector collector, JobMetrics metrics, DeNormalizationConfig config) { - super(collector, metrics); - this.config = config; - } - - public void toSuccessTopic(Event event) { - toTopic(config.successTopic(), event.did(), event.getJson()); - metrics.incSuccessCounter(); - } - - public void toFailedTopic(Event event, String failedMessage) { - event.markFailure(failedMessage, config); - toTopic(config.failedTopic(), event.did(), event.getJson()); - metrics.incFailedCounter(); - } - - public void toMalformedTopic(String message) { - toTopic(config.malformedTopic(), null, message); - metrics.incErrorCounter(); - } - - public void incrementSkippedCount(Event event) { - metrics.incSkippedCounter(); - } - - public void incExpiredEventCount() { - metrics.incExpiredEventCount(); - } - -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSource.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSource.java deleted file mode 100644 index 59a2c03b2a..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSource.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.gson.Gson; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; - -import java.util.Map; - -public class DeNormalizationSource { - static Logger LOGGER = new Logger(DeNormalizationSource.class); - - private IncomingMessageEnvelope envelope; - - public DeNormalizationSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - String message = (String) envelope.getMessage(); - @SuppressWarnings("unchecked") - Map jsonMap = (Map) new Gson().fromJson(message, Map.class); - return new Event(jsonMap); - } - - public String getMessage() { - return envelope.toString(); - } - -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationTask.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationTask.java deleted file mode 100644 index 044e6354df..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationTask.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.EventUpdaterFactory; -import org.ekstep.ep.samza.service.DeNormalizationService; -import org.ekstep.ep.samza.util.ContentDataCache; -import org.ekstep.ep.samza.util.DialCodeDataCache; -import org.ekstep.ep.samza.util.RedisConnect; -import org.ekstep.ep.samza.util.UserDataCache; - -public class DeNormalizationTask extends BaseSamzaTask { - - private DeNormalizationConfig config; - private UserDataCache userCache; - private ContentDataCache contentCache; - private DialCodeDataCache dialcodeCache; - private JobMetrics metrics; - private RedisConnect redisConnect; - private DeNormalizationService service; - - public DeNormalizationTask(Config config, TaskContext context, UserDataCache userCache, - ContentDataCache contentCache, DialCodeDataCache dialcodeCache, JobMetrics jobMetrics, RedisConnect redisConnect) { - init(config, context, userCache, contentCache, dialcodeCache, jobMetrics, redisConnect); - } - - public DeNormalizationTask() { - - } - - @Override - public void init(Config config, TaskContext context) { - init(config, context, userCache, contentCache, dialcodeCache, metrics, redisConnect); - } - - - public void init(Config config, TaskContext context, UserDataCache userCache, ContentDataCache contentCache, - DialCodeDataCache dialcodeCache, JobMetrics jobMetrics, RedisConnect redisConnect) { - - this.config = new DeNormalizationConfig(config); - this.metrics = jobMetrics == null ? new JobMetrics(context, this.config.jobName()) : jobMetrics; - this.userCache = userCache == null ? new UserDataCache(config, metrics, redisConnect) : userCache; - this.contentCache = contentCache == null ? new ContentDataCache(config, metrics) : contentCache; - this.dialcodeCache = dialcodeCache == null ? new DialCodeDataCache(config, metrics) : dialcodeCache; - service = new DeNormalizationService(this.config, new EventUpdaterFactory(this.contentCache, this.userCache, this.dialcodeCache)); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) { - DeNormalizationSource source = new DeNormalizationSource(envelope); - DeNormalizationSink sink = new DeNormalizationSink(collector, metrics, config); - service.process(source, sink); - } - -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/ContentDataCache.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/ContentDataCache.java deleted file mode 100644 index 998b1eec6d..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/ContentDataCache.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.ekstep.ep.samza.util; - -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.core.JobMetrics; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public class ContentDataCache extends DataCache { - - public ContentDataCache(Config config, JobMetrics metrics) { - - super(config.getList("content.metadata.fields", Arrays.asList("name", "objectType", "contentType", "mediaType", "language", "medium", "mimeType", "createdBy", "createdFor", "framework", "board", "subject", "status", "pkgVersion", "lastSubmittedOn", "lastUpdatedOn", "lastPublishedOn"))); - this.databaseIndex = config.getInt("redis.contentDB.index", 5); - this.redisConnect = new RedisConnect(config); - this.redisConnection = this.redisConnect.getConnection(databaseIndex); - this.metrics = metrics; - } -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DataCache.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DataCache.java deleted file mode 100644 index 8219ff22c5..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DataCache.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.ekstep.ep.samza.util; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.exceptions.JedisException; -import java.lang.reflect.Type; -import java.util.*; - -public class DataCache { - - private static Logger LOGGER = new Logger(DataCache.class); - - protected RedisConnect redisConnect; - protected int databaseIndex; - protected Jedis redisConnection; - protected JobMetrics metrics; - protected List fieldsList; - private Gson gson = new Gson(); - - public DataCache(List fieldsList) { - this.fieldsList = fieldsList; - } - - public Map getData(String key) { - Map cacheDataMap; - try { - cacheDataMap = getDataFromCache(key); - } catch (JedisException ex) { - LOGGER.error("", "Exception when retrieving data from redis cache ", ex); - - this.redisConnection.close(); - this.redisConnection = redisConnect.getConnection(databaseIndex); - cacheDataMap = getDataFromCache(key); - } - if (cacheDataMap != null && !cacheDataMap.isEmpty()) { - metrics.incCacheHitCounter(); - } - return cacheDataMap; - } - - private Map getDataFromCache(String key) { - Map cacheData = new HashMap<>(); - String dataNode = redisConnection.get(key); - if(dataNode != null && !dataNode.isEmpty()) { - Type type = new TypeToken>() {}.getType(); - Map parsedData = gson.fromJson(dataNode, type); - parsedData.keySet().retainAll(fieldsList); - parsedData.values().removeAll(Collections.singleton("")); - for (Map.Entry entry : parsedData.entrySet()) { - cacheData.put(entry.getKey().toLowerCase().replace("_", ""), entry.getValue()); - } - } - return cacheData; - } - - public List getData(List keys) { - List list = new ArrayList<>(); - for (String entry : keys) { - Map data = getData(entry); - if (data != null && !data.isEmpty()) { - list.add(data); - } - } - return list; - } -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DialCodeDataCache.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DialCodeDataCache.java deleted file mode 100644 index a9e988b74d..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DialCodeDataCache.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.ekstep.ep.samza.util; - -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.core.JobMetrics; - -import java.util.Arrays; - -public class DialCodeDataCache extends DataCache { - - - public DialCodeDataCache(Config config, JobMetrics metrics) { - - super(config.getList("dialcode.metadata.fields", Arrays.asList("identifier", "channel", "batchcode", "publisher", "generated_on", "published_on", "status"))); - this.databaseIndex = config.getInt("redis.dialcodeDB.index", 6); - this.redisConnect = new RedisConnect(config); - this.redisConnection = this.redisConnect.getConnection(databaseIndex); - this.metrics = metrics; - } - -} diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/UserDataCache.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/UserDataCache.java deleted file mode 100644 index 86471c8e55..0000000000 --- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/UserDataCache.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.ekstep.ep.samza.util; - -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.exceptions.JedisException; - -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -public class UserDataCache extends DataCache { - - private static Logger LOGGER = new Logger(UserDataCache.class); - - private RedisConnect redisPool; - private Jedis redisConnection; - private String userSignInTypeDefault; - private String userLoginInTypeDefault; - private Type mapType = new TypeToken>() { - }.getType(); - private Gson gson = new Gson(); - private JobMetrics metrics; - private int databaseIndex; - - - public UserDataCache(Config config, JobMetrics metrics, RedisConnect redisConnect) { - - super(config.getList("user.metadata.fields", Arrays.asList("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype"))); - this.metrics = metrics; - this.databaseIndex = config.getInt("redis.userDB.index", 4); - this.redisPool = null == redisConnect ? new RedisConnect(config) : redisConnect; - this.redisConnection = this.redisPool.getConnection(databaseIndex); - this.userSignInTypeDefault = config.get("user.signin.type.default", "Anonymous"); - this.userLoginInTypeDefault = config.get("user.login.type.default", "NA"); - } - - public Map getUserData(String userId) { - if ("anonymous".equalsIgnoreCase(userId)) return null; - Map userDataMap; - try { - userDataMap = getUserDataFromCache(userId); - } catch (JedisException ex) { - this.redisConnection.close(); - this.redisConnection = redisPool.getConnection(databaseIndex); - userDataMap = getUserDataFromCache(userId); - } - if (null != userDataMap && !userDataMap.isEmpty()) { - userDataMap.keySet().retainAll(this.fieldsList); - metrics.incUserCacheHitCount(); - } - userDataMap = getUserSigninLoginDetails(userDataMap); - - if (userDataMap.size() <= 2) { //Since SigninType and LoginType are default values, incrementing no data metric only if other user details are not present - metrics.incEmptyCacheValueCounter(); - } - return userDataMap; - } - - private Map getUserSigninLoginDetails(Map userDataMap) { - Map userMap = null!=userDataMap ? userDataMap : new HashMap<>(); - if (!userMap.containsKey("usersignintype")) { - userMap.put("usersignintype", userSignInTypeDefault); - } - if (!userMap.containsKey("userlogintype")) - userMap.put("userlogintype", userLoginInTypeDefault); - return userMap; - } - - private Map getUserDataFromCache(String userId) { - Map cacheData = new HashMap<>(); - String data = redisConnection.get(userId); - if (data != null && !data.isEmpty()) { - cacheData = gson.fromJson(data, mapType); - } - return cacheData; - } -} diff --git a/data-pipeline/de-normalization/src/main/resources/log4j.xml b/data-pipeline/de-normalization/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/de-normalization/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/ContentDataUpdaterTest.java b/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/ContentDataUpdaterTest.java deleted file mode 100644 index bddf1087f8..0000000000 --- a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/ContentDataUpdaterTest.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.fiftyonred.mock_jedis.MockJedis; -import com.google.gson.Gson; -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.util.ContentDataCache; -import org.ekstep.ep.samza.util.DataCache; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.Before; -import org.junit.Test; -import redis.clients.jedis.Jedis; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.stub; - -public class ContentDataUpdaterTest { - - private Config configMock; - private RedisConnect redisConnectMock; - private ContentDataUpdater contentDataUpdater; - private ContentDataCache dataCacheMock; - private Jedis jedisMock; - - @SuppressWarnings("unchecked") - @Before - public void setUp() { - - jedisMock = new MockJedis("test"); - configMock = mock(Config.class); - redisConnectMock = mock(RedisConnect.class); - jedisMock.flushAll(); - } - - @Test - public void shouldUpdateEventWithConvertedTimeFields() throws Exception { - Map data = new HashMap(); - data.put("lastupdatedon", "2019-02-12T11:35:35.919+0000"); - data.put("lastsubmittedon", "2019-02-12T11:35:35.919+0000"); - data.put("lastpublishedon", "2019-02-12T11:35:35.919"); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - dataCacheMock = mock(ContentDataCache.class); - contentDataUpdater = new ContentDataUpdater(dataCacheMock); - stub(dataCacheMock.getData("test-content-1")).toReturn(data); - - Map objectMap = new HashMap(); objectMap.put("id", "test-content-1"); objectMap.put("ver", "1.0"); - objectMap.put("type", "Content"); - Map eventMap = new HashMap(); - eventMap.put("object", objectMap); - Event input = new Event(eventMap); - contentDataUpdater.update(input, "test-content-1"); - String output = input.toString(); - assertTrue(output.contains("lastsubmittedon=1549971335919")); - assertTrue(output.contains("lastpublishedon=1549971335919")); - assertTrue(output.contains("lastupdatedon=1549971335919")); - } - - @Test - public void shouldUpdateEventWithOutConvertion() throws Exception { - Map data = new HashMap(); - data.put("lastupdatedon", 1549971335919L); - data.put("lastsubmittedon", 1549971335919L); - data.put("lastpublishedon", 1.549971335919E12); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - dataCacheMock = mock(ContentDataCache.class); - contentDataUpdater = new ContentDataUpdater(dataCacheMock); - stub(dataCacheMock.getData("test-content-1")).toReturn(data); - - Map objectMap = new HashMap(); objectMap.put("id", "test-content-1"); objectMap.put("ver", "1.0"); - objectMap.put("type", "Content"); - Map eventMap = new HashMap(); - eventMap.put("object", objectMap); - Event input = new Event(eventMap); - contentDataUpdater.update(input, "test-content-1"); - String output = input.toString(); - assertTrue(output.contains("lastsubmittedon=1549971335919")); - assertTrue(output.contains("lastpublishedon=1.549971335919E12")); - assertTrue(output.contains("lastupdatedon=1549971335919")); - } - - @Test - public void shouldUpdateEventWith0ForIncorrectDateFormat() throws Exception { - Map data = new HashMap(); - data.put("lastupdatedon", "2019-02-12"); - data.put("lastsubmittedon", "2019-02-12"); - data.put("lastpublishedon", "2019-02-12"); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - dataCacheMock = mock(ContentDataCache.class); - contentDataUpdater = new ContentDataUpdater(dataCacheMock); - stub(dataCacheMock.getData("test-content-1")).toReturn(data); - - Map objectMap = new HashMap(); objectMap.put("id", "test-content-1"); objectMap.put("ver", "1.0"); - objectMap.put("type", "Content"); - Map eventMap = new HashMap(); - eventMap.put("object", objectMap); - Event input = new Event(eventMap); - contentDataUpdater.update(input, "test-content-1"); - String output = input.toString(); - assertTrue(output.contains("lastsubmittedon=0")); - assertTrue(output.contains("lastpublishedon=0")); - assertTrue(output.contains("lastupdatedon=0")); - } - - -} diff --git a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/EventTest.java b/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/EventTest.java deleted file mode 100644 index 1107365656..0000000000 --- a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/EventTest.java +++ /dev/null @@ -1,112 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; -import org.joda.time.DateTime; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class EventTest { - - @Test - public void shouldUpdateEventWithAlteredETSField() throws Exception { - - Map eventMap = new HashMap(); - Long ets = new DateTime().plusDays(2).getMillis(); - eventMap.put("ets", ets); - Event input = new Event(eventMap); - Long output = input.compareAndAlterEts(); - assertTrue(output < ets); - } - - @Test - public void shouldNotUpdateEventWithETSField() throws Exception { - - Map eventMap = new HashMap(); - Long ets = new DateTime().getMillis(); - eventMap.put("ets", ets); - Event input = new Event(eventMap); - Long output = input.compareAndAlterEts(); - assertTrue(output == ets); - } - - @Test - public void shouldReturnTrueForOlderData() throws Exception { - - Map eventMap = new HashMap(); - Long ets = new DateTime().minusMonths(7).getMillis(); - eventMap.put("ets", ets); - Event input = new Event(eventMap); - assertTrue(input.isOlder(6)); - } - - @Test - public void shouldReturnFalseForOlderData() throws Exception { - - Map eventMap = new HashMap(); - Long ets = new DateTime().minusMonths(3).getMillis(); - eventMap.put("ets", ets); - Event input = new Event(eventMap); - assertFalse(input.isOlder(6)); - } - - @Test - public void shouldReturnLatestVersion() throws Exception { - - Map eventMap = new HashMap(); - eventMap.put("ver", "3.0"); - Event input = new Event(eventMap); - String updatedVer = input.getUpgradedVersion(); - assertTrue(updatedVer.equals("3.1")); - } - - @Test - public void shouldReturnSummaryLatestVersion() throws Exception { - - Map eventMap = new HashMap(); - eventMap.put("ver", "2.1"); - Event input = new Event(eventMap); - String updatedVer = input.getUpgradedVersion(); - assertTrue(updatedVer.equals("2.2")); - } - - @Test - public void ShouldaddISOStateCodeToDeviceData() { - - String DEVICE_PROFILE_DETAILS = "{\n" + - "\"fcm_token\" : \"\",\n" + - "\"city\" : \"Bengaluru\",\n" + - "\"device_id\" : \"232455\",\n" + - "\"device_spec\" : \"{'os':'Android 6.0','cpu':'abi: armeabi-v7a ARMv7 Processor rev 4 (v7l)','make':'Motorola XT1706'}\",\n" + - "\"state\" : \"Karnataka\",\n" + - "\"uaspec\" : \"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\n" + - "\"country\" : \"India\",\n" + - "\"country_code\" : \"IN\",\n" + - "\"producer_id\" : \"dev.sunbird.portal\",\n" + - "\"state_code_custom\" : 29,\n" + - "\"state_code\" : \"KA\",\n" + - "\"state_custom\" : \"Karnataka\",\n" + - "\"district_custom\" : \"Karnataka\",\n" + - "\"first_access\": 1568377184000,\n" + - "\"api_last_updated_on\": 1568377184000,\n" + - "\"user_declared_district\" : \"Bengaluru\",\n" + - "\"user_declared_state\" : \"Karnataka\"\n" + - "}"; - - Event event = new Event(new Gson().fromJson(DEVICE_PROFILE_DETAILS, Map.class)); - Map deviceData = new HashMap(); - deviceData.put("first_access", "1568377184000"); - deviceData.put("device_id", "232455"); - deviceData.put("statecode", "KA"); - event.addDeviceData(deviceData); - Map result = event.addISOStateCodeToDeviceData(deviceData); - Assert.assertEquals(result.get("iso3166statecode"), "IN-KA"); - - } - -} diff --git a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index 55bfed6e2a..0000000000 --- a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,1421 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; -import org.joda.time.DateTime; - -import java.util.Map; - -public class EventFixture { - - public static Long current_ets = new DateTime().getMillis(); - - public static final String UNPARSABLE_START_EVENT = "{\n" + - " \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n" + - " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"mode\": \"WIFI\",\n" + - " \"ver\": \"12\",\n" + - " \"size\": 12.67,\n" + - " \"err\": \"\",\n" + - " \"referrer\": [\n" + - " {\n" + - " \"action\": \"INSTALL\",\n" + - " \"utmsource\": \"Ekstep\",\n" + - " \"utmmedium\": \"Portal\",\n" + - " \"utmterm\": \"December 2016\",\n" + - " \"utmcontent\": \"Ramayana\",\n" + - " \"utmcampaign\": \"Epics of India\"\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " \"eid\": \"START\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genie.android\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"ets\": 1454064092546,\n" + - " \"uid\": \"\",\n" + - " \"ver\": \"2.0\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"correlationid\",\n" + - " \"type\": \"correlationtype\"\n" + - " "; - - public static final String INTERACT_EVENT = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"loc\":\"xyz\",\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"object\": {\n" + - " \"id\": \"do_31249561779090227216256\",\n" + - " \"type\": \"Content\",\n" + - " \"version\": \"\"\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" + - "}", current_ets); - - public static final String INTERACT_EVENT_WITH_OBJECT_ROLLUP = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"loc\":\"xyz\",\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"object\": {\n" + - " \"id\": \"do_31277438304183091217888\",\n" + - " \"type\": \"TextBook\",\n" + - " \"version\": \"\",\n" + - " \"rollup\": { \n" + - " \"l1\": \"do_31249561779090227216256\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" + - "}", current_ets); - - public static final String INTERACT_EVENT_WITH_OBJECTID_EQUALS_ROLLUPID = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"loc\":\"xyz\",\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"object\": {\n" + - " \"id\": \"do_31277438304183091217888\",\n" + - " \"type\": \"TextBook\",\n" + - " \"version\": \"\",\n" + - " \"rollup\": { \n" + - " \"l1\": \"do_31277438304183091217888\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" + - "}", current_ets); - - public static final String INTERACT_EVENT_WITH_DEVICEDATA = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"60c02e78-64f9-400c-be3f-d91256d58cf1\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"id\": \"draftContentId\",\n" + - " \"type\": \"click\",\n" + - " \"pageid\": \"DraftContent\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"uid\": \"60c02e78-64f9-400c-be3f-d91256d58cf1\",\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.11.0\",\n" + - " \"pid\": \"sunbird-portal\",\n" + - " \"id\": \"staging.diksha.portal\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"workspace\",\n" + - " \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" + - " \"sid\": \"7ALbNUfErYtkGqPdyK0kjfRSETJXdKp4\",\n" + - " \"cdata\": [\n" + - " \n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"012550822176260096119\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"INTERACT:4b645cf3d2a587413603de219eece5ed\",\n" + - " \"object\": {\n" + - " \"ver\": \"1.0\",\n" + - " \"id\": \"do_21269871962088243212780\",\n" + - " \"type\": \"draft\",\n" + - " \"rollup\": {\n" + - " \n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"012550822176260096119\"\n" + - " ],\n" + - " \"syncts\": 1.550140278251E12,\n" + - " \"@timestamp\": \"2019-02-14T10:31:18.251Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true,\n" + - " \"device_location_retrieved\": true,\n" + - " \"user_location_retrieved\": false\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-02-14T10:23:54.456+0000\",\n" + - " \"devicedata\": {\n" + - " \"country\": \"India\",\n" + - " \"city\": \"Bengaluru\",\n" + - " \"countrycode\": \"IN\",\n" + - " \"state\": \"Karnataka\",\n" + - " \"statecode\": \"KA\"\n" + - " },\n" + - " \"userdata\": {\n" + - " \"district\": \"\",\n" + - " \"state\": \"\"\n" + - " }\n" + - "}", current_ets); - - public static final String INTERACT_EVENT_WITH_EMPTY_LOC = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"60c02e78-64f9-400c-be3f-d91256d58cf1\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"id\": \"draftContentId\",\n" + - " \"type\": \"click\",\n" + - " \"pageid\": \"DraftContent\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"uid\": \"60c02e78-64f9-400c-be3f-d91256d58cf1\",\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.11.0\",\n" + - " \"pid\": \"sunbird-portal\",\n" + - " \"id\": \"staging.diksha.portal\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"workspace\",\n" + - " \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" + - " \"sid\": \"7ALbNUfErYtkGqPdyK0kjfRSETJXdKp4\",\n" + - " \"cdata\": [\n" + - " \n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"012550822176260096119\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"INTERACT:4b645cf3d2a587413603de219eece5ed\",\n" + - " \"object\": {\n" + - " \"ver\": \"1.0\",\n" + - " \"id\": \"do_21269871962088243212780\",\n" + - " \"type\": \"draft\",\n" + - " \"rollup\": {\n" + - " \n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"012550822176260096119\"\n" + - " ],\n" + - " \"syncts\": 1.550140278251E12,\n" + - " \"@timestamp\": \"2019-02-14T10:31:18.251Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true,\n" + - " \"device_location_retrieved\": true,\n" + - " \"user_location_retrieved\": false\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-02-14T10:23:54.456+0000\",\n" + - " \"devicedata\": {\n" + - " \"country\": \"\",\n" + - " \"city\": \"\",\n" + - " \"countrycode\": \"\",\n" + - " \"state\": \"\",\n" + - " \"statecode\": \"\"\n" + - " },\n" + - " \"userdata\": {\n" + - " \"district\": \"\",\n" + - " \"state\": \"\"\n" + - " }\n" + - "}", current_ets); - - public static final String INTERACT_EVENT_WITHOUT_DID = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"loc\":\"xyz\",\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"object\": {\n" + - " \"id\": \"do_31249561779090227216256\",\n" + - " \"type\": \"Content\",\n" + - " \"version\": \"\"\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" + - "}", current_ets); - - public static final String INTERACT_EVENT_WITH_ACTOR_AS_SYSTEM = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"System\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"loc\":\"xyz\",\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"object\": {\n" + - " \"id\": \"do_31249561779090227216256\",\n" + - " \"type\": \"Content\",\n" + - " \"version\": \"\"\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" + - "}", current_ets); - - public static final String INTERACT_EVENT_WITHOUT_OBJECT = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"loc\":\"xyz\",\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" + - "}", current_ets); - - public static final String SEARCH_EVENT_WITHOUT_DIALCODE = String.format("{\n" + - " \"eid\": \"SEARCH\",\n" + - " \"ets\": %s,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"LP.1543688467595.c66eb854-82c4-4b73-8902-82209be643ed\",\n" + - " \"actor\": {\n" + - " \"id\": \"org.ekstep.learning.platform\",\n" + - " \"type\": \"System\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.ntp.learning.platform\",\n" + - " \"pid\": \"search-service\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"env\": \"search\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"size\": 51,\n" + - " \"query\": \"\",\n" + - " \"filters\": {\n" + - " \"contentType\": \"Course\",\n" + - " \"objectType\": [\n" + - " \"Content\",\n" + - " \"ContentImage\"\n" + - " ],\n" + - " \"status\": [\n" + - " \"Live\"\n" + - " ],\n" + - " \"channel\": {\n" + - " \"ne\": [\n" + - " \"0124433024890224640\",\n" + - " \"0124446042259128320\",\n" + - " \"0124487522476933120\",\n" + - " \"0125840271570288640\",\n" + - " \"0124453662635048969\"\n" + - " ]\n" + - " },\n" + - " \"framework\": {},\n" + - " \"mimeType\": {},\n" + - " \"resourceType\": {}\n" + - " },\n" + - " \"sort\": {},\n" + - " \"type\": \"content\",\n" + - " \"topn\": [\n" + - " {\n" + - " \"identifier\": \"do_312461520193110016213245\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"syncts\": 1543688467885,\n" + - " \"@timestamp\": \"2018-12-01T18:21:07.885Z\",\n" + - " \"ts\": \"2018-12-01T18:21:07.595+0000\"\n" + - " }", current_ets); - - public static final String SEARCH_EVENT_WITH_DIALCODE_AS_STRING = String.format("{\n" + - " \"eid\": \"SEARCH\",\n" + - " \"ets\": %s,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" + - " \"actor\": {\n" + - " \"id\": \"org.ekstep.learning.platform\",\n" + - " \"type\": \"System\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.ntp.learning.platform\",\n" + - " \"pid\": \"search-service\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"env\": \"search\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"size\": 1,\n" + - " \"query\": \"\",\n" + - " \"filters\": {\n" + - " \"dialcodes\": \"8ZEDTP\",\n" + - " \"channel\": {\n" + - " \"ne\": [\n" + - " \"0124433024890224640\",\n" + - " \"0124446042259128320\",\n" + - " \"0124487522476933120\",\n" + - " \"0125840271570288640\",\n" + - " \"0124453662635048969\"\n" + - " ]\n" + - " },\n" + - " \"framework\": {},\n" + - " \"contentType\": {},\n" + - " \"mimeType\": {},\n" + - " \"resourceType\": {},\n" + - " \"objectType\": [\n" + - " \"Content\",\n" + - " \"ContentImage\"\n" + - " ]\n" + - " },\n" + - " \"sort\": {},\n" + - " \"type\": \"content\",\n" + - " \"topn\": [\n" + - " {\n" + - " \"identifier\": \"do_312531599251210240213439\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"syncts\": 1543688463882,\n" + - " \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" + - " \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" + - " }", current_ets); - - public static final String SEARCH_EVENT_WITH_DIALCODE_AS_LIST = String.format("{\n" + - " \"eid\": \"SEARCH\",\n" + - " \"ets\": %s,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" + - " \"actor\": {\n" + - " \"id\": \"org.ekstep.learning.platform\",\n" + - " \"type\": \"System\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.ntp.learning.platform\",\n" + - " \"pid\": \"search-service\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"env\": \"search\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"size\": 1,\n" + - " \"query\": \"\",\n" + - " \"filters\": {\n" + - " \"dialcodes\": [\"8ZEDTP\", \"4ZEDTP\"],\n" + - " \"channel\": {\n" + - " \"ne\": [\n" + - " \"0124433024890224640\",\n" + - " \"0124446042259128320\",\n" + - " \"0124487522476933120\",\n" + - " \"0125840271570288640\",\n" + - " \"0124453662635048969\"\n" + - " ]\n" + - " },\n" + - " \"framework\": {},\n" + - " \"contentType\": {},\n" + - " \"mimeType\": {},\n" + - " \"resourceType\": {},\n" + - " \"objectType\": [\n" + - " \"Content\",\n" + - " \"ContentImage\"\n" + - " ]\n" + - " },\n" + - " \"sort\": {},\n" + - " \"type\": \"content\",\n" + - " \"topn\": [\n" + - " {\n" + - " \"identifier\": \"do_312531599251210240213439\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"syncts\": 1543688463882,\n" + - " \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" + - " \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" + - " }", current_ets); - - public static final String IMPRESSION_EVENT_WITH_DIALCODE_AS_OBJECT = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"anonymous\"\n" + - " },\n" + - " \"eid\": \"IMPRESSION\",\n" + - " \"edata\": {\n" + - " \"visits\": [\n" + - " \n" + - " ],\n" + - " \"type\": \"view\",\n" + - " \"pageid\": \"dialcode\",\n" + - " \"subtype\": \"pageexit\",\n" + - " \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"uid\": \"anonymous\",\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.7.1\",\n" + - " \"pid\": \"sunbird-portal\",\n" + - " \"id\": \"prod.diksha.portal\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"public\",\n" + - " \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" + - " \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" + - " \"cdata\": [\n" + - " \n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" + - " \"object\": {\n" + - " \"ver\": \"1.0\",\n" + - " \"id\": \"977D3I\",\n" + - " \"type\": \"dialcode\",\n" + - " \"rollup\": {\n" + - " \n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " ],\n" + - " \"syncts\": 1550501698819,\n" + - " \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true,\n" + - " \"ldata_retrieved\": false\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" + - " \"ldata\": {\n" + - " \"country_code\": \"\",\n" + - " \"country\": \"\",\n" + - " \"city\": \"\",\n" + - " \"state\": \"\",\n" + - " \"state_code\": \"\"\n" + - " } \n" + - "}", current_ets); - - public static final String IMPRESSION_EVENT_WITH_QR_AS_OBJECT = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"anonymous\"\n" + - " },\n" + - " \"eid\": \"IMPRESSION\",\n" + - " \"edata\": {\n" + - " \"visits\": [\n" + - " \n" + - " ],\n" + - " \"type\": \"view\",\n" + - " \"pageid\": \"dialcode\",\n" + - " \"subtype\": \"pageexit\",\n" + - " \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"uid\": \"anonymous\",\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.7.1\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"public\",\n" + - " \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" + - " \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" + - " \"cdata\": [\n" + - " \n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" + - " \"object\": {\n" + - " \"ver\": \"1.0\",\n" + - " \"id\": \"977D3I\",\n" + - " \"type\": \"qr\",\n" + - " \"rollup\": {\n" + - " \n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " ],\n" + - " \"syncts\": 1550501698819,\n" + - " \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true,\n" + - " \"ldata_retrieved\": false\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" + - " \"ldata\": {\n" + - " \"country_code\": \"\",\n" + - " \"country\": \"\",\n" + - " \"city\": \"\",\n" + - " \"state\": \"\",\n" + - " \"state_code\": \"\"\n" + - " } \n" + - "}", current_ets); - - public static final String IMPRESSION_EVENT_WITH_DIALCODE_CAMELCASE_AS_OBJECT = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"anonymous\"\n" + - " },\n" + - " \"eid\": \"IMPRESSION\",\n" + - " \"edata\": {\n" + - " \"visits\": [\n" + - " \n" + - " ],\n" + - " \"type\": \"view\",\n" + - " \"pageid\": \"dialcode\",\n" + - " \"subtype\": \"pageexit\",\n" + - " \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"uid\": \"anonymous\",\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.7.1\",\n" + - " \"pid\": \"sunbird-portal\",\n" + - " \"id\": \"prod.diksha.portal\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"public\",\n" + - " \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" + - " \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" + - " \"cdata\": [\n" + - " \n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" + - " \"object\": {\n" + - " \"ver\": \"1.0\",\n" + - " \"id\": \"977D3I\",\n" + - " \"type\": \"DialCode\",\n" + - " \"rollup\": {\n" + - " \n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " ],\n" + - " \"syncts\": 1550501698819,\n" + - " \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true,\n" + - " \"ldata_retrieved\": false\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" + - " \"ldata\": {\n" + - " \"country_code\": \"\",\n" + - " \"country\": \"\",\n" + - " \"city\": \"\",\n" + - " \"state\": \"\",\n" + - " \"state_code\": \"\"\n" + - " } \n" + - "}", current_ets); - - public static final String SEARCH_EVENT_WITH_CAMELCASE_DIALCODE_AS_STRING = String.format("{\n" + - " \"eid\": \"SEARCH\",\n" + - " \"ets\": %s,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" + - " \"actor\": {\n" + - " \"id\": \"org.ekstep.learning.platform\",\n" + - " \"type\": \"System\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.ntp.learning.platform\",\n" + - " \"pid\": \"search-service\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"env\": \"search\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"size\": 1,\n" + - " \"query\": \"\",\n" + - " \"filters\": {\n" + - " \"dialCodes\": \"8ZEDTP\",\n" + - " \"channel\": {\n" + - " \"ne\": [\n" + - " \"0124433024890224640\",\n" + - " \"0124446042259128320\",\n" + - " \"0124487522476933120\",\n" + - " \"0125840271570288640\",\n" + - " \"0124453662635048969\"\n" + - " ]\n" + - " },\n" + - " \"framework\": {},\n" + - " \"contentType\": {},\n" + - " \"mimeType\": {},\n" + - " \"resourceType\": {},\n" + - " \"objectType\": [\n" + - " \"Content\",\n" + - " \"ContentImage\"\n" + - " ]\n" + - " },\n" + - " \"sort\": {},\n" + - " \"type\": \"content\",\n" + - " \"topn\": [\n" + - " {\n" + - " \"identifier\": \"do_312531599251210240213439\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"syncts\": 1543688463882,\n" + - " \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" + - " \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" + - " }", current_ets); - - public static final String SEARCH_EVENT_WITH_FUTURE_ETS = "{\n" + - " \"eid\": \"SEARCH\",\n" + - " \"ets\": 2530937155000,\n" + - " \"ver\": \"2.1\",\n" + - " \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" + - " \"actor\": {\n" + - " \"id\": \"org.ekstep.learning.platform\",\n" + - " \"type\": \"System\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.ntp.learning.platform\",\n" + - " \"pid\": \"search-service\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"env\": \"search\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"size\": 1,\n" + - " \"query\": \"\",\n" + - " \"filters\": {\n" + - " \"dialCodes\": \"8ZEDTP\",\n" + - " \"channel\": {\n" + - " \"ne\": [\n" + - " \"0124433024890224640\",\n" + - " \"0124446042259128320\",\n" + - " \"0124487522476933120\",\n" + - " \"0125840271570288640\",\n" + - " \"0124453662635048969\"\n" + - " ]\n" + - " },\n" + - " \"framework\": {},\n" + - " \"contentType\": {},\n" + - " \"mimeType\": {},\n" + - " \"resourceType\": {},\n" + - " \"objectType\": [\n" + - " \"Content\",\n" + - " \"ContentImage\"\n" + - " ]\n" + - " },\n" + - " \"sort\": {},\n" + - " \"type\": \"content\",\n" + - " \"topn\": [\n" + - " {\n" + - " \"identifier\": \"do_312531599251210240213439\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"syncts\": 1543688463882,\n" + - " \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" + - " \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" + - " }"; - - public static final String SEARCH_EVENT_WITH_OLDER_ETS = "{\n" + - " \"eid\": \"SEARCH\",\n" + - " \"ets\": 1520753300000,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" + - " \"actor\": {\n" + - " \"id\": \"org.ekstep.learning.platform\",\n" + - " \"type\": \"System\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.ntp.learning.platform\",\n" + - " \"pid\": \"search-service\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"env\": \"search\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"size\": 1,\n" + - " \"query\": \"\",\n" + - " \"filters\": {\n" + - " \"dialCodes\": \"8ZEDTP\",\n" + - " \"channel\": {\n" + - " \"ne\": [\n" + - " \"0124433024890224640\",\n" + - " \"0124446042259128320\",\n" + - " \"0124487522476933120\",\n" + - " \"0125840271570288640\",\n" + - " \"0124453662635048969\"\n" + - " ]\n" + - " },\n" + - " \"framework\": {},\n" + - " \"contentType\": {},\n" + - " \"mimeType\": {},\n" + - " \"resourceType\": {},\n" + - " \"objectType\": [\n" + - " \"Content\",\n" + - " \"ContentImage\"\n" + - " ]\n" + - " },\n" + - " \"sort\": {},\n" + - " \"type\": \"content\",\n" + - " \"topn\": [\n" + - " {\n" + - " \"identifier\": \"do_312531599251210240213439\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"syncts\": 1543688463882,\n" + - " \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" + - " \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" + - " }"; - - public static final String WFS_EVENT = String.format("{\n" + - " \"eid\": \"ME_WORKFLOW_SUMMARY\",\n" + - " \"ets\": %s,\n" + - " \"syncts\": 1.553952011044E12,\n" + - " \"ver\": \"1.1\",\n" + - " \"mid\": \"8D7C477F841B6F82937BBC6F61E7C2FD\",\n" + - " \"uid\": \"393407b1-66b1-4c86-9080-b2bce9842886\",\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"id\": \"AnalyticsDataPipeline\",\n" + - " \"ver\": \"1.0\",\n" + - " \"model\": \"WorkflowSummarizer\"\n" + - " },\n" + - " \"granularity\": \"SESSION\",\n" + - " \"date_range\": {\n" + - " \"from\": 1.553951969817E12,\n" + - " \"to\": 1.553951999398E12\n" + - " },\n" + - " \"cdata\": [\n" + - " \n" + - " ]\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.app\",\n" + - " \"ver\": \"2.1.45\",\n" + - " \"pid\": \"sunbird.app\"\n" + - " },\n" + - " \"sid\": \"4a58927b-ad4d-491e-a9e2-73b592a60a22\",\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"type\": \"session\",\n" + - " \"mode\": \"\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"interact_events_per_min\": 5.0,\n" + - " \"start_time\": 1.553951969817E12,\n" + - " \"interact_events_count\": 5.0,\n" + - " \"item_responses\": [\n" + - " \n" + - " ],\n" + - " \"end_time\": 1.553951999398E12,\n" + - " \"events_summary\": [\n" + - " {\n" + - " \"id\": \"START\",\n" + - " \"count\": 4.0\n" + - " },\n" + - " {\n" + - " \"id\": \"IMPRESSION\",\n" + - " \"count\": 4.0\n" + - " },\n" + - " {\n" + - " \"id\": \"INTERACT\",\n" + - " \"count\": 12.0\n" + - " },\n" + - " {\n" + - " \"id\": \"SHARE\",\n" + - " \"count\": 1.0\n" + - " }\n" + - " ],\n" + - " \"page_summary\": [\n" + - " {\n" + - " \"id\": \"dial-code-scan-result\",\n" + - " \"type\": \"view\",\n" + - " \"env\": \"home\",\n" + - " \"time_spent\": 4.46,\n" + - " \"visit_count\": 1.0\n" + - " },\n" + - " {\n" + - " \"id\": \"user-type-selection\",\n" + - " \"type\": \"search\",\n" + - " \"env\": \"home\",\n" + - " \"time_spent\": 6.08,\n" + - " \"visit_count\": 1.0\n" + - " },\n" + - " {\n" + - " \"id\": \"content-detail\",\n" + - " \"type\": \"detail\",\n" + - " \"env\": \"home\",\n" + - " \"time_spent\": 11.73,\n" + - " \"visit_count\": 1.0\n" + - " },\n" + - " {\n" + - " \"id\": \"qr-code-scanner\",\n" + - " \"type\": \"view\",\n" + - " \"env\": \"home\",\n" + - " \"time_spent\": 7.6,\n" + - " \"visit_count\": 1.0\n" + - " }\n" + - " ],\n" + - " \"time_diff\": 29.58,\n" + - " \"telemetry_version\": \"3.0\",\n" + - " \"env_summary\": [\n" + - " {\n" + - " \"env\": \"home\",\n" + - " \"time_spent\": 29.87,\n" + - " \"count\": 1.0\n" + - " }\n" + - " ],\n" + - " \"time_spent\": 29.6\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"object\": {\n" + - " \"id\": \"\",\n" + - " \"type\": \"\"\n" + - " },\n" + - " \"ts\": \"2019-03-31T03:07:04.736+0000\",\n" + - " \"devicedata\": {\n" + - " \"country\": \"\",\n" + - " \"city\": \"\",\n" + - " \"countrycode\": \"\",\n" + - " \"state\": \"\",\n" + - " \"statecode\": \"\"\n" + - " },\n" + - " \"flags\": {\n" + - " \"dv_processed\": false\n" + - " }\n" + - "}", current_ets); - - public static final String DEVICE_SUMMARY_EVENT = String.format("{\n" + - " \"eid\": \"ME_DEVICE_SUMMARY\",\n" + - " \"ets\": %s,\n" + - " \"syncts\": 1554101904592,\n" + - " \"ver\": \"1.0\",\n" + - " \"mid\": \"3ACBFD49BAFC74832A44347FF4F1E611\",\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"id\": \"AnalyticsDataPipeline\",\n" + - " \"ver\": \"1.0\",\n" + - " \"model\": \"DeviceSummary\"\n" + - " },\n" + - " \"granularity\": \"DAY\",\n" + - " \"date_range\": {\n" + - " \"from\": 1554101897620,\n" + - " \"to\": 1554101904209\n" + - " }\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"did\": \"3f2b155788bef740fe741c92b8d80cac\",\n" + - " \"channel\": \"01231711180382208027\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"firstAccess\": 1553574785315,\n" + - " \"dial_stats\": {\n" + - " \"total_count\": 0,\n" + - " \"success_count\": 0,\n" + - " \"failure_count\": 0\n" + - " },\n" + - " \"content_downloads\": 0,\n" + - " \"contents_played\": 0,\n" + - " \"total_ts\": 6,\n" + - " \"total_launches\": 1,\n" + - " \"unique_contents_played\": 0\n" + - " }\n" + - " },\n" + - " \"ts\": \"2019-04-02T03:07:09.447+0000\",\n" + - " \"devicedata\": {\n" + - " \"statecustomcode\": \"\",\n" + - " \"country\": \"\",\n" + - " \"city\": \"\",\n" + - " \"countrycode\": \"\",\n" + - " \"state\": \"\",\n" + - " \"statecode\": \"\",\n" + - " \"districtcustom\": \"\",\n" + - " \"statecustomname\": \"\"\n" + - " },\n" + - " \"flags\": {\n" + - " \"device_location_retrieved\": true\n" + - " }\n" + - " }", current_ets); - - public static final String ANY_STRING = "Hey Samza, Whats Up?"; - public static final String EMPTY_JSON = "{}"; - - public static final String LOG_EVENT = String.format("{\n" + - " \"@timestamp\":\"2019-03-20T00:00:01.176Z\",\n" + - "\"actor\":{\n" + - "\"id\":\"0b251080-3230-415e-a593-ab7c1fac7ae3\",\n" + - "\"type\":\"User\"\n" + - "},\n" + - "\"context\":{\n" + - "\"cdata\":[\n" + - - "],\n" + - "\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - /*"\"did\":\"923c675274cfbf19fd0402fe4d2c37afd597f0ab\",\n"+*/ - "\"env\":\"home\",\n" + - "\"pdata\":{\n" + - "\"id\":\"prod.diksha.app\",\n" + - "\"pid\":\"sunbird.app\",\n" + - "\"ver\":\"2.1.45\"\n" + - "},\n" + - "\"sid\":\"57b5b7ea-93c5-48d6-ba51-f5f9a3570ffe\"\n" + - "},\n" + - "\"edata\":{\n" + - "\"level\":\"INFO\",\n" + - "\"message\":\"content-detail\",\n" + - "\"params\":[\n" + - "{\n" + - "\"PopupType\":\"automatic\"\n" + - "}\n" + - "],\n" + - "\"type\":\"api_access\"\n" + - "},\n" + - "\"eid\":\"LOG\",\n" + - "\"ets\":%s2,\n" + - "\"flags\":{\n" + - "\"dd_processed\":true,\n" + - "\"tv_processed\":true\n" + - "},\n" + - "\"mid\":\"ca17e5bd-71d4-487a-92cd-0fb377e7a591\",\n" + - "\"syncts\":1.553040001176E12,\n" + - "\"tags\":[\n" + - "],\n" + - "\"type\":\"events\",\n" + - "\"ver\":\"3.0\"\n" + - " }", current_ets); - - public static final String ERROR_EVENT = String.format("{" + - " \"eid\":\"ERROR\"," + - " \"ets\":%s," + - " \"ver\":\"3.0\"," + - " \"mid\":\"LP.1553040097857.bf0e4e15-014e-4a22-ba00-e02ff3c38784\"," + - " \"actor\":{" + - " \"id\":\"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\"," + - " \"type\":\"System\"" + - " }," + - " \"context\":{" + - " \"channel\":\"01235953109336064029450\"," + - " \"pdata\":{" + - " \"id\":\"prod.ntp.learning.platform\"," + - " \"pid\":\"learning-service\"," + - " \"ver\":\"1.0\"" + - " }," + - " \"env\":\"framework\"" + - " }," + - " \"edata\":{" + - " \"err\":\"ERR_DATA_NOT_FOUND\"," + - " \"stacktrace\":\"ERR_DATA_NOT_FOUND: Data not found with id : null\n\tat\"," + - " \"errtype\":\"system\"" + - " }," + - " \"flags\":{" + - " \"tv_processed\":true," + - " \"dd_processed\":true" + - " }," + - " \"type\":\"events\"," + - " \"syncts\":1.553040098435E12," + - " \"@timestamp\":\"2019-03-20T00:01:38.435Z\"" + - "}", current_ets); - - public static final String TEST_LOG_EVENT = String.format("{\n" + - " \"@timestamp\":\"2019-03-20T00:00:01.176Z\",\n" + - "\"actor\":{\n" + - "\"id\":\"0b251080-3230-415e-a593-ab7c1fac7ae3\",\n" + - "\"type\":\"User\"\n" + - "},\n" + - "\"context\":{\n" + - "\"cdata\":[\n" + - - "],\n" + - "\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - "\"env\":\"home\",\n" + - "\"pdata\":{\n" + - "\"id\":\"prod.diksha.app\",\n" + - "\"pid\":\"sunbird.app\",\n" + - "\"ver\":\"2.1.45\"\n" + - "},\n" + - "\"sid\":\"57b5b7ea-93c5-48d6-ba51-f5f9a3570ffe\"\n" + - "},\n" + - "\"edata\":{\n" + - "\"level\":\"INFO\",\n" + - "\"message\":\"content-detail\",\n" + - "\"params\":[\n" + - "{\n" + - "\"PopupType\":\"automatic\"\n" + - "}\n" + - "],\n" + - "\"type\":\"view\"\n" + - "},\n" + - "\"eid\":\"LOG\",\n" + - "\"ets\":%s,\n" + - "\"flags\":{\n" + - "\"dd_processed\":true,\n" + - "\"tv_processed\":true\n" + - "},\n" + - "\"mid\":\"ca17e5bd-71d4-487a-92cd-0fb377e7a591\",\n" + - "\"syncts\":1.553040001176E12,\n" + - "\"tags\":[\n" + - "],\n" + - "\"type\":\"events\",\n" + - "\"ver\":\"3.0\"\n" + - " }", current_ets); - - public static final String AUDIT_EVENT = String.format("{\n" + - " \"actor\": {\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"eid\": \"AUDIT\",\n" + - " \"edata\": {\n" + - " \"state\": \"Create\",\n" + - " \"prevstate\": \"\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"metadata\": {\n" + - " \"checksum\": \"AUDIT:18413285-6bb0-40ed-b9d1-44a184a3a625\",\n" + - " \"source_eid\": \"GE_CREATE_USER\",\n" + - " \"source_mid\": \"18413285-6bb0-40ed-b9d1-44a184a3a625\",\n" + - " \"index_name\": \"telemetry-2019.06\",\n" + - " \"index_type\": \"events_v1\"\n" + - " },\n" + - " \"@timestamp\": \"2019-06-02T17:21:35.546Z\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"channel\": \"in.tnpilot\",\n" + - " \"env\": \"Genie\",\n" + - " \"sid\": \"54ff57dd-0f65-4001-90df-a2eed195bb86\",\n" + - " \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" + - " \"pdata\": {\n" + - " \"id\": \"tnpilot\",\n" + - " \"pid\": \"tnpilot\",\n" + - " \"ver\": \"1.0.25\"\n" + - " },\n" + - " \"cdata\": [\n" + - " \n" + - " ]\n" + - " },\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"v2_converted\": true,\n" + - " \"dd_processed\": true,\n" + - " \"ldata_obtained\": false,\n" + - " \"ldata_processed\": true,\n" + - " \"odn_skipped\": true\n" + - " },\n" + - " \"mid\": \"AUDIT:18413285-6bb0-40ed-b9d1-44a184a3a625\",\n" + - " \"object\": {\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\",\n" + - " \"type\": \"User\",\n" + - " \"parent\": {\n" + - " \n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-06-02T17:07:37.141+0000\"\n" + - "}", current_ets); - - public static String IMPRESSION_EVENT = String.format("{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"e9d73b7b-211a-43f4-8c01-cc9333757a9d\"\n" + - " },\n" + - " \"eid\": \"IMPRESSION\",\n" + - " \"edata\": {\n" + - " \"type\": \"view\",\n" + - " \"pageid\": \"splash\",\n" + - " \"uri\": \"splash\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": %s,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"ver\": \"2.1.91\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"home\",\n" + - " \"did\": \"f3b08a7848b85f50b611570e547093bae1e30f95\",\n" + - " \"cdata\": [],\n" + - " \"sid\": \"3e2f8b9c-1b19-4d5e-8c52-7e7ca1b24fdc\"\n" + - " },\n" + - " \"mid\": \"d07a626f-b1aa-48fd-89b4-73eb53012a0a\",\n" + - " \"object\": {\n" + - " \"id\": \"\",\n" + - " \"type\": \"\",\n" + - " \"version\": \"\",\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"syncts\": 1562146974715,\n" + - " \"@timestamp\": \"2019-07-03T09:42:54.715Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}", current_ets); - - public static Map getMap(String message) { - return (Map) new Gson().fromJson(message, Map.class); - } -} diff --git a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/task/DeNormalizationTaskTest.java b/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/task/DeNormalizationTaskTest.java deleted file mode 100644 index cb9b186d3a..0000000000 --- a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/task/DeNormalizationTaskTest.java +++ /dev/null @@ -1,728 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.fiftyonred.mock_jedis.MockJedis; -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.util.*; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.invocation.InvocationOnMock; -import redis.clients.jedis.Jedis; - -import java.lang.reflect.Type; -import java.util.*; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -public class DeNormalizationTaskTest { - - private static final String SUCCESS_TOPIC = "telemetry.with_denorm"; - private static final String FAILED_TOPIC = "telemetry.failed"; - private static final String MALFORMED_TOPIC = "telemetry.malformed"; - private static final Integer ignorePeriodInMonths = 6; - - private MessageCollector collectorMock; - private TaskContext contextMock; - private MetricsRegistry metricsRegistry; - private Counter counter; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private Config configMock; - private DeNormalizationTask deNormalizationTask; - - private ContentDataCache contentCacheMock; - private DialCodeDataCache dailcodeCacheMock; - private JobMetrics jobMetrics; - private Jedis jedisMock = new MockJedis("test"); - private Type mapType = new TypeToken>(){}.getType(); - - @SuppressWarnings("unchecked") - @Before - public void setUp() { - collectorMock = mock(MessageCollector.class); - contextMock = mock(TaskContext.class); - metricsRegistry = mock(MetricsRegistry.class); - counter = mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - configMock = mock(Config.class); - RedisConnect redisConnectMock = mock(RedisConnect.class); - contentCacheMock = mock(ContentDataCache.class); - dailcodeCacheMock = mock(DialCodeDataCache.class); - jobMetrics = mock(JobMetrics.class); - - - stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC); - stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC); - stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC); - stub(configMock.getInt("telemetry.ignore.period.months", ignorePeriodInMonths)).toReturn(ignorePeriodInMonths); - stub(configMock.getList("user.metadata.fields", - Arrays.asList("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype"))) - .toReturn(Arrays.asList("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype")); - stub(configMock.getInt("redis.userDB.index", 4)).toReturn(4); - stub(redisConnectMock.getConnection(4)).toReturn(jedisMock); - List defaultSummaryEvents = new ArrayList<>(); - defaultSummaryEvents.add("ME_WORKFLOW_SUMMARY"); - stub(configMock.getList("summary.filter.events", defaultSummaryEvents)).toReturn(defaultSummaryEvents); - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - stub(envelopeMock.getOffset()).toReturn("2"); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "telemetry.with_location", new Partition(1))); - stub(configMock.get("user.signin.type.default", "Anonymous")).toReturn("Anonymous"); - stub(configMock.get("user.login.type.default", "NA")).toReturn("NA"); - - UserDataCache userCacheMock = new UserDataCache(configMock,jobMetrics, redisConnectMock); - deNormalizationTask = new DeNormalizationTask(configMock, contextMock, userCacheMock , contentCacheMock, dailcodeCacheMock, jobMetrics, redisConnectMock); - } - - @Test - public void shouldSendEventsToSuccessTopicIfDidIsNullWithUserContentEmptyData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITHOUT_DID); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - assertNull(outputEvent.get("contentdata")); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("user_data_retrieved"), false); - assertEquals(flags.get("content_data_retrieved"), false); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicIfDidIsNullWithUserContentData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITHOUT_DID); - Map content = new HashMap<>(); - content.put("name", "content-1"); content.put("objecttype", "Content"); content.put("contenttype", "TextBook"); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(content); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}"); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - Map userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType); - assertEquals(userData.size(), 5); - Map contentData = new Gson().fromJson(outputEvent.get("contentdata").toString(), mapType); - assertEquals(contentData.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("user_data_retrieved"), true); - assertEquals(flags.get("content_data_retrieved"), true); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithUserContentData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}"); - Map content = new HashMap<>(); - content.put("name", "content-1"); content.put("objecttype", "Content"); content.put("contenttype", "TextBook"); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(content); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>(){}.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - Map userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType); - assertEquals(userData.size(), 5); - Map contentData = new Gson().fromJson(outputEvent.get("contentdata").toString(), mapType); - assertEquals(contentData.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("user_data_retrieved"), true); - assertEquals(flags.get("content_data_retrieved"), true); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithoutRollUpID() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}"); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>(){}.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - System.out.println(outputMessage); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - Map userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType); - assertEquals(userData.size(), 5); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("user_data_retrieved"), true); - assertEquals(flags.get("content_data_retrieved"), false); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithRollUpIdInCollectionData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_OBJECT_ROLLUP); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}"); - Map collection = new HashMap<>(); - collection.put("name", "collection-1"); - collection.put("objecttype", "TextBook"); - collection.put("contenttype", "TextBook"); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(collection); - deNormalizationTask.process(envelopeMock,collectorMock, coordinatorMock); - Type mapType = new TypeToken>(){}.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - System.out.println(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map collectionData = new Gson().fromJson(outputEvent.get("collectiondata").toString(), mapType); - assertEquals(collectionData.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("collection_data_retrieved"), true); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithObjectIdInCollectionData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_OBJECT_ROLLUP); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}"); - Map collection = new HashMap<>(); - collection.put("name", "collection-1"); - collection.put("objecttype", "TextBook"); - collection.put("contenttype", "TextBook"); - stub(contentCacheMock.getData("do_31277438304183091217888")).toReturn(collection); - deNormalizationTask.process(envelopeMock,collectorMock, coordinatorMock); - Type mapType = new TypeToken>(){}.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map collectionData = new Gson().fromJson(outputEvent.get("contentdata").toString(), mapType); - assertEquals(collectionData.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("collection_data_retrieved"), false); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithObjectIdEqualsRollUpIDInCollectionData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_OBJECTID_EQUALS_ROLLUPID); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}"); - Map collection = new HashMap<>(); - collection.put("name", "collection-1"); - collection.put("objecttype", "TextBook"); - collection.put("contenttype", "TextBook"); - Map content = new HashMap<>(); - content.put("name", "content-1"); content.put("objecttype", "Content"); content.put("contenttype", "TextBook"); - stub(contentCacheMock.getData("do_31277438304183091217888")).toReturn(collection); - stub(contentCacheMock.getData("do_31277438304183091217888")).toReturn(content); - deNormalizationTask.process(envelopeMock,collectorMock, coordinatorMock); - Type mapType = new TypeToken>(){}.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - System.out.println(outputEvent); - Map collectionData = new Gson().fromJson(outputEvent.get("contentdata").toString(), mapType); - assertEquals(collectionData.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("content_data_retrieved"), true); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicIfContentIdIsNullWithUserData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITHOUT_OBJECT); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"type\":\"Registered\"}"); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>(){}.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals("3.0", outputEvent.get("ver").toString()); - assertNull(outputEvent.get("contentdata")); - Map userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType); - assertEquals(userData.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(true, flags.get("user_data_retrieved")); - assertNull(flags.get("content_data_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicIfContentDataIsEmptyWithUserAsSystem() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_ACTOR_AS_SYSTEM); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"type\":\"Registered\"}"); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>(){}.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals("3.0", outputEvent.get("ver").toString()); - assertNull(outputEvent.get("contentdata")); - assertNull(outputEvent.get("userdata")); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertNull(flags.get("user_data_retrieved")); - assertEquals(false, flags.get("content_data_retrieved")); - return true; - } - })); - } - - public void shouldSendEventsToSuccessTopicWithStringDialCodeData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_DIALCODE_AS_STRING); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null); - List ids = new ArrayList(); ids.add("8ZEDTP"); - Map dataMap = new HashMap(); dataMap.put("identifier", "8ZEDTP"); dataMap.put("channel", "test-channel"); - dataMap.put("status", "Draft"); - List data = new ArrayList<>(); data.add(dataMap); - stub(dailcodeCacheMock.getData(ids)).toReturn(data); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - assertNull(outputEvent.get("contentdata")); - List> dialcodesList = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), List.class); - assertEquals(dialcodesList.size(), 1); - Map data = dialcodesList.get(0); - assertEquals(data.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertNull(flags.get("user_data_retrieved")); - assertNull(flags.get("content_data_retrieved")); - assertEquals(true, flags.get("dialcode_data_retrieved")); - return true; - } - })); - } - - public void shouldSendEventsToSuccessTopicWithListDialCodeData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_DIALCODE_AS_LIST); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null); - List ids = new ArrayList(); ids.add("8ZEDTP"); ids.add("4ZEDTP"); - Map dataMap1 = new HashMap(); dataMap1.put("identifier", "8ZEDTP"); dataMap1.put("channel", "test-channel"); - dataMap1.put("status", "Draft"); - Map dataMap2 = new HashMap(); dataMap2.put("identifier", "4ZEDTP"); dataMap2.put("channel", "test-channel"); - dataMap2.put("status", "Draft"); - List data = new ArrayList<>(); data.add(dataMap1); data.add(dataMap2); - stub(dailcodeCacheMock.getData(ids)).toReturn(data); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>(){}.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - assertNull(outputEvent.get("contentdata")); - assertNull(outputEvent.get("userdata")); - List> dialcodesList = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), List.class); - assertEquals(dialcodesList.size(), 2); - Map data1 = dialcodesList.get(0); - assertEquals(data1.size(), 3); - Map data2 = dialcodesList.get(1); - assertEquals(data2.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertNull(flags.get("user_data_retrieved")); - assertNull(flags.get("content_data_retrieved")); - assertEquals(true, flags.get("dialcode_data_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithOutDialCodeData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITHOUT_DIALCODE); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - assertNull(outputEvent.get("contentdata")); - assertNull(outputEvent.get("userdata")); - assertNull(outputEvent.get("dialcodedata")); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertNull(flags.get("user_data_retrieved")); - assertNull(flags.get("content_data_retrieved")); - assertNull(flags.get("dialcode_data_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithOutAnyDenormData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_DEVICEDATA); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("user_data_retrieved"), false); - assertEquals(flags.get("content_data_retrieved"), false); - return true; - } - })); - } - - public void shouldSendEventsToSuccessTopicWithExistingEmptyStateCode() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_EMPTY_LOC); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("user_data_retrieved"), false); - assertEquals(flags.get("content_data_retrieved"), false); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithDialCodeLowerCaseDataByObjectLookup() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_DIALCODE_AS_OBJECT); - stub(contentCacheMock.getData("977D3I")).toReturn(null); - Map dataMap = new HashMap(); dataMap.put("identifier", "977D3I"); dataMap.put("channel", "test-channel"); - dataMap.put("status", "Draft"); - stub(dailcodeCacheMock.getData("977D3I")).toReturn(dataMap); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - assertNull(outputEvent.get("contentdata")); - Map data = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), Map.class); - assertEquals(data.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(false, flags.get("user_data_retrieved")); - assertNull(flags.get("content_data_retrieved")); - assertEquals(true, flags.get("dialcode_data_retrieved")); - return true; - } - })); - } - - //@Test - public void shouldSendEventsToSuccessTopicWithStringDialCodeDataCamelCase() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_CAMELCASE_DIALCODE_AS_STRING); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null); - List ids = new ArrayList(); ids.add("8ZEDTP"); - Map dataMap = new HashMap(); dataMap.put("identifier", "8ZEDTP"); dataMap.put("channel", "test-channel"); - dataMap.put("status", "Draft"); - List data = new ArrayList<>(); data.add(dataMap); - stub(dailcodeCacheMock.getData(ids)).toReturn(data); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - assertEquals(outputMessage.contains("dialcodes"), true); - assertEquals(outputMessage.contains("dialCodes"), false); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithAlteredEts() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_FUTURE_ETS); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "2.1"); - Long ets = new Gson().fromJson(outputEvent.get("ets").toString(), Long.class); - assertFalse(2530937155000L == ets); - return true; - } - })); - } - - @Test - public void shouldSendEventToFaildTopicIfEventIsOlder() throws Exception{ - - stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_OLDER_ETS); - stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - } - - @Test - public void shouldSendSummaryEventsToSuccessTopicWithUserData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.WFS_EVENT); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}"); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "1.1"); - Map userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType); - assertEquals(userData.size(), 5); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("user_data_retrieved"), true); - assertEquals(flags.get("content_data_retrieved"), null); - return true; - } - })); - } - - @Test - public void shouldSkipOtherSummaryEvent() throws Exception{ - - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_SUMMARY_EVENT); - Answer answer = new Answer(); - doAnswer(answer).when(jobMetrics).incSkippedCounter(); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - assertEquals(true, answer.isSkipped); - } - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - return true; - } - }; - } - - @Test - public void shouldSendEventsToSuccessTopicForLogEvents() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.LOG_EVENT); - jedisMock.set("0b251080-3230-415e-a593-ab7c1fac7ae3","{\"grade\":[4,5],\"type\":\"Registered\"}"); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - Map userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType); - assertEquals(userData.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(true, flags.get("user_data_retrieved")); - assertNull(flags.get("content_data_retrieved")); - return true; - } - })); - } - - @Test - public void shouldPushToSuccessTopicForErrorEvents() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.ERROR_EVENT); - Answer answer = new Answer(); - doAnswer(answer).when(jobMetrics).incSuccessCounter(); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - assertEquals(true, answer.isSuccess); - } - - class Answer implements org.mockito.stubbing.Answer { - - private Boolean isSkipped = false; - private Boolean isSuccess = false; - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - isSkipped = true; - isSuccess = true; - return null; - } - } - - @Test - public void shouldSendEventsToSuccessTopicWithQRCodeDataByObjectLookup() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_QR_AS_OBJECT); - stub(contentCacheMock.getData("977D3I")).toReturn(null); - Map dataMap = new HashMap(); dataMap.put("identifier", "977D3I"); dataMap.put("channel", "test-channel"); - dataMap.put("status", "Draft"); - stub(dailcodeCacheMock.getData("977D3I")).toReturn(dataMap); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - assertNull(outputEvent.get("contentdata")); - assertNull(outputEvent.get("userdata")); - Map data = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), Map.class); - assertEquals(data.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(false, flags.get("user_data_retrieved")); - assertNull(flags.get("content_data_retrieved")); - assertEquals(true, flags.get("dialcode_data_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithDialCodeCamelCaseDataByObjectLookup() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_DIALCODE_CAMELCASE_AS_OBJECT); - stub(contentCacheMock.getData("977D3I")).toReturn(null); - Map dataMap = new HashMap(); dataMap.put("identifier", "977D3I"); dataMap.put("channel", "test-channel"); - dataMap.put("status", "Draft"); - stub(dailcodeCacheMock.getData("977D3I")).toReturn(dataMap); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - assertNull(outputEvent.get("contentdata")); - assertNull(outputEvent.get("userdata")); - Map data = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), Map.class); - assertEquals(data.size(), 3); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(false, flags.get("user_data_retrieved")); - assertNull(flags.get("content_data_retrieved")); - assertEquals(true, flags.get("dialcode_data_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendAUDITEventsToSuccessTopicWithUserData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT); - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}"); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - Map userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType); - String jedisMockData = jedisMock.get("393407b1-66b1-4c86-9080-b2bce9842886"); - assertEquals(userData.size(), 5); - assert jedisMockData.contains("grade"); - assert jedisMockData.contains("district"); - assert jedisMockData.contains("type"); - assert jedisMockData.contains("state"); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("user_data_retrieved"), true); - assertEquals(flags.get("content_data_retrieved"), null); - return true; - } - })); - } - - @Test - public void shouldStampUserSigninAndLoginTypeForAllEvents() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT); - String user_id="e9d73b7b-211a-43f4-8c01-cc9333757a9d"; - jedisMock.set(user_id,"{\"subject\":[],\"grade\":[],\"usersignintype\":\"Self-Signed-In\",\"userlogintype\":\"student\"}"); - deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals(outputEvent.get("ver").toString(), "3.0"); - Map userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType); - assertEquals(userData.size(), 4); - assertEquals(userData.get("usersignintype"),"Self-Signed-In"); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(flags.get("user_data_retrieved"), true); - assertEquals(flags.get("content_data_retrieved"), null); - return true; - } - })); - } -} diff --git a/data-pipeline/deploy.sh b/data-pipeline/deploy.sh deleted file mode 100755 index 71d833e6b1..0000000000 --- a/data-pipeline/deploy.sh +++ /dev/null @@ -1,5 +0,0 @@ -mvn clean package -rm -rf deploy/samza -mkdir -p deploy/samza -tar -xvf ./target/ekstep-samza-0.0.1-dist.tar.gz -C deploy/samza -deploy/samza/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file://$PWD/deploy/samza/config/reverse-search.properties \ No newline at end of file diff --git a/data-pipeline/derived-de-duplication/pom.xml b/data-pipeline/derived-de-duplication/pom.xml deleted file mode 100644 index 0492d41779..0000000000 --- a/data-pipeline/derived-de-duplication/pom.xml +++ /dev/null @@ -1,179 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - - org.ekstep.ecosystem.jobs - derived-de-duplication - 0.0.3 - jar - DerivedDeduplication - - Identifying Derived Duplicate Events - - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - com.fiftyonred - mock-jedis - 0.4.0 - test - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - - diff --git a/data-pipeline/derived-de-duplication/src/main/assembly/src.xml b/data-pipeline/derived-de-duplication/src/main/assembly/src.xml deleted file mode 100644 index a14eb7749d..0000000000 --- a/data-pipeline/derived-de-duplication/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/derived-de-duplication.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-kv-rocksdb_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:derived-de-duplication - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/derived-de-duplication/src/main/config/derived-de-duplication.properties b/data-pipeline/derived-de-duplication/src/main/config/derived-de-duplication.properties deleted file mode 100644 index 4bf2eb9d0c..0000000000 --- a/data-pipeline/derived-de-duplication/src/main/config/derived-de-duplication.properties +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.DerivedDeDuplication - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__telemetry_derived_de_duplication_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.DeDuplicationTask -task.inputs=kafka.__env__.telemetry.derived -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -# YARN container configuration -# yarn.am.container.memory.mb=__yarn_am_container_mb__ -# yarn.container.memory.mb=__dedup_job_yarn_container_memory_mb__ - -output.success.topic.name=__env__.telemetry.derived.unique -output.failed.topic.name=__env__.telemetry.failed -output.duplicate.topic.name=__env__.telemetry.duplicate -output.malformed.topic.name=__env__.telemetry.malformed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,failed-message-count,error-message-count,duplicate-event-count - -# redis -#redis.host=localhost -redis.host=__redis_host__ -#redis.port=6379 -redis.port=__redis_port__ -redis.connection.max=2 -redis.connection.idle.max=2 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.database.duplicationstore.id=11 -redis.database.key.expiry.seconds=86400 \ No newline at end of file diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index 69c2e4a7af..0000000000 --- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.ekstep.ep.samza.domain; - - -import org.apache.commons.lang.StringUtils; -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.task.DeDuplicationConfig; -import java.util.HashMap; -import java.util.Map; - -public class Event extends Events { - - public Event(Map map) { - super(map); - } - - - public void markSkipped() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.derived_dd_processed", false); - telemetry.add("flags.derived_dd_checksum_present", false); - } - - public void markDuplicate() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.derived_dd_processed", false); - telemetry.add("flags.derived_dd_duplicate_event", true); - } - - public void markSuccess() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.derived_dd_processed", true); - telemetry.add("type", "events"); - } - - public void markRedisFailure() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.derived_dd_processed", false); - telemetry.add("flags.derived_dd_redis_failure", true); - telemetry.add("type", "events"); - } - - public void markFailure(String error, DeDuplicationConfig config) { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.derived_dd_processed", false); - - telemetry.addFieldIfAbsent("metadata", new HashMap()); - telemetry.add("metadata.derived_dd_error", error); - telemetry.add("metadata.src", config.jobName()); - } - -} - diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java deleted file mode 100644 index c95f63525d..0000000000 --- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.JsonSyntaxException; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.DeDuplicationConfig; -import org.ekstep.ep.samza.task.DeDuplicationSink; -import org.ekstep.ep.samza.task.DeDuplicationSource; -import org.ekstep.ep.samza.util.DeDupEngine; - -import redis.clients.jedis.exceptions.JedisException; - -public class DeDuplicationService { - private static Logger LOGGER = new Logger(DeDuplicationService.class); - private final DeDupEngine deDupEngine; - private final DeDuplicationConfig config; - - - public DeDuplicationService(DeDupEngine deDupEngine, DeDuplicationConfig config) { - this.deDupEngine = deDupEngine; - this.config = config; - } - - public void process(DeDuplicationSource source, DeDuplicationSink sink) throws Exception { - Event event = null; - - try { - event = source.getEvent(); - String checksum = event.getChecksum(); - if (checksum == null) { - LOGGER.info(event.id(), "EVENT WITHOUT CHECKSUM & MID, PASSING THROUGH : {}", event); - event.markSkipped(); - sink.toSuccessTopic(event); - return; - } - - if (isDupCheckRequired(event)) { - if (!deDupEngine.isUniqueEvent(checksum)) { - LOGGER.info(event.id(), "DUPLICATE EVENT, CHECKSUM: {}", checksum); - event.markDuplicate(); - sink.toDuplicateTopic(event); - return; - } - - LOGGER.info(event.id(), "ADDING EVENT CHECKSUM TO STORE"); - - deDupEngine.storeChecksum(checksum); - } - event.markSuccess(); - sink.toSuccessTopic(event); - - } catch (JedisException e) { - LOGGER.error(null, "Exception when retrieving data from redis: ", e); - event.markRedisFailure(); - sink.toSuccessTopicIfRedisException(event); - throw e; - } catch (JsonSyntaxException e) { - LOGGER.error(null, "INVALID EVENT: " + source.getMessage()); - sink.toMalformedEventsTopic(source.getMessage()); - } - } - - public boolean isDupCheckRequired(Event event) { - return (config.inclusiveProducerIds().isEmpty() || (null != event.producerId() && config.inclusiveProducerIds().contains(event.producerId()))); - } -} - diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java deleted file mode 100644 index ea3660302e..0000000000 --- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.ekstep.ep.samza.task; - - -import org.apache.samza.config.Config; - -import java.util.ArrayList; -import java.util.List; - -public class DeDuplicationConfig { - - private final String JOB_NAME = "DerivedDeDuplication"; - private String successTopic; - private String failedTopic; - private String duplicateTopic; - private String malformedTopic; - private final int dupStore; - private int expirySeconds; - private List includedProducerIds; - - - public DeDuplicationConfig(Config config) { - successTopic = config.get("output.success.topic.name", "telemetry.derived.unique"); - failedTopic = config.get("output.failed.topic.name", "telemetry.failed"); - duplicateTopic = config.get("output.duplicate.topic.name", "telemetry.duplicate"); - malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed"); - dupStore = config.getInt("redis.database.duplicationstore.id", 11); - expirySeconds = config.getInt("redis.database.key.expiry.seconds", 86400); - if (!config.get("dedup.producer.include.ids", "").isEmpty()) { - includedProducerIds = config.getList("dedup.producer.include.ids", new ArrayList<>()); - } else { - includedProducerIds = new ArrayList<>(); - } - - } - - public String successTopic() { - return successTopic; - } - - public String failedTopic() { - return failedTopic; - } - - public String duplicateTopic() { - return duplicateTopic; - } - - public String malformedTopic() { - return malformedTopic; - } - - public String jobName() { - return JOB_NAME; - } - - public int dupStore() { - return dupStore; - } - - public int expirySeconds() { - return expirySeconds; - } - - public List inclusiveProducerIds() { - return includedProducerIds; - } - -} \ No newline at end of file diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java deleted file mode 100644 index a100e4be50..0000000000 --- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - -public class DeDuplicationSink extends BaseSink { - - private DeDuplicationConfig config; - - public DeDuplicationSink(MessageCollector collector, JobMetrics metrics, DeDuplicationConfig config) { - super(collector, metrics); - this.config = config; - } - - public void toSuccessTopic(Event event) { - toTopic(config.successTopic(), event.did(), event.getJson()); - metrics.incSuccessCounter(); - } - - public void toSuccessTopicIfRedisException(Event event) { - toTopic(config.successTopic(), event.did(), event.getJson()); - metrics.incCacheErrorCounter(); - } - - public void toDuplicateTopic(Event event) { - toTopic(config.duplicateTopic(), event.did(), event.getJson()); - metrics.incDuplicateCounter(); - } - - public void toMalformedEventsTopic(String message) { - toTopic(config.malformedTopic(), null, message); - metrics.incFailedCounter(); - } - - public void toErrorTopic(Event event) { - toTopic(config.failedTopic(), event.did(), event.getJson()); - metrics.incErrorCounter(); - } - -} diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java deleted file mode 100644 index 0b8fceec3b..0000000000 --- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.ekstep.ep.samza.task; - -import java.util.Map; - -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; - -import com.google.gson.Gson; - -public class DeDuplicationSource { - private static Logger LOGGER = new Logger(DeDuplicationSource.class); - - private IncomingMessageEnvelope envelope; - - public DeDuplicationSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - String message = (String) envelope.getMessage(); - @SuppressWarnings("unchecked") - Map jsonMap = (Map) new Gson().fromJson(message, Map.class); - return new Event(jsonMap); - } - - public String getMessage() { - return envelope.toString(); - } - public SystemStreamPartition getSystemStreamPartition() { return envelope.getSystemStreamPartition();} - public String getOffset() { return envelope.getOffset();} -} diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java deleted file mode 100644 index 529ca88107..0000000000 --- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.DeDuplicationService; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.ekstep.ep.samza.util.RedisConnect; - -public class DeDuplicationTask extends BaseSamzaTask { - - private DeDuplicationConfig config; - private JobMetrics metrics; - private DeDuplicationService service; - - public DeDuplicationTask(Config config, TaskContext context, - DeDupEngine deDupEngine) { - init(config, context, deDupEngine); - } - - public DeDuplicationTask() { - - } - - @Override - public void init(Config config, TaskContext context) { - init(config, context, null); - } - - private void init(Config config, TaskContext context, - DeDupEngine deDupEngine) { - this.config = new DeDuplicationConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - deDupEngine = deDupEngine == null ? - new DeDupEngine(new RedisConnect(config), this.config.dupStore(), this.config.expirySeconds()) : deDupEngine; - service = new DeDuplicationService(deDupEngine, this.config); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) throws Exception { - DeDuplicationSource source = new DeDuplicationSource(envelope); - DeDuplicationSink sink = new DeDuplicationSink(collector, metrics, config); - service.process(source, sink); - } - -} diff --git a/data-pipeline/derived-de-duplication/src/main/resources/log4j.xml b/data-pipeline/derived-de-duplication/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/derived-de-duplication/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java b/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java deleted file mode 100644 index 9e88c40ae7..0000000000 --- a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; -import org.ekstep.ep.samza.task.DeDuplicationConfig; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -import java.util.Map; - -public class EventTest { - - private DeDuplicationConfig configMock; - - @Test - public void shouldAddSkipFlags() { - String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}"; - Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class)); - event.markSkipped(); - Map flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class); - Assert.assertNotNull(flagData); - Assert.assertEquals(flagData.get("derived_dd_checksum_present"), false); - Assert.assertEquals(flagData.get("derived_dd_processed"), false); - - } - - @Test - public void shouldMarkTheRedisFailure() { - String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}"; - Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class)); - event.markRedisFailure(); - Map flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class); - Assert.assertNotNull(flagData); - Assert.assertEquals(flagData.get("derived_dd_redis_failure"), true); - Assert.assertEquals(flagData.get("derived_dd_processed"), false); - - } - - @Test - public void shouldMarkFailure() { - configMock = Mockito.mock(DeDuplicationConfig.class); - String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}"; - Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class)); - event.markFailure("Invalid Event", configMock); - Map flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class); - Assert.assertNotNull(flagData); - Assert.assertEquals(flagData.get("derived_dd_processed"), false); - Object metaData = event.getMap().get("metadata"); - Assert.assertNotNull(metaData); - - } - -} diff --git a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index ce61e49754..0000000000 --- a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,166 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import java.util.Map; - -public class EventFixture { - private static final String EVENT_WITH_CHECKSUM = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"9a78dacddfcdfae7a9679b143e348158\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"id\": \"do_312764599773110272147\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"pageid\": \"ContentApp-EndScreen\",\n" + - " \"subtype\": \"ContentID\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1561620619274,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.14.5\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\",\n" + - " \"id\": \"dev.diksha.portal\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"contentplayer\",\n" + - " \"did\": \"9a78dacddfcdfae7a9679b143e348158\",\n" + - " \"sid\": \"8c416b30-4e47-a695-b1d2-9cd030eadeff\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"DialCode\",\n" + - " \"id\": \"DNILBR\"\n" + - " },\n" + - " {\n" + - " \"type\": \"ContentSession\",\n" + - " \"id\": \"88de9e1dfa6f237c076278a02cc4b305\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"INTERACT:fd63ff6bd0eedf05cf57c0058588ae69\",\n" + - " \"object\": {\n" + - " \"ver\": \"1\",\n" + - " \"id\": \"do_312764599773110272147\",\n" + - " \"type\": \"Content\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_31276594374681395211909\"\n" + - " }\n" + - " },\n" + - " \"tags\": [],\n" + - " \"syncts\": 1561620619795,\n" + - " \"@timestamp\": \"2019-06-27T07:30:19.795Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - private static final String EVENT_WITH_MID = "{\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"loc\": \"\",\n" + - " \"uid\": \"6\",\n" + - " \"age\": 5,\n" + - " \"handle\": \"Jijesh\",\n" + - " \"standard\": -1,\n" + - " \"language\": \"ML\"\n" + - " }\n" + - " },\n" + - " \"eid\": \"GE_CREATE_PROFILE\",\n" + - " \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genieservice.android\",\n" + - " \"ver\": \"1.0.local-qa-debug\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"tags\": [],\n" + - " \"ts\": \"2015-09-27T13:03:43-04:00\",\n" + - " \"uid\": \"6\",\n" + - " \"ver\": \"1.0\",\n" + - " \"mid\": \"22e1430f2e5f339230dbf9595b060008\"" + - "}"; - - private static final String EVENT_WITHOUT_CHECKSUM = "{\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"loc\": \"\",\n" + - " \"uid\": \"6\",\n" + - " \"age\": 5,\n" + - " \"handle\": \"Jijesh\",\n" + - " \"standard\": -1,\n" + - " \"language\": \"ML\"\n" + - " }\n" + - " },\n" + - " \"eid\": \"GE_CREATE_PROFILE\",\n" + - " \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genieservice.android\",\n" + - " \"ver\": \"1.0.local-qa-debug\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"tags\": [],\n" + - " \"ts\": \"2015-09-27T13:03:43-04:00\",\n" + - " \"uid\": \"6\",\n" + - " \"ver\": \"1.0\"" + - "}"; - - private static final String EVENT_WITH_EMTPY_CHANNEL = "{\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"loc\": \"\",\n" + - " \"uid\": \"6\",\n" + - " \"age\": 5,\n" + - " \"handle\": \"Jijesh\",\n" + - " \"standard\": -1,\n" + - " \"language\": \"ML\"\n" + - " }\n" + - " },\n" + - " \"eid\": \"GE_CREATE_PROFILE\",\n" + - " \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genieservice.android\",\n" + - " \"ver\": \"1.0.local-qa-debug\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"context\": {\n" + - " \"channel\": \"\"\n" + - " },\n" + - " \"tags\": [],\n" + - " \"ts\": \"2015-09-27T13:03:43-04:00\",\n" + - " \"uid\": \"6\",\n" + - " \"ver\": \"1.0\"" + - "}"; - - public static Map EventWithChecksumMap() { - return new Gson().fromJson(EVENT_WITH_CHECKSUM, new TypeToken>() { - }.getType()); - } - - public static Map EventWithMidMap() { - return new Gson().fromJson(EVENT_WITH_MID, new TypeToken>() { - }.getType()); - } - - public static Map EventWithoutChecksumFieldMap() { - return new Gson().fromJson(EVENT_WITHOUT_CHECKSUM, new TypeToken>() { - }.getType()); - } - - public static Map EventWithEmptyChannel() { - return new Gson().fromJson(EVENT_WITH_EMTPY_CHANNEL, new TypeToken>() { - }.getType()); - } - - public static String EventWithChecksumJson() { - return EVENT_WITH_CHECKSUM; - } - -} - diff --git a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java deleted file mode 100644 index 185dc5d6b6..0000000000 --- a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ekstep.ep.samza.system; - - -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.task.DeDuplicationConfig; -import org.junit.Assert; -import org.junit.Test; -import java.util.Map; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class EventTest { - @Test - public void shouldReturnChecksumIfPresent(){ - - Event event = new Event(EventFixture.EventWithChecksumMap()); - Assert.assertEquals("INTERACT:fd63ff6bd0eedf05cf57c0058588ae69", (String) event.getChecksum()); - } - - @Test - public void shouldReturnMidIfPresent(){ - - Event event = new Event(EventFixture.EventWithMidMap()); - Assert.assertEquals("22e1430f2e5f339230dbf9595b060008", (String) event.getChecksum()); - } - - @Test - public void shouldReturnNullIfChecksumAndMidAreAbsent(){ - - Event event = new Event(EventFixture.EventWithoutChecksumFieldMap()); - Assert.assertNull(event.getChecksum()); - } -} - diff --git a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java b/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java deleted file mode 100644 index 6663ed8916..0000000000 --- a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.fiftyonred.mock_jedis.MockJedis; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; -import redis.clients.jedis.Jedis; - -import java.util.ArrayList; -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -public class DeDuplicationTaskTest { - - private static final String SUCCESS_TOPIC = "telemetry.derived.unique"; - private static final String FAILED_TOPIC = "telemetry.unique.fail"; - private static final String DUPLICATE_TOPIC = "telemetry.duplicate"; - private static final String MALFORMED_TOPIC = "telemetry.malformed"; - private MessageCollector collectorMock; - private TaskContext contextMock; - private MetricsRegistry metricsRegistry; - private Counter counter; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private Config configMock; - private DeDuplicationTask deDuplicationTask; - private DeDupEngine deDupEngineMock; - private Jedis jedisMock = new MockJedis("duplicationtest"); - private int dupStoreId = 1; - - @Before - public void setUp() { - collectorMock = mock(MessageCollector.class); - contextMock = Mockito.mock(TaskContext.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = Mockito.mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - configMock = Mockito.mock(Config.class); - RedisConnect redisConnectMock = mock(RedisConnect.class); - deDupEngineMock = mock(DeDupEngine.class); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC); - stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC); - stub(configMock.get("output.duplicate.topic.name", DUPLICATE_TOPIC)).toReturn(DUPLICATE_TOPIC); - stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC); - stub(metricsRegistry.newCounter(anyString(), anyString())) - .toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - stub(envelopeMock.getOffset()).toReturn("2"); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(0))); - stub(configMock.getInt("redis.database.duplicationstore.id", dupStoreId)).toReturn(dupStoreId); - when(configMock.get("dedup.producer.include.ids", "")).thenReturn("dev.diskha.portal"); - stub(configMock.getList("dedup.producer.include.ids", new ArrayList<>())).toReturn(Arrays.asList("dev.diksha.portal")); - - - deDuplicationTask = new DeDuplicationTask(configMock, contextMock, deDupEngineMock); - } - - @Test - public void ShouldSendEventToSuccessTopicIfEventIsUnique() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.EventWithChecksumJson()); - - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - } - - @Test - public void ShouldSendEventsToDuplicateTopicIfEventIsDuplicate() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.EventWithChecksumJson()); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(false); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), DUPLICATE_TOPIC))); - } - - @Test - public void ShouldSendEventsToMalformedTopicIfEventIsMalformed() throws Exception { - - when(envelopeMock.getMessage()).thenReturn("{'metadata':{'checksum':'sajksajska'}"); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - @Test(expected = Exception.class) - public void ShouldSendEventsToFailedTopicForAnyUnhandledException() throws Exception { - - when(envelopeMock.getMessage()).thenReturn(EventFixture.EventWithChecksumJson()); - when(deDupEngineMock.isUniqueEvent(anyString())).thenThrow(new Exception()); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - } - - @Test - public void ShouldStoreChecksumIfEventIsUnique() throws Exception { - - when(envelopeMock.getMessage()).thenReturn(EventFixture.EventWithChecksumJson()); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - - deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(deDupEngineMock, times(1)).isUniqueEvent(anyString()); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - } - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - return true; - } - }; - } -} diff --git a/data-pipeline/device-profile-updater/pom.xml b/data-pipeline/device-profile-updater/pom.xml deleted file mode 100644 index e0fb3eee0e..0000000000 --- a/data-pipeline/device-profile-updater/pom.xml +++ /dev/null @@ -1,185 +0,0 @@ - - - - jobs - org.ekstep.ecosystem - 0.0.1 - - 4.0.0 - - org.ekstep.ecosystem.jobs - device-profile-updater - 0.1.8 - DeviceProfileUpdater - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.opentable.components - otj-pg-embedded - 0.13.3 - test - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - redis.clients - jedis - 2.9.0 - - - com.datastax.cassandra - cassandra-driver-core - 3.1.0 - - - com.google.code.gson - gson - 2.4 - - - com.google.guava - guava - 18.0 - - - com.fiftyonred - mock-jedis - 0.4.0 - jar - test - - - org.jetbrains - annotations - RELEASE - compile - - - it.ozimov - embedded-redis - 0.7.1 - test - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - - \ No newline at end of file diff --git a/data-pipeline/device-profile-updater/src/main/assembly/src.xml b/data-pipeline/device-profile-updater/src/main/assembly/src.xml deleted file mode 100644 index 051f966cc6..0000000000 --- a/data-pipeline/device-profile-updater/src/main/assembly/src.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/device-profile-updater.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-kv-rocksdb_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:device-profile-updater - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - redis.clients:2.9.0 - - true - - - diff --git a/data-pipeline/device-profile-updater/src/main/config/device-profile-updater.properties b/data-pipeline/device-profile-updater/src/main/config/device-profile-updater.properties deleted file mode 100644 index 5926a24dd0..0000000000 --- a/data-pipeline/device-profile-updater/src/main/config/device-profile-updater.properties +++ /dev/null @@ -1,91 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.DeviceProfileUpdater - -# YARN -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__device_profile_updater_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.DeviceProfileUpdaterTask -task.inputs=kafka.__env__.events.deviceprofile -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -#systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__ingestion_zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -#systems.kafka.producer.bootstrap.servers=localhost:9092 -systems.kafka.producer.bootstrap.servers=__ingestion_kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics -output.metrics.topic.name=__env__.pipeline_metrics -output.malformed.topic.name=__env__.telemetry.malformed -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,failed-message-count,error-message-count,device-db-update-count,device-cache-update-count - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# postgres -postgres.host=__postgres_host__ -#postgres.host=localhost -postgres.port=__postgres_port__ -#postgres.port=5432 -postgres.user=__postgres_user__ -sensitive.postgres.password=__postgres_password__ -postgres.db=__postgres_db__ -postgres.device_profile_table=__env___device_profile -postgres.maxConnections = 2 - -# redis -#redis.host=localhost -redis.host=__metadata_redis_host__ -# redis.port=6379 -redis.port=__metadata_redis_port__ -redis.connection.max=2 -redis.connection.idle.max=2 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.database.deviceStore.id=2 - diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java deleted file mode 100644 index b6f35efa2a..0000000000 --- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; - -public class DeviceProfile { - private String countryCode; - private String country; - private String stateCode; - private String state; - private String city; - private String districtCustom; - private String stateCodeCustom; - private String stateCustomName; - private String userDeclaredState; - private String userDeclaredDistrict; - private Map uaspec; - private Map devicespec; - private Long firstAccess; - private Long user_declared_on; - private Long api_last_updated_on; - private Gson gson = new Gson(); - private Type type = new TypeToken>() {}.getType(); - - public DeviceProfile() { - this.countryCode = ""; - this.country = ""; - this.stateCode = ""; - this.state = ""; - this.city = ""; - this.districtCustom = ""; - this.stateCodeCustom = ""; - this.stateCustomName = ""; - this.userDeclaredState = ""; - this.userDeclaredDistrict = ""; - this.uaspec = new HashMap<>(); - this.devicespec = new HashMap<>(); - this.firstAccess = 0L; - this.user_declared_on = 0L; - this.api_last_updated_on = 0L; - } - - public Map toMap() { - Map values = new HashMap<>(); - values.put("country_code", DeviceProfile.getValueOrDefault(this.countryCode, "")); - values.put("country", DeviceProfile.getValueOrDefault(this.country, "")); - values.put("state_code", DeviceProfile.getValueOrDefault(this.stateCode, "")); - values.put("state", DeviceProfile.getValueOrDefault(this.state, "")); - values.put("city", DeviceProfile.getValueOrDefault(this.city, "")); - values.put("district_custom", DeviceProfile.getValueOrDefault(this.districtCustom, "")); - values.put("state_custom", DeviceProfile.getValueOrDefault(this.stateCustomName, "")); - values.put("state_code_custom", DeviceProfile.getValueOrDefault(this.stateCodeCustom, "")); - values.put("user_declared_state", DeviceProfile.getValueOrDefault(this.userDeclaredState, "")); - values.put("user_declared_district", DeviceProfile.getValueOrDefault(this.userDeclaredDistrict, "")); - values.put("uaspec", gson.toJson(DeviceProfile.getValueOrDefault(this.uaspec, new HashMap<>()))); - values.put("devicespec", gson.toJson(DeviceProfile.getValueOrDefault(this.devicespec, new HashMap<>()))); - values.put("firstaccess", DeviceProfile.getValueOrDefault(String.valueOf(this.firstAccess), "")); - values.put("user_declared_on", DeviceProfile.getValueOrDefault(String.valueOf(this.user_declared_on), "")); - values.put("api_last_updated_on", DeviceProfile.getValueOrDefault(String.valueOf(this.api_last_updated_on), "")); - return values; - } - - public DeviceProfile fromMap(Map map) { - this.countryCode = map.getOrDefault("country_code", ""); - this.country = map.getOrDefault("country", ""); - this.stateCode = map.getOrDefault("state_code", ""); - this.state = map.getOrDefault("state", ""); - this.city = map.getOrDefault("city", ""); - this.districtCustom = map.getOrDefault("district_custom", ""); - this.stateCustomName = map.getOrDefault("state_custom", ""); - this.stateCodeCustom = map.getOrDefault("state_code_custom", ""); - this.userDeclaredState = map.getOrDefault("user_declared_state", ""); - this.userDeclaredDistrict = map.getOrDefault("user_declared_district", ""); - this.uaspec = gson.fromJson(map.getOrDefault("uaspec", ""), type); - this.devicespec = gson.fromJson(map.getOrDefault("device_spec", ""), type); - this.firstAccess = Long.valueOf(map.getOrDefault("first_access", "0")); - this.user_declared_on = Long.valueOf(map.getOrDefault("api_last_updated_on", "0")); - this.api_last_updated_on = Long.valueOf(map.getOrDefault("api_last_updated_on", "0")); - return this; - } - - public static T getValueOrDefault(T value, T defaultValue) { - return value == null ? defaultValue : value; - } -} diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/service/DeviceProfileUpdaterService.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/service/DeviceProfileUpdaterService.java deleted file mode 100644 index 083541abf6..0000000000 --- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/service/DeviceProfileUpdaterService.java +++ /dev/null @@ -1,182 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.google.gson.JsonSyntaxException; -import org.apache.commons.lang3.StringUtils; -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.DeviceProfile; -import org.ekstep.ep.samza.task.DeviceProfileUpdaterSink; -import org.ekstep.ep.samza.task.DeviceProfileUpdaterSource; -import org.ekstep.ep.samza.util.PostgresConnect; -import org.ekstep.ep.samza.util.RedisConnect; -import org.postgresql.util.PGobject; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.exceptions.JedisException; -import java.lang.reflect.Type; -import com.google.gson.reflect.TypeToken; - -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.*; -import java.util.stream.Collectors; - -public class DeviceProfileUpdaterService { - - private static Logger LOGGER = new Logger(DeviceProfileUpdaterService.class); - private RedisConnect redisConnect; - private Jedis deviceStoreConnection; - private int deviceStoreDb; - private PostgresConnect postgresConnect; - private String postgres_table; - private Gson gson = new Gson(); - private Type mapType = new TypeToken>() { }.getType(); - - public DeviceProfileUpdaterService(Config config, RedisConnect redisConnect, PostgresConnect postgresConnect) { - this.redisConnect = redisConnect; - this.postgresConnect = postgresConnect; - this.deviceStoreDb = config.getInt("redis.database.deviceStore.id", 2); - this.deviceStoreConnection = redisConnect.getConnection(deviceStoreDb); - this.postgres_table = config.get("postgres.device_profile_table", "device_profile"); - } - - public void process(DeviceProfileUpdaterSource source, DeviceProfileUpdaterSink sink) throws Exception { - try { - Map message = source.getMap(); - updateDeviceDetails(message, sink); - } catch (JsonSyntaxException e) { - LOGGER.error(null, "INVALID EVENT: " + source.getMessage(), e); - sink.toMalformedTopic(source.getMessage()); - } - } - - private void updateDeviceDetails(Map deviceData, DeviceProfileUpdaterSink sink) throws Exception { - if (deviceData.size() > 0) { - - deviceData.values().removeAll(Collections.singleton("")); - deviceData.values().removeAll(Collections.singleton("{}")); - - DeviceProfile deviceProfile = new DeviceProfile().fromMap(deviceData); - String deviceId = deviceData.get("device_id"); - if (null != deviceId && !deviceId.isEmpty()) { - - // Update device profile details in Postgres DB - addDeviceDataToDB(deviceId, deviceData); - sink.deviceDBUpdateSuccess(); - - // Update device profile details in Redis cache - addDeviceDataToCache(deviceId, deviceProfile); - sink.deviceCacheUpdateSuccess(); - - sink.success(); - LOGGER.info(deviceId,"Updated successfully"); - } - else { sink.failed(); } - } - - } - - private void addDeviceDataToCache(String deviceId, DeviceProfile deviceProfile) { - try { - addToCache(deviceId, deviceProfile, deviceStoreConnection); - } catch (JedisException ex) { - this.deviceStoreConnection.close(); - this.deviceStoreConnection = redisConnect.getConnection(deviceStoreDb); - addToCache(deviceId, deviceProfile, deviceStoreConnection); - } - } - - private void addDeviceDataToDB(String deviceId, Map deviceData) throws Exception { - Long firstAccess = Long.parseLong(deviceData.get("first_access")); - // - Long lastUpdatedDate = Long.parseLong(deviceData.get("api_last_updated_on")); - List parsedKeys = new ArrayList<>(Arrays.asList("first_access", "api_last_updated_on")); - deviceData.keySet().removeAll(parsedKeys); - String columns = formatValues(deviceData.keySet(),","); - String values = formatPrepareStatement(deviceData.values().size(),"?,"); - String postgresQuery = String.format("INSERT INTO %s (api_last_updated_on,updated_date,%s) VALUES(?,?,%s?) ON CONFLICT(device_id) DO UPDATE SET (api_last_updated_on,updated_date,%s)=(?,?,%s?);",postgres_table, columns, values, columns, values); - PreparedStatement preparedStatement = postgresConnect.getConnection().prepareStatement(postgresQuery); - - preparedStatement.setTimestamp(1, new Timestamp(lastUpdatedDate)); // Adding api_last_updated_on as timestamp to index 1 of preparestatement - preparedStatement.setTimestamp(deviceData.values().size()+3, new Timestamp(lastUpdatedDate)); // Adding api_last_updated_on as timestamp to 3rd index after the map size(for on conflict value) - preparedStatement.setTimestamp(2, new Timestamp(System.currentTimeMillis())); // Adding updated_date as timestamp to index 2 of preparestatement - preparedStatement.setTimestamp(deviceData.values().size()+4, new Timestamp(System.currentTimeMillis())); // Adding updated_date as timestamp to 4th index after the map size(for on conflict value) - - setPrepareStatement(preparedStatement,2, deviceData); // Adding map values to preparestatement from index after the api_last_updated_on and updated_on - setPrepareStatement(preparedStatement,deviceData.values().size()+4, deviceData); // Adding map values from 4th index after map size as index(1-api_last_updated_on, 2-updated_on, 3-(size+2)map-values) - - preparedStatement.executeUpdate(); - preparedStatement.close(); - String updateFirstAccessQuery = String.format("UPDATE %s SET first_access = '%s' WHERE device_id = '%s' AND first_access IS NULL", - postgres_table, new Timestamp(firstAccess).toString(), deviceId); - postgresConnect.execute(updateFirstAccessQuery); - - if(null != deviceData.get("user_declared_state")) { - String updateUserDeclaredOnQuery = String.format("UPDATE %s SET user_declared_on = '%s' WHERE device_id = '%s' AND user_declared_on IS NULL", - postgres_table, new Timestamp(lastUpdatedDate).toString(), deviceId); - postgresConnect.execute(updateUserDeclaredOnQuery); - } - - } - - private void addToCache(String deviceId, DeviceProfile deviceProfile, Jedis redisConnection) { - Map deviceMap = deviceProfile.toMap(); - deviceMap.values().removeAll(Collections.singleton("")); - deviceMap.values().removeAll(Collections.singleton("{}")); - if(deviceMap.get("user_declared_state") == null) { - deviceMap.remove("user_declared_on"); - } - if (redisConnection.exists(deviceId)) { - Map data = redisConnection.hgetAll(deviceId); - if(data.get("firstaccess") != null && !("0").equals(data.get("firstaccess"))) { - deviceMap.remove("firstaccess"); - } - if(data.get("user_declared_on") != null && deviceMap.get("user_declared_on") != null) { - deviceMap.remove("user_declared_on"); - } - redisConnection.hmset(deviceId, deviceMap); - } else { - redisConnection.hmset(deviceId, deviceMap); - } - LOGGER.debug(null, String.format("Device details for device id %s updated successfully", deviceId)); - } - - private String formatValues(Collection values, String delimiter) { - return values.stream().map(Object::toString).collect(Collectors.joining(delimiter)); - } - - private String formatPrepareStatement(Integer length, String delimiter) { - return StringUtils.repeat(delimiter,length-1); - } - - private void setPrepareStatement(PreparedStatement preparedStatement, Integer index, Map deviceData) throws SQLException { - for (String value : deviceData.values()) { - index++; - PGobject jsonObject = new PGobject(); - if(isJson(value)) { - jsonObject.setType("json"); - jsonObject.setValue(gson.fromJson(value, JsonObject.class).toString()); - preparedStatement.setObject(index, jsonObject); - } - else { - preparedStatement.setString(index, value); - } - } - } - - public boolean isJson(String json) { - try { - gson.fromJson(json, Object.class); - Object jsonObjType = gson.fromJson(json, Object.class).getClass(); - if(jsonObjType.equals(String.class) || jsonObjType.equals(Double.class)){ - return false; - } - return true; - } catch (JsonSyntaxException ex) { - return false; - } - } - -} diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterConfig.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterConfig.java deleted file mode 100644 index 97bb2abe9d..0000000000 --- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterConfig.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.ekstep.ep.samza.task; -import org.apache.samza.config.Config; - -public class DeviceProfileUpdaterConfig { - - private final String JOB_NAME = "DeviceProfileUpdater"; - private String malformedTopic; - - public DeviceProfileUpdaterConfig(Config config) { - malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed"); - } - - public String malformedTopic() { - return malformedTopic; - } - - public String jobName() { - return JOB_NAME; - } -} diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSink.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSink.java deleted file mode 100644 index f4403bf2c8..0000000000 --- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSink.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; - -public class DeviceProfileUpdaterSink extends BaseSink { - private DeviceProfileUpdaterConfig config; - - public DeviceProfileUpdaterSink(MessageCollector collector, JobMetrics metrics, DeviceProfileUpdaterConfig config) { - super(collector, metrics); - this.config = config; - } - - public void deviceDBUpdateSuccess() { metrics.deviceDBUpdateSuccess(); } - - public void deviceCacheUpdateSuccess() { metrics.deviceCacheUpdateSuccess(); } - - public void success() { - metrics.incSuccessCounter(); - } - - public void failed() { - metrics.incFailedCounter(); - } - - public void toMalformedTopic(String message) { - toTopic(config.malformedTopic(), null, message); - metrics.incErrorCounter(); - } -} diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSource.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSource.java deleted file mode 100644 index 5c26ee45c2..0000000000 --- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSource.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; - -import java.lang.reflect.Type; -import java.util.Map; - -public class DeviceProfileUpdaterSource { - - private IncomingMessageEnvelope envelope; - - public DeviceProfileUpdaterSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - @SuppressWarnings("unchecked") - public Map getMap() { - String message = (String) envelope.getMessage(); - Type mapType = new TypeToken>(){}.getType(); - return new Gson().fromJson(message, mapType); - } - - public String getMessage() { - return envelope.toString(); - } - -} diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterTask.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterTask.java deleted file mode 100644 index efcc0f4d7b..0000000000 --- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterTask.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.DeviceProfileUpdaterService; -import org.ekstep.ep.samza.util.PostgresConnect; -import org.ekstep.ep.samza.util.RedisConnect; - -public class DeviceProfileUpdaterTask extends BaseSamzaTask { - - private JobMetrics metrics; - private DeviceProfileUpdaterService service; - private DeviceProfileUpdaterConfig config; - private RedisConnect redisConnect; - private PostgresConnect postgresConnect; - - public DeviceProfileUpdaterTask(Config config, TaskContext context) throws Exception { init(config, context); } - - public DeviceProfileUpdaterTask() { - } - - @Override - public void init(Config config, TaskContext context) throws Exception{ - redisConnect = new RedisConnect(config); - this.config = new DeviceProfileUpdaterConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - postgresConnect = new PostgresConnect(config); - service = new DeviceProfileUpdaterService(config, redisConnect, postgresConnect); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, - TaskCoordinator taskCoordinator) throws Exception { - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelope); - DeviceProfileUpdaterSink sink = new DeviceProfileUpdaterSink(collector, metrics, config); - service.process(source, sink); - } - -} - diff --git a/data-pipeline/device-profile-updater/src/main/resources/log4j.xml b/data-pipeline/device-profile-updater/src/main/resources/log4j.xml deleted file mode 100644 index 7e0a688c9b..0000000000 --- a/data-pipeline/device-profile-updater/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/DeviceProfileServiceTest.java b/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/DeviceProfileServiceTest.java deleted file mode 100644 index 8060b0b421..0000000000 --- a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/DeviceProfileServiceTest.java +++ /dev/null @@ -1,300 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.fiftyonred.mock_jedis.MockJedis; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import com.opentable.db.postgres.embedded.EmbeddedPostgres; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.ekstep.ep.samza.service.Fixtures.EventFixture; -import org.ekstep.ep.samza.task.DeviceProfileUpdaterTask; -import org.ekstep.ep.samza.task.DeviceProfileUpdaterSource; -import org.ekstep.ep.samza.task.DeviceProfileUpdaterSink; -import org.ekstep.ep.samza.util.PostgresConnect; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.exceptions.JedisException; -import redis.embedded.RedisServer; - -import java.lang.reflect.Type; -import java.sql.*; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.mockito.Mockito.*; - -public class DeviceProfileServiceTest { - - private RedisConnect redisConnectMock; - private PostgresConnect postgresConnectMock; - RedisServer redisServer; - private IncomingMessageEnvelope envelopeMock; - private Jedis jedisMock; - private Connection connectionMock; - private Statement statementMock; - private Statement statement; - private Connection connection; - private DeviceProfileUpdaterService deviceProfileUpdaterService; - private DeviceProfileUpdaterSink deviceProfileUpdaterSinkMock; - private Config configMock; - private Integer deviceStoreId = 2; - private String postgres_table; - private Gson gson = new Gson(); - private Type mapType = new TypeToken>() { }.getType(); - - @Before - public void setUp() throws Exception { - postgresConnectMock=mock(PostgresConnect.class); - - deviceProfileUpdaterSinkMock = mock(DeviceProfileUpdaterSink.class); - configMock = mock(Config.class); - connectionMock = mock(Connection.class); - statementMock = mock(Statement.class); - redisServer = new RedisServer(6379); - redisServer.start(); - envelopeMock = mock(IncomingMessageEnvelope.class); - stub(configMock.getInt("redis.database.deviceStore.id", deviceStoreId)).toReturn(deviceStoreId); - stub(configMock.get("input.device.topic.name","device.profile")).toReturn("events.deviceprofile"); - postgres_table = "device_profile"; - - EmbeddedPostgres pg = EmbeddedPostgres.start(); - connection = pg.getPostgresDatabase().getConnection(); - statement = connection.createStatement(); - - stub(connectionMock.createStatement()).toReturn(statement); - stub(postgresConnectMock.getConnection()).toReturn(connection); - stub(postgresConnectMock.resetConnection()).toReturn(connection); - statement.execute("CREATE TABLE device_profile(\n" + - " device_id text PRIMARY KEY,\n" + - " api_last_updated_on TIMESTAMP,\n" + - " avg_ts float,\n" + - " city TEXT,\n" + - " country TEXT,\n" + - " country_code TEXT,\n" + - " device_spec json,\n" + - " district_custom TEXT,\n" + - " fcm_token TEXT,\n" + - " first_access TIMESTAMP,\n" + - " last_access TIMESTAMP,\n" + - " user_declared_on TIMESTAMP,\n" + - " producer_id TEXT,\n" + - " state TEXT,\n" + - " state_code TEXT,\n" + - " state_code_custom TEXT,\n" + - " state_custom TEXT,\n" + - " total_launches bigint,\n" + - " total_ts float,\n" + - " uaspec json,\n" + - " updated_date TIMESTAMP,\n" + - " user_declared_district TEXT,\n" + - " user_declared_state TEXT)"); - - stub(configMock.get("redis.host", "localhost")).toReturn("localhost"); - stub(configMock.getInt("redis.port", 6379)).toReturn(6379); - stub(configMock.getInt("redis.connection.max", 2)).toReturn(2); - stub(configMock.getInt("redis.connection.idle.max", 2)).toReturn(2); - stub(configMock.getInt("redis.connection.idle.min", 1)).toReturn(1); - stub(configMock.getInt("redis.connection.minEvictableIdleTimeSeconds", 120)).toReturn(120); - stub(configMock.getInt("redis.connection.timeBetweenEvictionRunsSeconds", 300)).toReturn(300); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "events_deviceprofile", new Partition(0))); - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS); - stub(configMock.get("postgres.device_profile_table","device_profile")).toReturn("device_profile"); - redisConnectMock = new RedisConnect(configMock); - jedisMock = redisConnectMock.getConnection(deviceStoreId); - deviceProfileUpdaterService = new DeviceProfileUpdaterService(configMock, redisConnectMock, postgresConnectMock); - - } - - @After - public void tearDown() { - redisServer.stop(); - } - - @Test - public void shouldupdateCache() throws Exception { - stub(connectionMock.createStatement()).toReturn(statementMock); - jedisMock.flushAll(); - - Type mapType = new TypeToken>(){}.getType(); - Map event = gson.fromJson(EventFixture.DEVICE_PROFILE_DETAILS, mapType); - String device_id = event.get("device_id"); - - jedisMock.hmset(device_id, event); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - Map cachedData = jedisMock.hgetAll(device_id); - - assertEquals("232455", cachedData.get("device_id")); - assertEquals("Bengaluru", cachedData.get("city")); - assertEquals("Karnataka", cachedData.get("state")); - assertEquals("dev.sunbird.portal", cachedData.get("producer_id")); - assertEquals("IN", cachedData.get("country_code")); - assertEquals("Bengaluru",cachedData.get("user_declared_district")); - assertEquals("Karnataka",cachedData.get("user_declared_state")); - - verify(deviceProfileUpdaterSinkMock, times(1)).deviceCacheUpdateSuccess(); - } - - @Test - public void shouldNotUpdateFordidNull() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_WITH_NO_DEVICE_ID); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - verify(deviceProfileUpdaterSinkMock, times(1)).failed(); - - } - - @Test - public void shouldNotUpdateFirstAccessifPresent() throws Exception { - stub(connectionMock.createStatement()).toReturn(statementMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS); - Map newDeviceData = gson.fromJson(EventFixture.DEVICE_PROFILE_DETAILS, mapType); - Map deviceDetails = new HashMap<>(); - deviceDetails.put("firstaccess","156990957889"); - jedisMock.hmset(newDeviceData.get("device_id"), deviceDetails); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - Map data = jedisMock.hgetAll(newDeviceData.get("device_id")); - assertEquals("156990957889", data.get("firstaccess")); - } - - @Test - public void shouldUpdateFirstAccessifNotPresent() throws Exception{ - stub(connectionMock.createStatement()).toReturn(statementMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS); - Map newDeviceData = gson.fromJson(EventFixture.DEVICE_PROFILE_DETAILS, mapType); - String deviceId = newDeviceData.get("device_id"); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - Map data = jedisMock.hgetAll(deviceId); - assertEquals("1568377184000", data.get("firstaccess")); - } - - @Test - public void shouldAddDeviceDataToPostgres() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - ResultSet rs = statement.executeQuery("SELECT * from device_profile where device_id='232455'"); - while(rs.next()) { - assertEquals("Bengaluru", rs.getString("city")); - assertEquals("Karnataka", rs.getString("state")); - assertEquals("India", rs.getString("country")); - assertEquals("IN", rs.getString("country_code")); - assertEquals("dev.sunbird.portal", rs.getString("producer_id")); - assertEquals("Bengaluru", rs.getString("user_declared_district")); - assertEquals("Karnataka", rs.getString("user_declared_state")); - } - - verify(deviceProfileUpdaterSinkMock, times(1)).deviceDBUpdateSuccess(); - } - - @Test - public void shouldNotAddFirstAccessIfPresentInPostgres()throws Exception{ - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS); - - String query = String.format("INSERT INTO %s (device_id, first_access) VALUES ('232455','2019-09-24 01:03:04.999');", postgres_table); - statement.execute(query); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - ResultSet rs=statement.executeQuery(String.format("SELECT first_access FROM %s WHERE device_id='232455';", postgres_table)); - while(rs.next()) { - assertEquals("2019-09-24 01:03:04.999", rs.getString(1)); - } - } - - @Test - public void shouldAddUserDeclaredOnIfNotPresent() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS); - - Type mapType = new TypeToken>(){}.getType(); - Map event = gson.fromJson(EventFixture.DEVICE_PROFILE_DETAILS, mapType); - String device_id = event.get("device_id"); - - jedisMock.hmset(device_id, event); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - Map data=jedisMock.hgetAll(device_id); - assertEquals("1568377184000", data.get("user_declared_on")); - } - - @Test - public void shouldNotAddUserDeclaredOnIfStateIsNull() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_WITH_STATE_NULL); - - Type mapType = new TypeToken>(){}.getType(); - Map event = gson.fromJson(EventFixture.DEVICE_PROFILE_WITH_STATE_NULL, mapType); - String device_id = event.get("device_id"); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - Map data=jedisMock.hgetAll(device_id); - assertEquals(null, data.get("user_declared_on")); - } - - @Test - public void shouldNotAddUserDeclaredOnIfPresentInPostgres() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS); - - String query = String.format("INSERT INTO %s (device_id, user_declared_on) VALUES ('232455','2019-09-24 01:03:04.999');", postgres_table); - statement.execute(query); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - ResultSet rs=statement.executeQuery(String.format("SELECT user_declared_on FROM %s WHERE device_id='232455';", postgres_table)); - while(rs.next()) { - assertEquals("2019-09-24 01:03:04.999", rs.getString(1)); - } - } - - @Test - public void shouldInsertSpecialCharactersifPresent() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS_WITH_SPECIAL_CHAR); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - ResultSet rs=statement.executeQuery(String.format("SELECT user_declared_state FROM %s WHERE device_id='568089542';", postgres_table)); - while(rs.next()) { - assertEquals("Karnataka's", rs.getString(1)); - } - } - - @Test - public void shouldInsertSpaceCharactersifPresent() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS_WITH_SPACE_CHAR); - - DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock); - deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock); - - ResultSet rs=statement.executeQuery(String.format("SELECT user_declared_district FROM %s WHERE device_id='test-did';", postgres_table)); - while(rs.next()) { - assertEquals("BENGALURU URBAN SOUTH", rs.getString(1)); - } - } -} diff --git a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java b/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java deleted file mode 100644 index adf91f919f..0000000000 --- a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.ekstep.ep.samza.service.Fixtures; - -public class EventFixture { - public static final String DEVICE_PROFILE_DETAILS = "{\n" + - "\"fcm_token\" : \"\",\n" + - "\"city\" : \"Bengaluru\",\n" + - "\"device_id\" : \"232455\",\n" + - "\"device_spec\" : \"{'os':'Android 6.0','cpu':'abi: armeabi-v7a ARMv7 Processor rev 4 (v7l)','make':'Motorola XT1706'}\",\n" + - "\"state\" : \"Karnataka\",\n" + - "\"uaspec\" : \"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\n" + - "\"country\" : \"India\",\n" + - "\"country_code\" : \"IN\",\n" + - "\"producer_id\" : \"dev.sunbird.portal\",\n" + - "\"state_code_custom\" : 29,\n" + - "\"state_code\" : \"KA\",\n" + - "\"state_custom\" : \"Karnataka\",\n" + - "\"district_custom\" : \"Karnataka\",\n" + - "\"first_access\": 1568377184000,\n" + - "\"api_last_updated_on\": 1568377184000,\n" + - "\"user_declared_district\" : \"Bengaluru\",\n" + - "\"user_declared_state\" : \"Karnataka\"\n" + - "}"; - - public static final String DEVICE_PROFILE_WITH_STATE_NULL = "{\n" + - "\"fcm_token\" : \"\",\n" + - "\"city\" : \"Bengaluru\",\n" + - "\"device_id\" : \"232455\",\n" + - "\"device_spec\" : \"{'os':'Android 6.0','cpu':'abi: armeabi-v7a ARMv7 Processor rev 4 (v7l)','make':'Motorola XT1706'}\",\n" + - "\"state\" : \"Karnataka\",\n" + - "\"uaspec\" : \"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\n" + - "\"country\" : \"India\",\n" + - "\"country_code\" : \"IN\",\n" + - "\"producer_id\" : \"dev.sunbird.portal\",\n" + - "\"state_code_custom\" : 29,\n" + - "\"state_code\" : \"KA\",\n" + - "\"state_custom\" : \"Karnataka\",\n" + - "\"district_custom\" : \"Karnataka\",\n" + - "\"first_access\": 1568377184000,\n" + - "\"api_last_updated_on\": 1568377184000\n" + - "}"; - - public static final String DEVICE_PROFILE_WITH_NO_DEVICE_ID = "{\n" + - "\"fcm_token\" : \"\",\n" + - "\"city\" : \"Bengaluru\",\n" + - "\"device_id\" : \"\",\n" + - "\"device_spec\" : \"{'os':'Android 6.0','cpu':'abi: armeabi-v7a ARMv7 Processor rev 4 (v7l)','make':'Motorola XT1706'}\",\n" + - "\"state\" : \"Karnataka\",\n" + - "\"uaspec\" : \"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\n" + - "\"country\" : \"India\",\n" + - "\"country_code\" : \"IN\",\n" + - "\"producer_id\" : \"dev.sunbird.portal\",\n" + - "\"state_code_custom\" : 29,\n" + - "\"state_code\" : \"KA\",\n" + - "\"state_custom\" : \"Karnataka\",\n" + - "\"district_custom\" : \"Karnataka\",\n" + - "\"first_access\": 1568377184000,\n" + - "\"api_last_updated_on\": 1568377184000,\n" + - "\"user_declared_district\" : \"Bengaluru\",\n" + - "\"user_declared_state\" : \"Karnataka\"\n" + - "}"; - - public static final String DEVICE_PROFILE_DETAILS_WITH_SPECIAL_CHAR = "{\n" + - "\"fcm_token\" : \"\",\n" + - "\"device_id\" : \"568089542\",\n" + - "\"state\" : \"Karnataka\",\n" + - "\"first_access\": 1568377184000,\n" + - "\"api_last_updated_on\": 1568377184000,\n" + - "\"user_declared_district\" : \"Bengaluru\",\n" + - "\"user_declared_state\" : \"Karnataka's\"\n" + - "}"; - - public static final String DEVICE_PROFILE_DETAILS_WITH_SPACE_CHAR = "{\n" + - "\"fcm_token\" : \"\",\n" + - "\"device_id\" : \"test-did\",\n" + - "\"state\" : \"Karnataka\",\n" + - "\"first_access\": 1568377184000,\n" + - "\"api_last_updated_on\": 1568377184000,\n" + - "\"user_declared_district\" : \"BENGALURU URBAN SOUTH\",\n" + - "\"user_declared_state\" : \"Karnataka\"\n" + - "}"; - - public static final String MALFORMED_DEVICE_PROFILE_OBJECT= "{\"fcm_token\":\"\",\"city\":\"Bengaluru\",\"device_id\":\"232455\"\"device_spec”:”{‘make’}”,”state\":\"Karnataka\",\"uaspec\":\"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\"country\":\"India\",\"updated_date\":1567765886277,\"country_code\":\"IN\",\"producer_id\":\"dev.sunbird.portal\",\"state_code_custom\":\"29\",\"state_code\":\"KA\",\"state_custom\":\"Karnataka\",\"district_custom\":\"null\"}"; -} - diff --git a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/task/DeviceProfileConfigTest.java b/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/task/DeviceProfileConfigTest.java deleted file mode 100644 index 58dff6322e..0000000000 --- a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/task/DeviceProfileConfigTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.stub; - -public class DeviceProfileConfigTest { - private Config configMock; - - @Test - public void shouldGetTheConfigs() { - configMock = mock(Config.class); - stub(configMock.get("output.malformed.topic.name", "telemetry.malformed")).toReturn("telemetry.malformed"); - DeviceProfileUpdaterConfig config = new DeviceProfileUpdaterConfig(configMock); - assertTrue(config.jobName() == "DeviceProfileUpdater"); - assertTrue(config.malformedTopic() == "telemetry.malformed"); - } - -} diff --git a/data-pipeline/distribution/pom.xml b/data-pipeline/distribution/pom.xml deleted file mode 100644 index 455f7566ff..0000000000 --- a/data-pipeline/distribution/pom.xml +++ /dev/null @@ -1,176 +0,0 @@ - - - 4.0.0 - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - 0.8.5 - - distribution - - pom - - Distribution - - - - - org.ekstep.ecosystem.jobs - telemetry-extractor - 0.1.5 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - telemetry-validator - 0.2.2 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - de-duplication - 0.5.1 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - derived-de-duplication - 0.0.3 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - telemetry-router - 0.1.6 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - telemetry-redacter - 0.0.1 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - telemetry-location-updater - 0.2.3 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - de-normalization - 0.1.7 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - druid-events-validator - 0.1.4 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - events-router - 0.1.7 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - user-cache-updater - 0.0.2 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - content-cache-updater - 0.0.4 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - device-profile-updater - 0.1.8 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - assessment-aggregator - 0.0.4 - tar.gz - distribution - - - org.ekstep.ecosystem.jobs - share-events-flattener - 0.0.1 - tar.gz - distribution - - - - - - - maven-assembly-plugin - - - distro-assembly - package - - single - - - - src/main/assembly/src.xml - - - - - - - org.jacoco - jacoco-maven-plugin - ${jacoco.plugin.version} - - - report-aggregate - prepare-package - - report-aggregate - - - - **/jacoco-unit.exec - - ${project.reporting.outputDirectory}/jacoco-aggregate - - - - - - - diff --git a/data-pipeline/distribution/src/main/assembly/src.xml b/data-pipeline/distribution/src/main/assembly/src.xml deleted file mode 100644 index ce6067813c..0000000000 --- a/data-pipeline/distribution/src/main/assembly/src.xml +++ /dev/null @@ -1,16 +0,0 @@ - - distribution - false - - - tar.gz - - - - - false - false - - - - diff --git a/data-pipeline/druid-events-validator/pom.xml b/data-pipeline/druid-events-validator/pom.xml deleted file mode 100644 index b626ef7a6b..0000000000 --- a/data-pipeline/druid-events-validator/pom.xml +++ /dev/null @@ -1,180 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - org.ekstep.ecosystem.jobs - druid-events-validator - 0.1.4 - jar - DruidEventsValidator - - Validating DeNormalised Events Against Schema - - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - com.github.java-json-tools - json-schema-validator - 2.2.8 - - - com.google.guava - guava - 18.0 - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - diff --git a/data-pipeline/druid-events-validator/src/main/assembly/src.xml b/data-pipeline/druid-events-validator/src/main/assembly/src.xml deleted file mode 100644 index c52cb70b84..0000000000 --- a/data-pipeline/druid-events-validator/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/druid-events-validator.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:druid-events-validator - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/druid-events-validator/src/main/config/druid-events-validator.properties b/data-pipeline/druid-events-validator/src/main/config/druid-events-validator.properties deleted file mode 100644 index ecdc9fcb9c..0000000000 --- a/data-pipeline/druid-events-validator/src/main/config/druid-events-validator.properties +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.DruidEventsValidator - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__druid_events_validator_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.DruidEventsValidatorTask -task.inputs=kafka.__env__.telemetry.denorm -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -# Always start consuming from "kafka.__env__.telemetry.denorm" to only process messages that are published after the job is started -systems.kafka.samza.offset.default=upcoming -#systems.kafka.samza.reset.offset=true -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -output.success.topic.name=__env__.telemetry.denorm.valid -output.failed.topic.name=__env__.telemetry.failed -output.malformed.topic.name=__env__.telemetry.malformed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -telemetry.schema.path=schemas/telemetry -summary.schema.path=schemas/summary -event.schema.file=envelope.json -search.schema.file=search.json -log.schema.file=log.json -pipeline.metrics.list=success-message-count,failed-message-count,error-message-count \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index 723d10d5e8..0000000000 --- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.task.DruidEventsValidatorConfig; - -import java.util.HashMap; -import java.util.Map; - -public class Event extends Events { - - - public Event(Map map) { - super(map); - } - - public void markSuccess() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.dv_processed", true); - telemetry.add("type", "events"); - } - - public void markFailure(String error, DruidEventsValidatorConfig config) { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.dv_processed", false); - - telemetry.addFieldIfAbsent("metadata", new HashMap()); - if (null != error) { - telemetry.add("metadata.dv_error", error); - telemetry.add("metadata.src", config.jobName()); - } - - } - - public boolean isSummaryEvent() { - return eid() != null && eid().equals("ME_WORKFLOW_SUMMARY"); - } - - public boolean isSearchEvent() { - return "SEARCH".equalsIgnoreCase(eid()); - } - - public boolean isLogEvent() { - return "LOG".equalsIgnoreCase(eid()); - } -} diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/service/DruidEventsValidatorService.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/service/DruidEventsValidatorService.java deleted file mode 100644 index c711a58770..0000000000 --- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/service/DruidEventsValidatorService.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import com.google.gson.JsonSyntaxException; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.DruidEventsValidatorConfig; -import org.ekstep.ep.samza.task.DruidEventsValidatorSink; -import org.ekstep.ep.samza.task.DruidEventsValidatorSource; -import org.ekstep.ep.samza.util.SchemaValidator; - -import static java.text.MessageFormat.format; - -public class DruidEventsValidatorService { - private static Logger LOGGER = new Logger(DruidEventsValidatorService.class); - private final DruidEventsValidatorConfig config; - private final SchemaValidator schemaValidator; - - public DruidEventsValidatorService(DruidEventsValidatorConfig config, SchemaValidator schemaValidator) { - this.config = config; - this.schemaValidator = schemaValidator; - } - - public void process(DruidEventsValidatorSource source, DruidEventsValidatorSink sink, JsonSchemaFactory jsonSchemaFactory) { - Event event = null; - try { - event = source.getEvent(); - ProcessingReport report = schemaValidator.validate(event); - if (report.isSuccess()) { - event.markSuccess(); - sink.toSuccessTopic(event); - } else { - String fieldName = this.getInvalidFieldName(report.toString()); - LOGGER.error(null, "VALIDATION FAILED: " + report.toString()); - sink.toFailedTopic(event, "Invalid field:" + fieldName); - } - } catch (JsonSyntaxException e) { - LOGGER.error(null, "INVALID EVENT: " + source.getMessage()); - sink.toMalformedEventsTopic(source.getMessage()); - } catch (Exception e) { - LOGGER.error(null, format( - "EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:", - event), e); - sink.toErrorTopic(event, e.getMessage()); - } - } - - private String getInvalidFieldName(String errorInfo) { - String[] message = errorInfo.split("reports:"); - String noFieldNameMsg = "unable to get the field name"; - if (message.length > 1) { - String[] fields = message[1].split(","); - if (fields.length > 2) { - String[] pointer = fields[3].split("\"pointer\":"); - return pointer[1].substring(0, pointer[1].length() - 1); - } - } - return noFieldNameMsg; - } - -} diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorConfig.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorConfig.java deleted file mode 100644 index 215714a270..0000000000 --- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.ekstep.ep.samza.task; - - -import org.apache.samza.config.Config; - -public class DruidEventsValidatorConfig { - - private final String JOB_NAME = "DruidEventsValidator"; - private String telemetrySchemaPath; - private String summarySchemaPath; - private String defaultSchemafile; - private String successTopic; - private String failedTopic; - private String malformedTopic; - private String searchSchemafile; - private String logSchemafile; - - public DruidEventsValidatorConfig(Config config) { - successTopic = config.get("output.success.topic.name", "telemetry.denorm.valid"); - failedTopic = config.get("output.failed.topic.name", "telemetry.failed"); - malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed"); - telemetrySchemaPath = config.get("telemetry.schema.path", "schemas/telemetry"); - summarySchemaPath = config.get("summary.schema.path", "schemas/summary"); - defaultSchemafile = config.get("event.schema.file","envelope.json"); - searchSchemafile = config.get("search.schema.file","search.json"); - logSchemafile = config.get("log.schema.file","log.json"); - } - - public String successTopic() { - return successTopic; - } - - public String failedTopic() { - return failedTopic; - } - - public String malformedTopic() { - return malformedTopic; - } - - public String telemetrySchemaPath() { return telemetrySchemaPath; } - - public String summarySchemaPath() { return summarySchemaPath; } - - public String defaultSchemafile(){return defaultSchemafile;} - - public String jobName() { - return JOB_NAME; - } - - public String searchSchemafile() { return searchSchemafile; } - - public String logSchemafile() { return logSchemafile; } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSink.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSink.java deleted file mode 100644 index cb056b760d..0000000000 --- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSink.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.ekstep.ep.samza.task; - -// import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - -public class DruidEventsValidatorSink extends BaseSink { - - // private JobMetrics metrics; - private DruidEventsValidatorConfig config; - - public DruidEventsValidatorSink(MessageCollector collector, JobMetrics metrics, - DruidEventsValidatorConfig config) { - - super(collector, metrics); - // this.metrics = metrics; - this.config = config; - } - - public void toSuccessTopic(Event event) { - toTopic(config.successTopic(), event.mid(), event.getJson()); - metrics.incSuccessCounter(); - } - - public void toFailedTopic(Event event, String failedMessage) { - event.markFailure(failedMessage, config); - toTopic(config.failedTopic(), event.mid(), event.getJson()); - metrics.incFailedCounter(); - } - - public void toErrorTopic(Event event, String errorMessage) { - event.markFailure(errorMessage, config); - toTopic(config.failedTopic(), event.mid(), event.getJson()); - metrics.incErrorCounter(); - } - - public void toMalformedEventsTopic(String message) { - toTopic(config.malformedTopic(), null, message); - metrics.incFailedCounter(); - } - -} diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSource.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSource.java deleted file mode 100644 index 3942e8cc89..0000000000 --- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSource.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ekstep.ep.samza.task; - -import java.util.Map; - -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; - -import com.google.gson.Gson; - -public class DruidEventsValidatorSource { - static Logger LOGGER = new Logger(DruidEventsValidatorSource.class); - - private IncomingMessageEnvelope envelope; - - public DruidEventsValidatorSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - return new Event(getMap()); - } - - @SuppressWarnings("unchecked") - private Map getMap() { - String message = (String) envelope.getMessage(); - return (Map) new Gson().fromJson(message, Map.class); - } - - public String getMessage() { - return envelope.toString(); - } - - -} diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorTask.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorTask.java deleted file mode 100644 index 3f243bfed5..0000000000 --- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorTask.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.service.DruidEventsValidatorService; -import org.ekstep.ep.samza.util.SchemaValidator; - -import com.github.fge.jsonschema.main.JsonSchemaFactory; - -public class DruidEventsValidatorTask extends BaseSamzaTask { - - static Logger LOGGER = new Logger(DruidEventsValidatorTask.class); - private DruidEventsValidatorConfig config; - private JobMetrics metrics; - private DruidEventsValidatorService service; - private JsonSchemaFactory jsonSchemaFactory; - private SchemaValidator schemaValidator; - - public DruidEventsValidatorTask(Config config, TaskContext context) throws Exception { - init(config, context); - } - - public DruidEventsValidatorTask() { - - } - - @Override - public void init(Config config, TaskContext context) throws Exception { - this.config = new DruidEventsValidatorConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - this.schemaValidator = schemaValidator == null ? new SchemaValidator(this.config) : schemaValidator; - service = new DruidEventsValidatorService(this.config, this.schemaValidator); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, - TaskCoordinator taskCoordinator) throws Exception { - DruidEventsValidatorSource source = new DruidEventsValidatorSource(envelope); - DruidEventsValidatorSink sink = new DruidEventsValidatorSink(collector, metrics, config); - jsonSchemaFactory = JsonSchemaFactory.byDefault(); - service.process(source, sink, jsonSchemaFactory); - } - -} diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/util/SchemaValidator.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/util/SchemaValidator.java deleted file mode 100644 index 7d22e8d045..0000000000 --- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/util/SchemaValidator.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.ekstep.ep.samza.util; - -import com.fasterxml.jackson.databind.JsonNode; -import com.github.fge.jackson.JsonLoader; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchema; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import com.google.common.io.ByteStreams; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.DruidEventsValidatorConfig; - -import java.io.IOException; -import java.text.MessageFormat; - -public class SchemaValidator { - - private final JsonSchema telemetryJsonSchema; - private final JsonSchema summaryJsonSchema; - private final JsonSchema searchEventJsonSchema; - private final JsonSchema logEventJsonSchema; - private JsonSchemaFactory schemaFactory; - - public SchemaValidator(DruidEventsValidatorConfig config) throws IOException, ProcessingException { - this.schemaFactory = JsonSchemaFactory.byDefault(); - String telemetrySchemaPath = MessageFormat.format("{0}/{1}", config.telemetrySchemaPath(), - config.defaultSchemafile()); - String summaryEventSchemapath = MessageFormat.format("{0}/{1}", config.summarySchemaPath(), - config.defaultSchemafile()); - String searchEventSchemaPath = MessageFormat.format("{0}/{1}", config.telemetrySchemaPath(), - config.searchSchemafile()); - String logEventSchemaPath = MessageFormat.format("{0}/{1}", config.telemetrySchemaPath(), - config.logSchemafile()); - - String telemetrySchema = new String(ByteStreams.toByteArray(this.getClass().getClassLoader(). - getResourceAsStream(telemetrySchemaPath))); - String summarySchema = new String(ByteStreams.toByteArray(this.getClass().getClassLoader(). - getResourceAsStream(summaryEventSchemapath))); - String searchEventSchema = new String(ByteStreams.toByteArray(this.getClass().getClassLoader(). - getResourceAsStream(searchEventSchemaPath))); - String logEventSchema = new String(ByteStreams.toByteArray(this.getClass().getClassLoader(). - getResourceAsStream(logEventSchemaPath))); - - this.telemetryJsonSchema = schemaFactory.getJsonSchema(JsonLoader.fromString(telemetrySchema)); - this.summaryJsonSchema = schemaFactory.getJsonSchema(JsonLoader.fromString(summarySchema)); - this.searchEventJsonSchema = schemaFactory.getJsonSchema(JsonLoader.fromString(searchEventSchema)); - this.logEventJsonSchema = schemaFactory.getJsonSchema(JsonLoader.fromString(logEventSchema)); - } - - public ProcessingReport validate(Event event) throws IOException, ProcessingException { - - JsonNode eventJson = JsonLoader.fromString(event.getJson()); - ProcessingReport report; - if (event.isSearchEvent()) { - report = searchEventJsonSchema.validate(eventJson); - } else if (event.isSummaryEvent()) { - report = summaryJsonSchema.validate(eventJson); - } else if (event.isLogEvent()) { - report = logEventJsonSchema.validate(eventJson); - } else { - report = telemetryJsonSchema.validate(eventJson); - } - return report; - } - -} diff --git a/data-pipeline/druid-events-validator/src/main/resources/log4j.xml b/data-pipeline/druid-events-validator/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/cdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/cdata.json deleted file mode 100644 index d855ee7b51..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/cdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "id": "http://api.sunbird.org/summary/cdata/type", - "type": "string" - }, - "id": { - "id": "http://api.sunbird.org/summary/cdata/id", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ "type", "id"] - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/collectiondata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/collectiondata.json deleted file mode 100644 index f069aa65b6..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/collectiondata.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "type": "object", - "properties": { - "name": { - "id": "http://api.sunbird.org/telemetry/collectiondata/name", - "type": "string" - }, - "objecttype": { - "id": "http://api.sunbird.org/telemetry/collectiondata/objecttype", - "type": "string" - }, - "contenttype": { - "id": "http://api.sunbird.org/telemetry/collectiondata/contenttype", - "type": "string" - }, - "mediatype": { - "id": "http://api.sunbird.org/telemetry/collectiondata/mediatype", - "type": "string" - }, - "language": { - "id": "http://api.sunbird.org/telemetry/collectiondata/language", - "type": "array" - }, - "medium": { - "id": "http://api.sunbird.org/telemetry/collectiondata/medium", - "type": "array" - }, - "gradelevel": { - "id": "http://api.sunbird.org/telemetry/collectiondata/gradelevel", - "type": "array" - }, - "subject": { - "id": "http://api.sunbird.org/telemetry/collectiondata/subject", - "type": "array" - }, - "mimetype": { - "id": "http://api.sunbird.org/telemetry/collectiondata/mimetype", - "type": "string" - }, - "createdby": { - "id": "http://api.sunbird.org/telemetry/collectiondata/createdby", - "type": "string" - }, - "createdfor": { - "id": "http://api.sunbird.org/telemetry/collectiondata/createdfor", - "type": "array" - }, - "framework": { - "id": "http://api.sunbird.org/telemetry/collectiondata/framework", - "type": "string" - }, - "board": { - "id": "http://api.sunbird.org/telemetry/collectiondata/board", - "type": "string" - }, - "status": { - "id": "http://api.sunbird.org/telemetry/collectiondata/status", - "type": "string" - }, - "pkgversion": { - "id": "http://api.sunbird.org/telemetry/collectiondata/pkgversion", - "type": "number" - }, - "lastsubmittedon": { - "id": "http://api.sunbird.org/telemetry/collectiondata/lastsubmittedon", - "type": "number" - }, - "lastpublishedon": { - "id": "http://api.sunbird.org/telemetry/collectiondata/lastpublishedon", - "type": "number" - }, - "lastupdatedon": { - "id": "http://api.sunbird.org/telemetry/collectiondata/lastupdatedon", - "type": "number" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/common.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/common.json deleted file mode 100644 index 6b2ee98324..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/common.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.sunbird.org/summary/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.sunbird.org/summary/ver", - "type": "string" - }, - "mid": { - "id": "http://api.sunbird.org/summary/mid", - "type": "string", - "minLength": 1 - }, - "context": { - "$ref": "resource:/schemas/summary/context.json" - }, - "object": { - "$ref": "resource:/schemas/summary/object.json" - }, - "dimensions": { - "$ref": "resource:/schemas/summary/dimension.json" - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "devicedata": { - "$ref": "resource:/schemas/summary/devicedata.json" - }, - "userdata": { - "$ref": "resource:/schemas/summary/userdata.json" - }, - "contentdata": { - "$ref": "resource:/schemas/summary/contentdata.json" - }, - "collectiondata": { - "$ref": "resource:/schemas/summary/collectiondata.json" - }, - "derivedlocationdata": { - "$ref": "resource:/schemas/summary/derivedlocationdata.json" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/contentdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/contentdata.json deleted file mode 100644 index e1fa75cc75..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/contentdata.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "type": "object", - "properties": { - "name": { - "id": "http://api.sunbird.org/telemetry/contentdata/name", - "type": "string" - }, - "objecttype": { - "id": "http://api.sunbird.org/telemetry/contentdata/objecttype", - "type": "string" - }, - "contenttype": { - "id": "http://api.sunbird.org/telemetry/contentdata/contenttype", - "type": "string" - }, - "mediatype": { - "id": "http://api.sunbird.org/telemetry/contentdata/mediatype", - "type": "string" - }, - "language": { - "id": "http://api.sunbird.org/telemetry/contentdata/language", - "type": "array" - }, - "medium": { - "id": "http://api.sunbird.org/telemetry/contentdata/medium", - "type": "array" - }, - "gradelevel": { - "id": "http://api.sunbird.org/telemetry/contentdata/gradelevel", - "type": "array" - }, - "createdby": { - "id": "http://api.sunbird.org/telemetry/contentdata/createdby", - "type": "string" - }, - "createdfor": { - "id": "http://api.sunbird.org/telemetry/contentdata/createdfor", - "type": "array" - }, - "subject": { - "id": "http://api.sunbird.org/telemetry/contentdata/subject", - "type": "array" - }, - "mimetype": { - "id": "http://api.sunbird.org/telemetry/contentdata/mimetype", - "type": "string" - }, - "framework": { - "id": "http://api.sunbird.org/telemetry/contentdata/framework", - "type": "string" - }, - "board": { - "id": "http://api.sunbird.org/telemetry/contentdata/board", - "type": "string" - }, - "status": { - "id": "http://api.sunbird.org/telemetry/contentdata/status", - "type": "string" - }, - "pkgversion": { - "id": "http://api.sunbird.org/telemetry/contentdata/pkgversion", - "type": "number" - }, - "lastsubmittedon": { - "id": "http://api.sunbird.org/telemetry/contentdata/lastsubmittedon", - "type": "number" - }, - "lastpublishedon": { - "id": "http://api.sunbird.org/telemetry/contentdata/lastpublishedon", - "type": "number" - }, - "lastupdatedon": { - "id": "http://api.sunbird.org/telemetry/contentdata/lastupdatedon", - "type": "number" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/context.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/context.json deleted file mode 100644 index 4d039d4abc..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/context.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "type": "object", - "properties": { - "pdata": { - "$ref": "resource:/schemas/summary/pdata.json" - }, - "granularity": { - "id": "http://api.sunbird.org/summary/context/granularity", - "type": "string" - }, - "date_range": { - "$ref": "resource:/schemas/summary/datarange.json" - }, - "rollup": { - "$ref": "resource:/schemas/summary/rollup.json" - }, - "did": { - "id": "http://api.sunbird.org/summary/context/did", - "type": "string" - }, - "cdata": { - "$ref": "resource:/schemas/summary/cdata.json" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/datarange.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/datarange.json deleted file mode 100644 index 036c7cc798..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/datarange.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "object", - "properties": { - "from": { - "id": "http://api.sunbird.org/summary/datarange/from", - "type": "number" - }, - "to": { - "id": "http://api.sunbird.org/summary/datarange/to", - "type": "number" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/derivedlocationdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/derivedlocationdata.json deleted file mode 100644 index e121bd4f46..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/derivedlocationdata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "type": "object", - "properties": { - "state": { - "id": "http://api.sunbird.org/telemetry/derivedlocationdata/state", - "type": "string" - }, - "district":{ - "id": "http://api.sunbird.org/telemetry/derivedlocationdata/district", - "type": "string" - }, - "from":{ - "id": "http://api.sunbird.org/telemetry/derivedlocationdata/from", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/devicedata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/devicedata.json deleted file mode 100644 index 555aa35265..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/devicedata.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "type": "object", - "properties": { - "uaspec": { - "$ref": "resource:/schemas/summary/uaspec.json" - }, - "firstaccess": { - "id": "http://api.sunbird.org/telemetry/devicedata/firstaccess", - "type": "number" - }, - "state": { - "id": "http://api.sunbird.org/telemetry/devicedata/state", - "type": "string" - }, - "statecode": { - "id": "http://api.sunbird.org/telemetry/devicedata/statecode", - "type": "string" - }, - "city": { - "id": "http://api.sunbird.org/telemetry/devicedata/city", - "type": "string" - }, - "countrycode": { - "id": "http://api.sunbird.org/telemetry/devicedata/countrycode", - "type": "string" - }, - "country": { - "id": "http://api.sunbird.org/telemetry/devicedata/country", - "type": "string" - }, - "os": { - "id": "http://api.sunbird.org/telemetry/devicedata/os", - "type": "string" - }, - "make": { - "id": "http://api.sunbird.org/telemetry/devicedata/make", - "type": "string" - }, - "id": { - "id": "http://api.sunbird.org/telemetry/devicedata/id", - "type": "string" - }, - "mem": { - "id": "http://api.sunbird.org/telemetry/devicedata/mem", - "type": "string" - }, - "idisk": { - "id": "http://api.sunbird.org/telemetry/devicedata/idisk", - "type": "string" - }, - "edisk": { - "id": "http://api.sunbird.org/telemetry/devicedata/edisk", - "type": "string" - }, - "scrn": { - "id": "http://api.sunbird.org/telemetry/devicedata/scrn", - "type": "string" - }, - "camera": { - "id": "http://api.sunbird.org/telemetry/devicedata/camera", - "type": "string" - }, - "cpu": { - "id": "http://api.sunbird.org/telemetry/devicedata/cpu", - "type": "string" - }, - "sims": { - "id": "http://api.sunbird.org/telemetry/devicedata/sims", - "type": "string" - }, - "iso3166statecode": { - "id": "http://api.sunbird.org/telemetry/devicedata/iso3166statecode", - "type": "string" - }, - "statecustomcode":{ - "id": "http://api.sunbird.org/telemetry/devicedata/statecustomcode", - "type": "string" - }, - "statecustomname":{ - "id": "http://api.sunbird.org/telemetry/devicedata/statecustomname", - "type": "string" - }, - "districtcustom":{ - "id": "http://api.sunbird.org/telemetry/devicedata/districtcustom", - "type": "string" - }, - "userdeclared": { - "type": "object", - "properties": { - "state": { - "id": "http://api.sunbird.org/telemetry/devicedata/userdeclared/state", - "type": "string" - }, - "district": { - "id": "http://api.sunbird.org/telemetry/devicedata/userdeclared/district", - "type": "string" - } - } - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension.json deleted file mode 100644 index 65a5624207..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "type": "object", - "properties": { - "channel": { - "id": "http://api.sunbird.org/summary/dimension/channel", - "type":"string" - }, - "sid": { - "id": "http://api.sunbird.org/summary/dimension/sid", - "type": "string" - }, - "did": { - "id": "http://api.sunbird.org/summary/dimension/did", - "type": "string" - }, - "pdata": { - "$ref": "resource:/schemas/summary/dimension_pdata.json" - }, - "type": { - "id": "http://api.sunbird.org/telemetry/dimension/type", - "type": "string" - }, - "mode": { - "id": "http://api.sunbird.org/telemetry/dimension/mode", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension_pdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension_pdata.json deleted file mode 100644 index 2467bd693d..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension_pdata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.sunbird.org/summary/pdata/id", - "type": "string" - }, - "pid": { - "id": "http://api.sunbird.org/summary/pdata/pid", - "type": "string" - }, - "ver": { - "id": "http://api.sunbird.org/summary/pdata/ver", - "type": "string" - } - }, - "required": [ - "id" - ] -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/envelope.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/envelope.json deleted file mode 100644 index b22367f837..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/envelope.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "id":"http://api.sunbird.org/summary/envelope", - "type":"object", - "required":[ - "eid", - "ets", - "ver", - "mid", - "context", - "edata" - ], - "allOf":[ - { - "$ref":"resource:/schemas/summary/common.json" - }, - { - "properties":{ - "eid":{ - "id":"http://api.sunbird.org/telemetry/mid", - "type":"string", - "minLength": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/me_workflow_summary.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/me_workflow_summary.json deleted file mode 100644 index d8a85ee0ae..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/me_workflow_summary.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "id": "http://api.sunbird.org/summary/me_workflow_summary", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "syncts", - "mid", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/schemas/summary/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.sunbird.org/summary/eid", - "enum": [ - "ME_WORKFLOW_SUMMARY" - ] - }, - "edata": { - "id": "http://api.sunbird.org/summary/edata", - "type": "object" - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/object.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/object.json deleted file mode 100644 index bfebbd3c6c..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/object.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.sunbird.org/summary/object/id", - "type": "string" - }, - "type": { - "id": "http://api.sunbird.org/summary/object/type", - "type": "string" - }, - "ver": { - "id": "http://api.sunbird.org/summary/object/ver", - "type": "string" - }, - "rollup": { - "$ref": "resource:/schemas/summary/rollup.json" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/pdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/pdata.json deleted file mode 100644 index dd29b0cc45..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/pdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.sunbird.org/summary/pdata/id", - "type": "string" - }, - "model": { - "id": "http://api.sunbird.org/summary/pdata/model", - "type": "string" - }, - "ver": { - "id": "http://api.sunbird.org/summary/pdata/ver", - "type": "string" - } - }, - "required": ["id"] -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/rollup.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/rollup.json deleted file mode 100644 index baf81742a3..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/rollup.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "type": "object", - "properties": { - "l1": { - "id": "http://api.sunbird.org/summary/context/rollup/l1", - "type": "string" - }, - "l2": { - "id": "http://api.sunbird.org/summary/context/rollup/l2", - "type": "string" - }, - "l3": { - "id": "http://api.sunbird.org/summary/context/rollup/l3", - "type": "string" - }, - "l4": { - "id": "http://api.sunbird.org/summary/context/rollup/l4", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/uaspec.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/uaspec.json deleted file mode 100644 index 138155b0b3..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/uaspec.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "type": "object", - "properties": { - "agent": { - "id": "http://api.sunbird.org/telemetry/uaspec/agent", - "type": "string" - }, - "ver": { - "id": "http://api.sunbird.org/telemetry/uaspec/ver", - "type": "string" - }, - "system": { - "id": "http://api.sunbird.org/telemetry/uaspec/system", - "type": "string" - }, - "platform": { - "id": "http://api.sunbird.org/telemetry/uaspec/platform", - "type": "string" - }, - "raw": { - "id": "http://api.sunbird.org/telemetry/uaspec/raw", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/userdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/userdata.json deleted file mode 100644 index 8e96683f62..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/userdata.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "type": "object", - "properties": { - "gradelist": { - "id": "http://api.sunbird.org/telemetry/userdata/gradelist", - "type": "array" - }, - "languagelist": { - "id": "http://api.sunbird.org/telemetry/userdata/languagelist", - "type": "array" - }, - "subjectlist": { - "id": "http://api.sunbird.org/telemetry/userdata/subjectlist", - "type": "array" - }, - "usertype": { - "id": "http://api.sunbird.org/telemetry/userdata/usertype", - "type": "string" - }, - "state": { - "id": "http://api.sunbird.org/telemetry/userdata/state", - "type": "string" - }, - "district": { - "id": "http://api.sunbird.org/telemetry/userdata/district", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/collectiondata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/collectiondata.json deleted file mode 100644 index f069aa65b6..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/collectiondata.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "type": "object", - "properties": { - "name": { - "id": "http://api.sunbird.org/telemetry/collectiondata/name", - "type": "string" - }, - "objecttype": { - "id": "http://api.sunbird.org/telemetry/collectiondata/objecttype", - "type": "string" - }, - "contenttype": { - "id": "http://api.sunbird.org/telemetry/collectiondata/contenttype", - "type": "string" - }, - "mediatype": { - "id": "http://api.sunbird.org/telemetry/collectiondata/mediatype", - "type": "string" - }, - "language": { - "id": "http://api.sunbird.org/telemetry/collectiondata/language", - "type": "array" - }, - "medium": { - "id": "http://api.sunbird.org/telemetry/collectiondata/medium", - "type": "array" - }, - "gradelevel": { - "id": "http://api.sunbird.org/telemetry/collectiondata/gradelevel", - "type": "array" - }, - "subject": { - "id": "http://api.sunbird.org/telemetry/collectiondata/subject", - "type": "array" - }, - "mimetype": { - "id": "http://api.sunbird.org/telemetry/collectiondata/mimetype", - "type": "string" - }, - "createdby": { - "id": "http://api.sunbird.org/telemetry/collectiondata/createdby", - "type": "string" - }, - "createdfor": { - "id": "http://api.sunbird.org/telemetry/collectiondata/createdfor", - "type": "array" - }, - "framework": { - "id": "http://api.sunbird.org/telemetry/collectiondata/framework", - "type": "string" - }, - "board": { - "id": "http://api.sunbird.org/telemetry/collectiondata/board", - "type": "string" - }, - "status": { - "id": "http://api.sunbird.org/telemetry/collectiondata/status", - "type": "string" - }, - "pkgversion": { - "id": "http://api.sunbird.org/telemetry/collectiondata/pkgversion", - "type": "number" - }, - "lastsubmittedon": { - "id": "http://api.sunbird.org/telemetry/collectiondata/lastsubmittedon", - "type": "number" - }, - "lastpublishedon": { - "id": "http://api.sunbird.org/telemetry/collectiondata/lastpublishedon", - "type": "number" - }, - "lastupdatedon": { - "id": "http://api.sunbird.org/telemetry/collectiondata/lastupdatedon", - "type": "number" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/common.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/common.json deleted file mode 100644 index a54d9cd888..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/common.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "type": "object", - "properties": { - "ver": { - "id": "http://api.sunbird.org/telemetry/ver", - "type": "string" - - }, - "devicedata": { - "$ref": "resource:/schemas/telemetry/devicedata.json" - }, - "contentdata": { - "$ref": "resource:/schemas/telemetry/contentdata.json" - }, - "collectiondata": { - "$ref": "resource:/schemas/telemetry/collectiondata.json" - }, - "dialcodedata": { - "$ref": "resource:/schemas/telemetry/dialcodedata.json" - }, - "userdata": { - "$ref": "resource:/schemas/telemetry/userdata.json" - }, - "derivedlocationdata": { - "$ref": "resource:/schemas/telemetry/derivedlocationdata.json" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/contentdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/contentdata.json deleted file mode 100644 index a21f612cab..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/contentdata.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "type": "object", - "properties": { - "name": { - "id": "http://api.sunbird.org/telemetry/contentdata/name", - "type": "string" - }, - "objecttype": { - "id": "http://api.sunbird.org/telemetry/contentdata/objecttype", - "type": "string" - }, - "contenttype": { - "id": "http://api.sunbird.org/telemetry/contentdata/contenttype", - "type": "string" - }, - "mediatype": { - "id": "http://api.sunbird.org/telemetry/contentdata/mediatype", - "type": "string" - }, - "language": { - "id": "http://api.sunbird.org/telemetry/contentdata/language", - "type": "array" - }, - "medium": { - "id": "http://api.sunbird.org/telemetry/contentdata/medium", - "type": "array" - }, - "gradelevel": { - "id": "http://api.sunbird.org/telemetry/contentdata/gradelevel", - "type": "array" - }, - "subject": { - "id": "http://api.sunbird.org/telemetry/contentdata/subject", - "type": "array" - }, - "mimetype": { - "id": "http://api.sunbird.org/telemetry/contentdata/mimetype", - "type": "string" - }, - "createdby": { - "id": "http://api.sunbird.org/telemetry/contentdata/createdby", - "type": "string" - }, - "createdfor": { - "id": "http://api.sunbird.org/telemetry/contentdata/createdfor", - "type": "array" - }, - "framework": { - "id": "http://api.sunbird.org/telemetry/contentdata/framework", - "type": "string" - }, - "board": { - "id": "http://api.sunbird.org/telemetry/contentdata/board", - "type": "string" - }, - "status": { - "id": "http://api.sunbird.org/telemetry/contentdata/status", - "type": "string" - }, - "pkgversion": { - "id": "http://api.sunbird.org/telemetry/contentdata/pkgversion", - "type": "number" - }, - "lastsubmittedon": { - "id": "http://api.sunbird.org/telemetry/contentdata/lastsubmittedon", - "type": "number" - }, - "lastpublishedon": { - "id": "http://api.sunbird.org/telemetry/contentdata/lastpublishedon", - "type": "number" - }, - "lastupdatedon": { - "id": "http://api.sunbird.org/telemetry/contentdata/lastupdatedon", - "type": "number" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/derivedlocationdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/derivedlocationdata.json deleted file mode 100644 index e121bd4f46..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/derivedlocationdata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "type": "object", - "properties": { - "state": { - "id": "http://api.sunbird.org/telemetry/derivedlocationdata/state", - "type": "string" - }, - "district":{ - "id": "http://api.sunbird.org/telemetry/derivedlocationdata/district", - "type": "string" - }, - "from":{ - "id": "http://api.sunbird.org/telemetry/derivedlocationdata/from", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/devicedata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/devicedata.json deleted file mode 100644 index e24b79e01a..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/devicedata.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "type": "object", - "properties": { - "uaspec": { - "$ref": "resource:/schemas/telemetry/uaspec.json" - }, - "firstaccess": { - "id": "http://api.sunbird.org/telemetry/devicedata/firstaccess", - "type": "number" - }, - "state": { - "id": "http://api.sunbird.org/telemetry/devicedata/state", - "type": "string" - }, - "statecode": { - "id": "http://api.sunbird.org/telemetry/devicedata/statecode", - "type": "string" - }, - "city": { - "id": "http://api.sunbird.org/telemetry/devicedata/city", - "type": "string" - }, - "countrycode": { - "id": "http://api.sunbird.org/telemetry/devicedata/countrycode", - "type": "string" - }, - "country": { - "id": "http://api.sunbird.org/telemetry/devicedata/country", - "type": "string" - }, - "os": { - "id": "http://api.sunbird.org/telemetry/devicedata/os", - "type": "string" - }, - "make": { - "id": "http://api.sunbird.org/telemetry/devicedata/make", - "type": "string" - }, - "id": { - "id": "http://api.sunbird.org/telemetry/devicedata/id", - "type": "string" - }, - "mem": { - "id": "http://api.sunbird.org/telemetry/devicedata/mem", - "type": "string" - }, - "idisk": { - "id": "http://api.sunbird.org/telemetry/devicedata/idisk", - "type": "string" - }, - "edisk": { - "id": "http://api.sunbird.org/telemetry/devicedata/edisk", - "type": "string" - }, - "scrn": { - "id": "http://api.sunbird.org/telemetry/devicedata/scrn", - "type": "string" - }, - "camera": { - "id": "http://api.sunbird.org/telemetry/devicedata/camera", - "type": "string" - }, - "cpu": { - "id": "http://api.sunbird.org/telemetry/devicedata/cpu", - "type": "string" - }, - "sims": { - "id": "http://api.sunbird.org/telemetry/devicedata/sims", - "type": "string" - }, - "iso3166statecode": { - "id": "http://api.sunbird.org/telemetry/devicedata/iso3166statecode", - "type": "string" - }, - "statecustomcode":{ - "id": "http://api.sunbird.org/telemetry/devicedata/statecustomcode", - "type": "string" - }, - "statecustomname":{ - "id": "http://api.sunbird.org/telemetry/devicedata/statecustomname", - "type": "string" - }, - "districtcustom":{ - "id": "http://api.sunbird.org/telemetry/devicedata/districtcustom", - "type": "string" - }, - "userdeclared": { - "type": "object", - "properties": { - "state": { - "id": "http://api.sunbird.org/telemetry/devicedata/userdeclared/state", - "type": "string" - }, - "district": { - "id": "http://api.sunbird.org/telemetry/devicedata/userdeclared/district", - "type": "string" - } - } - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/dialcodedata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/dialcodedata.json deleted file mode 100644 index 06e979a5ff..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/dialcodedata.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "type": [ - "object", - "array" - ], - "properties": { - "channel": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/channel", - "type": "string" - }, - "identifier": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/identifier", - "type": "string" - }, - "batchcode": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/batchcode", - "type": "string" - }, - "publisher": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/publisher", - "type": "string" - }, - "generatedon": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/generatedon", - "type": "number" - }, - "publishedon": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/publishedon", - "type": "number" - }, - "objecttype": { - "id": "http://api.sunbird.org/telemetry/contentdata/objecttype", - "type": "string" - } - }, - "items": { - "type": "object", - "properties": { - "channel": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/channel", - "type": "string" - }, - "identifier": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/identifier", - "type": "string" - }, - "batchcode": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/batchcode", - "type": "string" - }, - "publisher": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/publisher", - "type": "string" - }, - "generatedon": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/generatedon", - "type": "number" - }, - "publishedon": { - "id": "http://api.sunbird.org/telemetry/dialcodedata/publishedon", - "type": "number" - }, - "objecttype": { - "id": "http://api.sunbird.org/telemetry/contentdata/objecttype", - "type": "string" - } - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/envelope.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/envelope.json deleted file mode 100644 index 8eef044eed..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/envelope.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "id":"http://api.sunbird.org/telemetry/envelope", - "type":"object", - "allOf":[ - { - "$ref":"resource:/schemas/telemetry/common.json" - }, - { - "properties":{ - "eid":{ - "id":"http://api.sunbird.org/telemetry/mid", - "type":"string", - "minLength": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/filter.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/filter.json deleted file mode 100644 index c6f61d886d..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/filter.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - }, - "anyOf": [ - { - "not": { - "required": [ - "dialCodes" - ] - } - } - ], - "additionalProperties": true -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/log.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/log.json deleted file mode 100644 index 114b80780c..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/log.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "id": "http://api.sunbird.org/telemetry/log", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/schemas/telemetry/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.sunbird.org/telemetry/eid", - "enum": [ - "LOG" - ] - }, - "edata": { - "id": "http://api.sunbird.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "level", - "message" - ], - "properties": { - "type": { - "id": "http://api.sunbird.org/telemetry/edata/type", - "type": "string" - }, - "level": { - "id": "http://api.sunbird.org/telemetry/edata/level", - "type": "string" - }, - "message": { - "id": "http://api.sunbird.org/telemetry/edata/message", - "type": "string" - }, - "pageid": { - "id": "http://api.sunbird.org/telemetry/edata/pageid", - "type": "string" - }, - "params": { - "id": "http://api.sunbird.org/telemetry/edata/params", - "type": "array", - "items": { - "type": "object" - }, - "properties": { - "rid": { - "id": "http://api.sunbird.org/telemetry/edata/params/rid", - "type": "string" - }, - "uip": { - "id": "http://api.sunbird.org/telemetry/edata/params/uip", - "type": "string" - }, - "title": { - "id": "http://api.sunbird.org/telemetry/edata/params/title", - "type": "string" - }, - "category": { - "id": "http://api.sunbird.org/telemetry/edata/params/category", - "type": "string" - }, - "url": { - "id": "http://api.sunbird.org/telemetry/edata/params/url", - "type": "string" - }, - "size": { - "id": "http://api.sunbird.org/telemetry/edata/params/size", - "type": "number" - }, - "duration": { - "id": "http://api.sunbird.org/telemetry/edata/params/duration", - "type": "number" - }, - "status": { - "id": "http://api.sunbird.org/telemetry/edata/params/status", - "type": "number" - }, - "protocol": { - "id": "http://api.sunbird.org/telemetry/edata/params/protocol", - "type": "string" - }, - "method": { - "id": "http://api.sunbird.org/telemetry/edata/params/method", - "type": "string" - }, - "req": { - "id": "http://api.sunbird.org/telemetry/edata/params/req", - "type": "string" - }, - "consumer_id": { - "id": "http://api.sunbird.org/telemetry/edata/params/consumer_id", - "type": "string" - }, - "ver": { - "id": "http://api.sunbird.org/telemetry/edata/params/ver", - "type": "string" - }, - "events_count": { - "id": "http://api.sunbird.org/telemetry/edata/params/events_count", - "type": "string" - }, - "sync_status": { - "id": "http://api.sunbird.org/telemetry/edata/params/sync_status", - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/search.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/search.json deleted file mode 100644 index 73ddce9313..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/search.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "id": "http://api.sunbird.org/telemetry/search", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/schemas/telemetry/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.sunbird.org/telemetry/eid", - "enum": [ - "SEARCH" - ] - }, - "edata": { - "id": "http://api.sunbird.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "query", - "size", - "topn" - ], - "properties": { - "type": { - "id": "http://api.sunbird.org/telemetry/edata/type", - "type": "string" - }, - "query": { - "id": "http://api.sunbird.org/telemetry/edata/query", - "type": "string" - }, - "sort": { - "id": "http://api.sunbird.org/telemetry/edata/sort", - "type": "object" - }, - "correlationid": { - "id": "http://api.sunbird.org/telemetry/edata/correlationid", - "type": "string" - }, - "size": { - "id": "http://api.sunbird.org/telemetry/edata/size", - "type": "number" - }, - "filters": { - "$ref": "resource:/schemas/telemetry/filter.json" - }, - "topn": { - "id": "http://api.sunbird.org/telemetry/edata/topn", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/uaspec.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/uaspec.json deleted file mode 100644 index 138155b0b3..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/uaspec.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "type": "object", - "properties": { - "agent": { - "id": "http://api.sunbird.org/telemetry/uaspec/agent", - "type": "string" - }, - "ver": { - "id": "http://api.sunbird.org/telemetry/uaspec/ver", - "type": "string" - }, - "system": { - "id": "http://api.sunbird.org/telemetry/uaspec/system", - "type": "string" - }, - "platform": { - "id": "http://api.sunbird.org/telemetry/uaspec/platform", - "type": "string" - }, - "raw": { - "id": "http://api.sunbird.org/telemetry/uaspec/raw", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/userdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/userdata.json deleted file mode 100644 index bd80c89196..0000000000 --- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/userdata.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "type": "object", - "properties": { - "gradelist": { - "id": "http://api.sunbird.org/telemetry/userdata/gradelist", - "type": "array" - }, - "languagelist": { - "id": "http://api.sunbird.org/telemetry/userdata/languagelist", - "type": "array" - }, - "subjectlist": { - "id": "http://api.sunbird.org/telemetry/userdata/subjectlist", - "type": "array" - }, - "usertype": { - "id": "http://api.sunbird.org/telemetry/userdata/usertype", - "type": "string" - }, - "state": { - "id": "http://api.sunbird.org/telemetry/userdata/state", - "type": "string" - }, - "district": { - "id": "http://api.sunbird.org/telemetry/userdata/district", - "type": "string" - }, - "usersignintype": { - "id": "http://api.sunbird.org/telemetry/usersignintype/district", - "type": "string" - }, - "userlogintype": { - "id": "http://api.sunbird.org/telemetry/userdata/userlogintype", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/SummaryV1.java b/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/SummaryV1.java deleted file mode 100644 index d584ec988a..0000000000 --- a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/SummaryV1.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -public class SummaryV1 { - /** - * Valid Summary Event - */ - public static final String VALID_EVENT = "{\"eid\":\"ME_WORKFLOW_SUMMARY\",\"ets\":1551409245701,\"syncts\":1551245418895,\"ver\":\"1.0\",\"mid\":\"4E12E343B0FD99D17490C1BD0DB69B4F\",\"uid\":\"8ec91293-21d8-4af3-aeaa-275b35dc8c98\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"WorkflowSummarizer\"},\"granularity\":\"SESSION\",\"date_range\":{\"from\":1551244161206,\"to\":1551245418814},\"rollup\":{},\"cdata\":[{\"id\":\"9d282a8cbaf1546462a4851691b3bc00\",\"type\":\"ContentSession\"}]},\"dimensions\":{\"did\":\"518bab62deac8129f563077b4f1ba516\",\"pdata\":{\"id\":\"staging.diksha.portal\",\"ver\":\"1.11.0\",\"pid\":\"sunbird-portal.contenteditor.contentplayer\"},\"sid\":\"nlYPjt8_GoxB8LDtldOChvz7YH3odbKH\",\"channel\":\"0124784842112040965\",\"type\":\"content\",\"mode\":\"edit\"},\"edata\":{\"eks\":{\"interact_events_per_min\":6.57,\"start_time\":1551244161206,\"interact_events_count\":63,\"item_responses\":[],\"end_time\":1551245418814,\"events_summary\":[{\"id\":\"START\",\"count\":1},{\"id\":\"IMPRESSION\",\"count\":21},{\"id\":\"INTERACT\",\"count\":63},{\"id\":\"END\",\"count\":1}],\"page_summary\":[{\"id\":\"d9a95168-3e15-4aac-91e6-b8505116a497\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":193.83,\"visit_count\":1},{\"id\":\"c0e843b3-ec91-42ed-8e21-21250d1cb205\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":15.07,\"visit_count\":1},{\"id\":\"aaada2d5-6199-40d5-a85c-e2c778e8b18e\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":2.06,\"visit_count\":1},{\"id\":\"5d710ee4-b589-49bb-ad1d-8c1d08e6a83a\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":17.33,\"visit_count\":1},{\"id\":\"62481bee-3d01-4d5c-b0e9-50b837056604\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":12.75,\"visit_count\":1},{\"id\":\"6cb0248e-b9ae-4beb-817a-0ec86b638d07\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":119.6,\"visit_count\":1},{\"id\":\"36a9709b-e218-49b4-a19c-1a44947ab84d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":113.67,\"visit_count\":1},{\"id\":\"dcd4adc7-7959-45c5-8aa5-0a75b269850d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":1365.73,\"visit_count\":1},{\"id\":\"c40dd5a6-6375-42c7-8ce1-dba65a8a1826\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":5.36,\"visit_count\":1},{\"id\":\"2b0278a0-6673-4ba0-9ba9-17b5bcc619d0\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":35.19,\"visit_count\":1},{\"id\":\"244d4eee-17d4-4bad-8be3-0a8849dea53c\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":19.36,\"visit_count\":1},{\"id\":\"5786c403-d267-4329-a61a-3dccc466a4ed\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":9.07,\"visit_count\":1}],\"time_diff\":1257.61,\"telemetry_version\":\"3.0\",\"env_summary\":[{\"env\":\"contentplayer\",\"time_spent\":1909.02,\"count\":1}],\"time_spent\":575.19}},\"tags\":[],\"object\":{\"id\":\"do_21268948156283289611498\",\"type\":\"Content\",\"ver\":\"2\"}}"; - - /** - * Invalid dimension data type Event - */ - public static final String INVALID_EVENT= "{\"eid\":\"ME_WORKFLOW_SUMMARY\",\"ets\":1551409245701,\"syncts\":1551245418895,\"ver\":\"1.0\",\"mid\":\"4E12E343B0FD99D17490C1BD0DB69B4F\",\"uid\":\"8ec91293-21d8-4af3-aeaa-275b35dc8c98\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"WorkflowSummarizer\"},\"granularity\":\"SESSION\",\"date_range\":{\"from\":1551244161206,\"to\":1551245418814},\"rollup\":{},\"cdata\":[{\"id\":\"9d282a8cbaf1546462a4851691b3bc00\",\"type\":\"ContentSession\"}]},\"dimensions\":[],\"edata\":{\"eks\":{\"interact_events_per_min\":6.57,\"start_time\":1551244161206,\"interact_events_count\":63,\"item_responses\":[],\"end_time\":1551245418814,\"events_summary\":[{\"id\":\"START\",\"count\":1},{\"id\":\"IMPRESSION\",\"count\":21},{\"id\":\"INTERACT\",\"count\":63},{\"id\":\"END\",\"count\":1}],\"page_summary\":[{\"id\":\"d9a95168-3e15-4aac-91e6-b8505116a497\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":193.83,\"visit_count\":1},{\"id\":\"c0e843b3-ec91-42ed-8e21-21250d1cb205\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":15.07,\"visit_count\":1},{\"id\":\"aaada2d5-6199-40d5-a85c-e2c778e8b18e\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":2.06,\"visit_count\":1},{\"id\":\"5d710ee4-b589-49bb-ad1d-8c1d08e6a83a\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":17.33,\"visit_count\":1},{\"id\":\"62481bee-3d01-4d5c-b0e9-50b837056604\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":12.75,\"visit_count\":1},{\"id\":\"6cb0248e-b9ae-4beb-817a-0ec86b638d07\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":119.6,\"visit_count\":1},{\"id\":\"36a9709b-e218-49b4-a19c-1a44947ab84d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":113.67,\"visit_count\":1},{\"id\":\"dcd4adc7-7959-45c5-8aa5-0a75b269850d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":1365.73,\"visit_count\":1},{\"id\":\"c40dd5a6-6375-42c7-8ce1-dba65a8a1826\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":5.36,\"visit_count\":1},{\"id\":\"2b0278a0-6673-4ba0-9ba9-17b5bcc619d0\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":35.19,\"visit_count\":1},{\"id\":\"244d4eee-17d4-4bad-8be3-0a8849dea53c\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":19.36,\"visit_count\":1},{\"id\":\"5786c403-d267-4329-a61a-3dccc466a4ed\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":9.07,\"visit_count\":1}],\"time_diff\":1257.61,\"telemetry_version\":\"3.0\",\"env_summary\":[{\"env\":\"contentplayer\",\"time_spent\":1909.02,\"count\":1}],\"time_spent\":575.19}},\"tags\":[],\"object\":{\"id\":\"do_21268948156283289611498\",\"type\":\"Content\",\"ver\":\"2\"}}"; - - /** - * Invalid Tags data type - */ - public static final String INVALID_EVENT_CASE_1 ="{\"eid\":\"ME_WORKFLOW_SUMMARY\",\"ets\":1551409245701,\"syncts\":\"1551245418895\",\"ver\":\"1.0\",\"mid\":\"4E12E343B0FD99D17490C1BD0DB69B4F\",\"uid\":\"8ec91293-21d8-4af3-aeaa-275b35dc8c98\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"WorkflowSummarizer\"},\"granularity\":\"SESSION\",\"date_range\":{\"from\":1551244161206,\"to\":1551245418814},\"rollup\":{},\"cdata\":[{\"id\":\"9d282a8cbaf1546462a4851691b3bc00\",\"type\":\"ContentSession\"}]},\"dimensions\":{\"did\":\"518bab62deac8129f563077b4f1ba516\",\"pdata\":{\"id\":\"staging.diksha.portal\",\"ver\":\"1.11.0\",\"pid\":\"sunbird-portal.contenteditor.contentplayer\"},\"sid\":\"nlYPjt8_GoxB8LDtldOChvz7YH3odbKH\",\"channel\":\"0124784842112040965\",\"type\":\"content\",\"mode\":\"edit\"},\"edata\":{\"eks\":{\"interact_events_per_min\":6.57,\"start_time\":1551244161206,\"interact_events_count\":63,\"item_responses\":[],\"end_time\":1551245418814,\"events_summary\":[{\"id\":\"START\",\"count\":1},{\"id\":\"IMPRESSION\",\"count\":21},{\"id\":\"INTERACT\",\"count\":63},{\"id\":\"END\",\"count\":1}],\"page_summary\":[{\"id\":\"d9a95168-3e15-4aac-91e6-b8505116a497\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":193.83,\"visit_count\":1},{\"id\":\"c0e843b3-ec91-42ed-8e21-21250d1cb205\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":15.07,\"visit_count\":1},{\"id\":\"aaada2d5-6199-40d5-a85c-e2c778e8b18e\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":2.06,\"visit_count\":1},{\"id\":\"5d710ee4-b589-49bb-ad1d-8c1d08e6a83a\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":17.33,\"visit_count\":1},{\"id\":\"62481bee-3d01-4d5c-b0e9-50b837056604\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":12.75,\"visit_count\":1},{\"id\":\"6cb0248e-b9ae-4beb-817a-0ec86b638d07\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":119.6,\"visit_count\":1},{\"id\":\"36a9709b-e218-49b4-a19c-1a44947ab84d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":113.67,\"visit_count\":1},{\"id\":\"dcd4adc7-7959-45c5-8aa5-0a75b269850d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":1365.73,\"visit_count\":1},{\"id\":\"c40dd5a6-6375-42c7-8ce1-dba65a8a1826\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":5.36,\"visit_count\":1},{\"id\":\"2b0278a0-6673-4ba0-9ba9-17b5bcc619d0\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":35.19,\"visit_count\":1},{\"id\":\"244d4eee-17d4-4bad-8be3-0a8849dea53c\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":19.36,\"visit_count\":1},{\"id\":\"5786c403-d267-4329-a61a-3dccc466a4ed\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":9.07,\"visit_count\":1}],\"time_diff\":1257.61,\"telemetry_version\":\"3.0\",\"env_summary\":[{\"env\":\"contentplayer\",\"time_spent\":1909.02,\"count\":1}],\"time_spent\":575.19}},\"tags\":{},\"object\":{\"id\":\"do_21268948156283289611498\",\"type\":\"Content\",\"ver\":\"2\"}}"; - /** - * Invalid Tags data type - */ - public static final String INVALID_SUMMARY ="{\"eid\":\"ME_MONITOR_SUMMARY\",\"ets\":1553830612140,\"syncts\":1553817602545,\"ver\":\"2.1\",\"mid\":\"6ED077D95C3B097547544DFF127F647E\",\"uid\":\"\",\"channel\":\"\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"MonitorSummarizer\"},\"granularity\":\"DAY\",\"date_range\":{\"from\":1553817602545,\"to\":1553830609569}},\"dimensions\":{\"period\":20190329},\"edata\":{\"eks\":{\"total_events_generated\":2013,\"jobs_failed_count\":0,\"jobs_summary\":[{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"WorkFlowSummaryModel\",\"tag\":\"\",\"time_taken\":8,\"output_count\":52,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":38518,\"day\":20190329},{\"model\":\"LearnerSessionSummaryModel\",\"tag\":\"\",\"time_taken\":5,\"output_count\":21,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":38518,\"day\":20190329},{\"model\":\"ItemSummaryModel\",\"tag\":\"\",\"time_taken\":4,\"output_count\":6,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":21,\"day\":20190329},{\"model\":\"ItemUsageSummaryModel\",\"tag\":\"\",\"time_taken\":6,\"output_count\":24,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":6,\"day\":20190329},{\"model\":\"UsageSummaryModel\",\"tag\":\"\",\"time_taken\":5,\"output_count\":68,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":21,\"day\":20190329},{\"model\":\"UpdateItemSummaryDB\",\"tag\":\"\",\"time_taken\":1,\"output_count\":96,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":24,\"day\":20190329},{\"model\":\"WorkFlowUsageSummaryModel\",\"tag\":\"\",\"time_taken\":5,\"output_count\":348,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":52,\"day\":20190329},{\"model\":\"DeviceSummaryModel\",\"tag\":\"\",\"time_taken\":3,\"output_count\":3,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":38570,\"day\":20190329},{\"model\":\"UpdateDeviceProfileDB\",\"tag\":\"\",\"time_taken\":0,\"output_count\":1,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":3,\"day\":20190329},{\"model\":\"UpdateWorkFlowUsageDB\",\"tag\":\"\",\"time_taken\":1,\"output_count\":1392,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":348,\"day\":20190329},{\"model\":\"UpdateWorkFlowUsageMetricsModel\",\"tag\":\"\",\"time_taken\":0,\"output_count\":1,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":348,\"day\":20190329},{\"model\":\"UpdatePortalMetrics\",\"tag\":\"\",\"time_taken\":1,\"output_count\":1,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"DialcodeUsageSummaryModel\",\"tag\":\"\",\"time_taken\":12,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":38518,\"day\":20190329},{\"model\":\"ETBCoverageSummaryModel\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"UpdateDialcodeUsageDB\",\"tag\":\"\",\"time_taken\":1,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"DataExhaustJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329}],\"total_ts\":52,\"jobs_completed_count\":38,\"jobs_start_count\":38}},\"ts\":\"2019-03-29T03:36:52.140+0000\",\"ldata\":{\"country_code\":\"\",\"country\":\"\",\"city\":\"\",\"state\":\"\",\"state_code\":\"\"},\"flags\":{\"ldata_retrieved\":false}}"; - - public static final String UASPEC_SUMMARY="{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"KA\",\"country\":\"India\",\"city\":\"Banglore\",\"countrycode\":\"IND\",\"state\":\"Karnataka\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{\"platform\":123,\"ver\":\"\"},\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}"; - -} diff --git a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/TelemetryV3.java b/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/TelemetryV3.java deleted file mode 100644 index 82f2909332..0000000000 --- a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/TelemetryV3.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -public class TelemetryV3 { - /** - * Valid event with all proper data type - */ - public static final String VALID_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}"; - /** - * Passing Invalid event having wrong `pkgVersion` type - */ - public static final String INVALID_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":\"1\",\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}"; - - /** - * Passing Invalid event having wrong `gradelist` data type - */ - public static final String INVALID_USERDATA = "{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":43568797685,\"publishedon\":508978490534,\"status\":\"\",\"objecttype\":\"\"},\"userdata\":{\"gradelist\":{},\"languagelist\":[],\"subjectlist\":[],\"usertype\":\"\",\"state\":\"\",\"district\":\"\"}}"; - - /** - * Passing Invalid Device data having wrong `firstAccess` data type - */ - public static final String INVALID_DEVICEDATA_CASE_1 = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"KA\",\"country\":\"India\",\"city\":\"Banglore\",\"countrycode\":\"IND\",\"state\":\"Karnataka\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":\"1545920698694\"},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}"; - - /** - * Passing Invalid Device data having wrong `uaspec` data type - */ - public static final String INVALID_DEVICEDATA_CASE_2 = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"KA\",\"country\":\"India\",\"city\":\"Banglore\",\"countrycode\":\"IND\",\"state\":\"Karnataka\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":[],\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}"; - - /** - * Passing Invalid Device data having wrong `country` data type and missing few fields - */ - public static final String INVALID_DEVICEDATA_CASE_3 = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"KA\",\"country\":958743,\"city\":\"Banglore\",\"countrycode\":\"IND\",\"statecode\":\"\",\"districtcustom\":\"\",\"uaspec\":[],\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}"; - - /** - * Passing Invalid Device data having wrong `firstAccess` data type to float format - */ - public static final String INVALID_DEVICEDATA_CASE_4 = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":12},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}"; - - - /** - * Passing Invalid Event Having wrong data type format in Languages field - */ - public static final String INVALID_CONTENTDATA = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":125465324536475},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":{},\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}"; - - /** - * - */ - public static final String INVALID_DIALCODEDATA ="{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":43568797685,\"publishedon\":508978490534,\"objecttype\":1}}"; - - /** - * case sensitive dialcode key validation - * - */ - public static final String INVALID_DIALCODE_KEY ="{\"eid\":\"SEARCH\",\"ets\":1551977394372,\"ver\":\"3.0\",\"mid\":\"LP.1551977394372.d18d0bdf-9fbe-42ff-ac2c-3cb7306462a6\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"pdata\":{\"id\":\"prod.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\",\"did\":\"530f35ec69f30432bd5e87eee4e7d9a82da0e129\"},\"edata\":{\"size\":14,\"query\":\"\",\"filters\":{\"channels\":\"yes\",\"dialCodes\":\"\",\"objectType\":[\"Content\",\"ContentImage\"],\"contentType\":[\"Resource\"],\"status\":[\"Live\"],\"compatibilityLevel\":{\"min\":1,\"max\":4},\"channel\":{\"ne\":[\"0124433024890224640\",\"0124446042259128320\",\"0124487522476933120\",\"0125840271570288640\",\"0124453662635048969\"]},\"framework\":{},\"mimeType\":{},\"resourceType\":{}},\"sort\":{\"me_averageRating\":\"desc\"},\"type\":\"content\",\"topn\":[{\"identifier\":\"do_31251340904692121624142\"},{\"identifier\":\"do_31257347965634969621957\"},{\"identifier\":\"do_312580373603835904112177\"},{\"identifier\":\"do_312580362281156608111821\"},{\"identifier\":\"do_312580363689975808211635\"}]},\"cdata\":[{\"id\":\"prod.diksha.app\",\"type\":\"appId\"}],\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true},\"type\":\"events\",\"syncts\":1551977394569,\"@timestamp\":\"2019-03-07T16:49:54.569Z\",\"ts\":\"2019-03-07T16:49:54.372+0000\",\"devicedata\":{\"country\":\"India\",\"city\":\"Coimbatore\",\"countrycode\":\"IN\",\"state\":\"Tamil Nadu\",\"statecode\":\"TN\"}}"; - - /** - * - */ - public static final String VALID_DIALCODETYPE ="{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":[{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":43568797685,\"publishedon\":508978490534,\"status\":\"\",\"objecttype\":\"\"},{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":76854768347,\"publishedon\":508978490534,\"status\":\"\",\"objecttype\":\"\"}]}"; - - /** - * When array of object is having in the dialcodedata and which is having invalid property in one of the object in the array. ex: generatedon - */ - public static final String INVALID_DIALCODETYPE_CASE_1 ="{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":[{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":43568797685,\"publishedon\":508978490534,\"status\":\"\",\"objecttype\":\"\"},{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":\"43568797685\",\"publishedon\":508978490534,\"status\":true,\"objecttype\":\"\"}]}"; - - /** - * When dialcodedata object is having invalid properties - */ - public static final String INVALID_DIALCODETYPE_CASE_2 ="{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":{\"channel\":1234534534,\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":\"43568797685\",\"publishedon\":508978490534,\"status\":true,\"objecttype\":\"\"}}"; - - /** - * - */ - public static final String VALID_LOG_EVENT = "{\"eid\":\"LOG\",\"ets\":1561717784892,\"ver\":\"3.0\",\"mid\":\"LP.1561717784892.1345e38d-f799-403f-93d9-1db71011627f\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"pdata\":{\"id\":\"prod.diksha.app\",\"pid\":\"learning-service\",\"ver\":\"1.0\"},\"env\":\"content\",\"did\":\"beea4401850780ee685120597a6428ab14ff47a2\"},\"edata\":{\"level\":\"INFO\",\"type\":\"api_access\",\"message\":\"\",\"params\":[{\"duration\":5},{\"protocol\":\"HTTP/1.1\"},{\"size\":10350},{\"method\":\"GET\"},{\"rid\":\"ekstep.content.find\"},{\"uip\":\"11.4.0.37\"},{\"url\":\"/learning-service/content/v3/read/do_31274885743485747215761\"},{\"status\":200}]},\"syncts\":1561717784892,\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\",\"@timestamp\":\"2019-06-28T10:29:44.892Z\"}"; - - public static final String VALID_USER_DECLARED_AND_DERIVED_LOCATION_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"393407b1-66b1-4c86-9080-b2bce9842886\"},\"eid\":\"INTERACT\",\"edata\":{\"id\":\"ContentDetail\",\"pageid\":\"ContentDetail\",\"type\":\"TOUCH\",\"subtype\":\"ContentDownload-Initiate\"},\"ver\":\"3.0\",\"ets\":1.54157454518E12,\"context\":{\"pdata\":{\"ver\":\"2.1.8\",\"pid\":\"sunbird.app\",\"id\":\"prod.diksha.app\"},\"channel\":\"0123221617357783046602\",\"env\":\"sdk\",\"did\":\"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\"cdata\":[{\"type\":\"qr\",\"id\":\"K4KCXE\"},{\"type\":\"API\",\"id\":\"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"}],\"sid\":\"70ea93d0-e521-4030-934f-276e7194c225\"},\"mid\":\"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\"object\":{\"id\":\"do_31249561779090227216256\",\"type\":\"Content\",\"version\":\"\"},\"tags\":[],\"syncts\":1.539846605341E12,\"@timestamp\":\"2018-10-18T07:10:05.341Z\",\"derivedlocationdata\":{\"district\":\"Bengaluru\",\"state\":\"Karnataka\",\"from\":\"user-profile\"},\"devicedata\":{\"statecustomcode\":\"KA-Custom\",\"country\":\"India\",\"iso3166statecode\":\"IN-KA\",\"city\":\"Bangalore\",\"countrycode\":\"IN\",\"statecode\":\"KA\",\"devicespec\":{\"os\":\"Android 6.0\",\"make\":\"Motorola XT1706\"},\"districtcustom\":\"Banglore-Custom\",\"firstaccess\":1559484698000,\"uaspec\":{\"agent\":\"Mozilla\",\"ver\":\"5.0\",\"system\":\"iPad\",\"raw\":\"Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)\",\"platform\":\"AppleWebKit/531.21.10\"},\"state\":\"Karnataka\",\"statecustomname\":\"Karnatak-Custom\",\"userdeclared\":{\"district\":\"Bangalore\",\"state\":\"Karnataka\"}},\"flags\":{\"device_location_retrieved\":true,\"device_profile_retrieved\":true}}"; - - public static final String INVALID_USER_DECLARED_LOCATION_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"393407b1-66b1-4c86-9080-b2bce9842886\"},\"eid\":\"INTERACT\",\"edata\":{\"id\":\"ContentDetail\",\"pageid\":\"ContentDetail\",\"type\":\"TOUCH\",\"subtype\":\"ContentDownload-Initiate\"},\"ver\":\"3.0\",\"ets\":1.54157454518E12,\"context\":{\"pdata\":{\"ver\":\"2.1.8\",\"pid\":\"sunbird.app\",\"id\":\"prod.diksha.app\"},\"channel\":\"0123221617357783046602\",\"env\":\"sdk\",\"did\":\"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\"cdata\":[{\"type\":\"qr\",\"id\":\"K4KCXE\"},{\"type\":\"API\",\"id\":\"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"}],\"sid\":\"70ea93d0-e521-4030-934f-276e7194c225\"},\"mid\":\"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\"object\":{\"id\":\"do_31249561779090227216256\",\"type\":\"Content\",\"version\":\"\"},\"tags\":[],\"syncts\":1.539846605341E12,\"@timestamp\":\"2018-10-18T07:10:05.341Z\",\"derivedlocationdata\":{\"district\":\"Bengaluru\",\"state\":\"Karnataka\",\"from\":\"user-profile\"},\"devicedata\":{\"statecustomcode\":\"KA-Custom\",\"country\":\"India\",\"iso3166statecode\":\"IN-KA\",\"city\":\"Bangalore\",\"countrycode\":\"IN\",\"statecode\":\"KA\",\"devicespec\":{\"os\":\"Android 6.0\",\"make\":\"Motorola XT1706\"},\"districtcustom\":\"Banglore-Custom\",\"firstaccess\":1559484698000,\"uaspec\":{\"agent\":\"Mozilla\",\"ver\":\"5.0\",\"system\":\"iPad\",\"raw\":\"Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)\",\"platform\":\"AppleWebKit/531.21.10\"},\"state\":\"Karnataka\",\"statecustomname\":\"Karnatak-Custom\",\"userdeclared\":{\"district\":12,\"state\":\"Karnataka\"}},\"flags\":{\"device_location_retrieved\":true,\"device_profile_retrieved\":true}}"; - - public static final String IVALID_DERIVED_LOCATION_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"393407b1-66b1-4c86-9080-b2bce9842886\"},\"eid\":\"INTERACT\",\"edata\":{\"id\":\"ContentDetail\",\"pageid\":\"ContentDetail\",\"type\":\"TOUCH\",\"subtype\":\"ContentDownload-Initiate\"},\"ver\":\"3.0\",\"ets\":1.54157454518E12,\"context\":{\"pdata\":{\"ver\":\"2.1.8\",\"pid\":\"sunbird.app\",\"id\":\"prod.diksha.app\"},\"channel\":\"0123221617357783046602\",\"env\":\"sdk\",\"did\":\"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\"cdata\":[{\"type\":\"qr\",\"id\":\"K4KCXE\"},{\"type\":\"API\",\"id\":\"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"}],\"sid\":\"70ea93d0-e521-4030-934f-276e7194c225\"},\"mid\":\"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\"object\":{\"id\":\"do_31249561779090227216256\",\"type\":\"Content\",\"version\":\"\"},\"tags\":[],\"syncts\":1.539846605341E12,\"@timestamp\":\"2018-10-18T07:10:05.341Z\",\"derivedlocationdata\":{\"district\":\"Bengaluru\",\"state\":1234,\"from\":\"user-profile\"},\"devicedata\":{\"statecustomcode\":\"KA-Custom\",\"country\":\"India\",\"iso3166statecode\":\"IN-KA\",\"city\":\"Bangalore\",\"countrycode\":\"IN\",\"statecode\":\"KA\",\"devicespec\":{\"os\":\"Android 6.0\",\"make\":\"Motorola XT1706\"},\"districtcustom\":\"Banglore-Custom\",\"firstaccess\":1559484698000,\"uaspec\":{\"agent\":\"Mozilla\",\"ver\":\"5.0\",\"system\":\"iPad\",\"raw\":\"Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)\",\"platform\":\"AppleWebKit/531.21.10\"},\"state\":\"Karnataka\",\"statecustomname\":\"Karnatak-Custom\",\"userdeclared\":{\"district\":\"Bangalore\",\"state\":\"Karnataka\"}},\"flags\":{\"device_location_retrieved\":true,\"device_profile_retrieved\":true}}"; - - public static final String INVALID_JSON = "{\"eid\":\"LOG\",\"ets\":1570817279146,\"ver\":\"3.0\",\"mid\":\"LOG:2a210813fac274656f194f9807ad8abf\",\"actor\":{\"id\":\"1\",\"type\":\"service\"},\"context\":{\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"pdata\":{\"id\":\"prod.diksha.content-service\",\"ver\":\"1.0\",\"pid\":\"sunbird-content-service\"},\"env\":\"content\",\"sid\":\"\",\"did\":\"1e465bed2138b8f2764d946492293e2b2f628789\",\"cdata\":[],\"rollup\":{}},\"object\":{\"id\":\"do_312776645813747712111638\",\"type\":\"content\",\"ver\":\"\",\"rollup\":{}},\"tags\":,\"edata\":{\"type\":\"api_access\",\"level\":\"INFO\",\"message\":\"successful\",\"params\":[{\"rid\":\"0ece8640-ec52-11e9-a3d6-697ee5684e40\"},{\"title\":\"Content read api\"},{\"category\":\"contentread\"},{\"url\":\"content/read\"},{\"method\":\"GET\"}]},\"syncts\":1570817431399,\"@timestamp\":\"2019-10-11T18:10:31.399Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\"}"; - -} - diff --git a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/SummaryEventsValidatorTaskTest.java b/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/SummaryEventsValidatorTaskTest.java deleted file mode 100644 index 6762301687..0000000000 --- a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/SummaryEventsValidatorTaskTest.java +++ /dev/null @@ -1,148 +0,0 @@ - -package org.ekstep.ep.samza.task; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -import com.google.gson.Gson; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.fixtures.SummaryV1; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; - -import java.util.Map; - -public class SummaryEventsValidatorTaskTest { - - private static final String SUCCESS_TOPIC = "telemetry.denorm.valid"; - private static final String FAILED_TOPIC = "telemetry.failed"; - private static final String MALFORMED_TOPIC = "telemetry.malformed"; - private static final String TELEMETRY_SCHEMA_PATH = "schemas/telemetry"; - private static final String SUMMARY_SCHEMA_PATH = "schemas/summary"; - private static final String DEFAULT_SCHEMA_NAME = "envelope.json"; - private static final String SEARCH_SCHEMA_NAME = "search.json"; - private static final String LOG_SCHEMA_NAME = "log.json"; - private MessageCollector collectorMock; - private TaskContext contextMock; - private MetricsRegistry metricsRegistry; - private Counter counter; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private Config configMock; - private DruidEventsValidatorTask druidEventsValidatorTask; - - @Before - public void setUp() throws Exception { - collectorMock = mock(MessageCollector.class); - contextMock = Mockito.mock(TaskContext.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = Mockito.mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - configMock = Mockito.mock(Config.class); - - when(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).thenReturn(SUCCESS_TOPIC); - when(configMock.get("output.failed.topic.name", FAILED_TOPIC)).thenReturn(FAILED_TOPIC); - when(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).thenReturn(MALFORMED_TOPIC); - when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter); - when(contextMock.getMetricsRegistry()).thenReturn(metricsRegistry); - stub(envelopeMock.getOffset()).toReturn("2"); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "telemetry.denorm", new Partition(1))); - when(configMock.get("telemetry.schema.path", TELEMETRY_SCHEMA_PATH)).thenReturn(TELEMETRY_SCHEMA_PATH); - when(configMock.get("summary.schema.path", SUMMARY_SCHEMA_PATH)).thenReturn(SUMMARY_SCHEMA_PATH); - when(configMock.get("event.schema.file", DEFAULT_SCHEMA_NAME)).thenReturn(DEFAULT_SCHEMA_NAME); - when(configMock.get("search.schema.file",SEARCH_SCHEMA_NAME)).thenReturn(SEARCH_SCHEMA_NAME); - when(configMock.get("log.schema.file",LOG_SCHEMA_NAME)).thenReturn(LOG_SCHEMA_NAME); - druidEventsValidatorTask = new DruidEventsValidatorTask(configMock, contextMock); - } - - @Test - public void shouldSendEventToSuccessTopicIfEventIsValid() throws Exception { - stub(envelopeMock.getMessage()).toReturn(SummaryV1.VALID_EVENT); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(true, ""))); - } - - @Test - public void shouldSendEventToFaildTopicIfEventIsNotValid() throws Exception { - stub(envelopeMock.getMessage()).toReturn(SummaryV1.INVALID_EVENT); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "dimensions"))); - } - - @Test - public void shouldSendEventToFaildTopicIfEventIsNotValid_CASE_1() throws Exception { - stub(envelopeMock.getMessage()).toReturn(SummaryV1.INVALID_EVENT_CASE_1); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "tags"))); - } - - @Test - // When other than `ME_WORKFLOW_SUMMARY` Event is passed then it should send that event to success topic - public void ShouldSendEventToSuccessTopic() throws Exception { - stub(envelopeMock.getMessage()).toReturn(SummaryV1.INVALID_SUMMARY); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(true, ""))); - } - - @Test - // uaspec object should validate - public void ShouldValidateUaspecObject() throws Exception { - stub(envelopeMock.getMessage()).toReturn(SummaryV1.UASPEC_SUMMARY); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "platform"))); - } - - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - return true; - } - }; - } - public ArgumentMatcher validateEvent(final boolean dv_processed, final String field) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - Map event; - Map flags; - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String message = (String) outgoingMessageEnvelope.getMessage(); - event = new Gson().fromJson(message, Map.class); - flags = new Gson().fromJson(event.get("flags").toString(), Map.class); - assertEquals(dv_processed, flags.get("dv_processed")); - if (!dv_processed) { - assertEquals(true, event.get("metadata").toString().contains(field)); - } - return true; - } - }; - } -} - diff --git a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryEventsValidatorTaskTest.java b/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryEventsValidatorTaskTest.java deleted file mode 100644 index c795f77a4d..0000000000 --- a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryEventsValidatorTaskTest.java +++ /dev/null @@ -1,256 +0,0 @@ -package org.ekstep.ep.samza.task; - - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -import com.google.gson.Gson; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.fixtures.TelemetryV3; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; - -import java.util.Map; - -public class TelemetryEventsValidatorTaskTest { - - private static final String SUCCESS_TOPIC = "telemetry.denorm.valid"; - private static final String FAILED_TOPIC = "telemetry.failed"; - private static final String MALFORMED_TOPIC = "telemetry.malformed"; - private static final String TELEMETRY_SCHEMA_PATH = "schemas/telemetry"; - private static final String SUMMARY_SCHEMA_PATH = "schemas/summary"; - private static final String DEFAULT_SCHEMA_NAME = "envelope.json"; - private static final String SEARCH_SCHEMA_NAME = "search.json"; - private static final String LOG_SCHEMA_NAME = "log.json"; - private MessageCollector collectorMock; - private TaskContext contextMock; - private MetricsRegistry metricsRegistry; - private Counter counter; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private Config configMock; - private DruidEventsValidatorTask druidEventsValidatorTask; - - @Before - public void setUp() throws Exception { - collectorMock = mock(MessageCollector.class); - contextMock = Mockito.mock(TaskContext.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = Mockito.mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - configMock = Mockito.mock(Config.class); - - when(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).thenReturn(SUCCESS_TOPIC); - when(configMock.get("output.failed.topic.name", FAILED_TOPIC)).thenReturn(FAILED_TOPIC); - when(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).thenReturn(MALFORMED_TOPIC); - when(configMock.get("telemetry.schema.path", TELEMETRY_SCHEMA_PATH)).thenReturn(TELEMETRY_SCHEMA_PATH); - when(configMock.get("summary.schema.path", SUMMARY_SCHEMA_PATH)).thenReturn(SUMMARY_SCHEMA_PATH); - when(configMock.get("event.schema.file", DEFAULT_SCHEMA_NAME)).thenReturn(DEFAULT_SCHEMA_NAME); - when(configMock.get("search.schema.file",SEARCH_SCHEMA_NAME)).thenReturn(SEARCH_SCHEMA_NAME); - when(configMock.get("log.schema.file",LOG_SCHEMA_NAME)).thenReturn(LOG_SCHEMA_NAME); - when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter); - when(contextMock.getMetricsRegistry()).thenReturn(metricsRegistry); - stub(envelopeMock.getOffset()).toReturn("2"); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "telemetry.denorm", new Partition(1))); - druidEventsValidatorTask = new DruidEventsValidatorTask(configMock, contextMock); - } - - @Test - public void shouldSendEventToSuccessTopicIfEventIsValid() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.VALID_EVENT); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(true, ""))); - } - - @Test - public void shouldSendEventToFaildTopicIfEventIsNotValid() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_EVENT); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "contentdata/pkgversion"))); - } - - @Test - public void shouldSendEventToFaildTopicIfInvalidDeviceData_CASE1() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DEVICEDATA_CASE_1); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "devicedata/firstaccess"))); - } - - @Test - public void shouldSendEventToFaildTopicIfInvalidDeviceData_CASE2() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DEVICEDATA_CASE_2); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "devicedata/uaspec"))); - } - - @Test - public void shouldSendEventToFaildTopicIfInvalidDeviceData_CASE3() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DEVICEDATA_CASE_3); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "devicedata/country"))); - } - - @Test - public void shouldSendEventToSuccessTopicIfInvalidDeviceData_CASE4() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DEVICEDATA_CASE_4); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(true, ""))); - } - - @Test - public void shouldSendEventToFaildTopicIfInvalidContentData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_CONTENTDATA); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "contentdata/language"))); - } - - @Test - public void shouldSendEventToFaildTopicIfInvalidDialCodeData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DIALCODEDATA); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "dialcodedata/objecttype"))); - } - - @Test - public void shouldSendEventToFaildTopicIfInvalidUserData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_USERDATA); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "userdata/gradelist"))); - } - - @Test - // Case sensitive dialcode keyword validation - public void shouldSendEventToFaildTopicIfInvalidDialCodeKeywordAppears() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DIALCODE_KEY); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - } - - @Test - // Should support for the both array and object type format for dialcodedata . - public void shouldSendToSuccessTopic() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.VALID_DIALCODETYPE); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(true, ""))); - } - - @Test - // When array of object properties is having invalid type ex: generatedon. - public void shouldSendToFailedTopic() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DIALCODETYPE_CASE_1); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "generatedon"))); - } - - @Test - // When dialcodedata type is object and having invalid property type ex: channel - public void shouldSendEventToFailedTopic() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DIALCODETYPE_CASE_2); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "channel"))); - } - - @Test - public void shouldSendLogEventToSuccessTopic() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.VALID_LOG_EVENT); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(true, ""))); - } - - @Test - public void shouldSendEventToSuccessTopicForValidUserDeclaredAndDerivedLocationData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.VALID_USER_DECLARED_AND_DERIVED_LOCATION_EVENT); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(true, ""))); - } - - @Test - public void shouldSendEventToFaildTopicForInvalidDerivedLocationData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.IVALID_DERIVED_LOCATION_EVENT); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "derivedlocationdata/state"))); - } - - @Test - public void shouldSendEventToFaildTopicForInvalidUserDeclaredLocationData() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_USER_DECLARED_LOCATION_EVENT); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - verify(collectorMock).send(argThat(validateEvent(false, "devicedata/userdeclared/district"))); - } - - /** - * When Json exception occurs then it should send the event to malformed topic - * @throws Exception - */ - @Test - public void shouldAddInvalidEventToMalformedTopic() throws Exception { - stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_JSON); - druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - return true; - } - }; - } - - public ArgumentMatcher validateEvent(final boolean dv_processed, final String field) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - Map event; - Map flags; - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String message = (String) outgoingMessageEnvelope.getMessage(); - event = new Gson().fromJson(message, Map.class); - flags = new Gson().fromJson(event.get("flags").toString(), Map.class); - assertEquals(dv_processed, flags.get("dv_processed")); - if (!dv_processed) { - assertEquals(true, event.get("metadata").toString().contains(field)); - } - return true; - } - }; - } -} diff --git a/data-pipeline/ep-core/pom.xml b/data-pipeline/ep-core/pom.xml deleted file mode 100644 index 6cacadafc6..0000000000 --- a/data-pipeline/ep-core/pom.xml +++ /dev/null @@ -1,164 +0,0 @@ - - - - jobs - org.ekstep.ecosystem - 0.0.1 - - 4.0.0 - - ep-core - 0.0.1 - jar - ep-core - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-core_${scala.version} - ${samza.version} - - - org.apache.samza - samza-yarn_${scala.version} - ${samza.version} - - - org.apache.samza - samza-kafka_${scala.version} - ${samza.version} - - - org.apache.samza - samza-kv_${scala.version} - ${samza.version} - - - org.apache.samza - samza-kv-rocksdb_${scala.version} - ${samza.version} - - - com.google.code.gson - gson - 2.4 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - redis.clients - jedis - 2.9.0 - - - com.datastax.cassandra - cassandra-driver-core - 3.1.0 - - - postgresql - postgresql - 9.1-901.jdbc4 - - - junit - junit - 4.12 - test - - - com.opentable.components - otj-pg-embedded - 0.13.3 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.fiftyonred - mock-jedis - 0.4.0 - jar - test - - - it.ozimov - embedded-redis - 0.7.1 - test - - - com.squareup.okhttp3 - mockwebserver - 4.4.0 - test - - - com.konghq - unirest-java - 3.4.01 - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - org/ekstep/ep/samza/util/**/CassandraConnect.class - org/ekstep/ep/samza/core/**/Logger.class - - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - - \ No newline at end of file diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseCacheUpdaterService.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseCacheUpdaterService.java deleted file mode 100644 index 241bf2eb29..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseCacheUpdaterService.java +++ /dev/null @@ -1,87 +0,0 @@ -package org.ekstep.ep.samza.core; - -import com.datastax.driver.core.Row; -import com.datastax.driver.core.exceptions.DriverException; -import com.datastax.driver.core.querybuilder.Clause; -import com.datastax.driver.core.querybuilder.Insert; -import com.datastax.driver.core.querybuilder.QueryBuilder; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.exceptions.JedisException; -import org.ekstep.ep.samza.util.RedisConnect; -import org.ekstep.ep.samza.util.CassandraConnect; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class BaseCacheUpdaterService { - - private RedisConnect redisConnect; - private CassandraConnect cassandraConnect; - private Jedis connection; - - public BaseCacheUpdaterService(RedisConnect redisConnect) { - this.redisConnect = redisConnect; - connection = redisConnect.getConnection(); - } - - public BaseCacheUpdaterService(CassandraConnect cassandraConnect){ - this.cassandraConnect = cassandraConnect; - } - - public BaseCacheUpdaterService(RedisConnect redisConnect, CassandraConnect cassandraConnect) { - this.redisConnect = redisConnect; - connection = redisConnect.getConnection(); - this.cassandraConnect = cassandraConnect; - } - - public void addToCache(String key, String value, int storeId) { - try { - connection.select(storeId); - if (key != null && !key.isEmpty() && null != value && !value.isEmpty()) { - connection.set(key, value); - } - } catch(JedisException ex) { - connection = redisConnect.getConnection(storeId); - if (null != value) - connection.set(key, value); - } - } - - public String readFromCache(String key, int storeId) { - try { - connection.select(storeId); - return connection.get(key); - } - catch (JedisException ex) { - connection = redisConnect.getConnection(storeId); - return connection.get(key); - } - } - - public void updateToCassandra(String keyspace, String table, Map data) { - Insert query = QueryBuilder.insertInto(keyspace, table) - .values(new ArrayList<>(data.keySet()), new ArrayList<>(data.values())); - try { - cassandraConnect.upsert(query); - } catch(DriverException ex) { - cassandraConnect.reconnectCluster(); - cassandraConnect.upsert(query); - } - } - - public List readFromCassandra(String keyspace, String table, Clause clause) { - List rowSet = null; - String query = QueryBuilder.select().all() - .from(keyspace, table) - .where(clause) - .toString(); - try { - rowSet = cassandraConnect.find(query); - } catch(DriverException ex) { - cassandraConnect.reconnectCluster(); - rowSet = cassandraConnect.find(query); - } - return rowSet; - } -} diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseSink.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseSink.java deleted file mode 100644 index 24da8c58f0..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseSink.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.ekstep.ep.samza.core; - -import java.util.Map; - -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.task.MessageCollector; - -public class BaseSink { - - private MessageCollector collector; - protected JobMetrics metrics; - - public BaseSink(MessageCollector collector) { - this.collector = collector; - } - - public BaseSink(MessageCollector collector, JobMetrics metrics) { - this.collector = collector; - this.metrics = metrics; - } - - public void toTopic(String topic, String mid, String message) { - collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", topic), mid, message)); - } - - public void toTopic(String topic, String mid, Map message) { - collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", topic), mid, message)); - } - -} diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/JobMetrics.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/JobMetrics.java deleted file mode 100644 index d71103af54..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/JobMetrics.java +++ /dev/null @@ -1,403 +0,0 @@ -package org.ekstep.ep.samza.core; - -import com.google.gson.Gson; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.Metric; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.metrics.MetricsRegistryMap; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.TaskContext; -import org.joda.time.DateTime; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - - -public class JobMetrics { - private static Logger LOGGER = new Logger(JobMetrics.class); - private final String jobName; - private final Counter successMessageCount; - private final Counter failedMessageCount; - private final Counter skippedMessageCount; - private final Counter errorMessageCount; - private final Counter batchSuccessCount; - private final Counter batchErrorCount; - private final Counter primaryRouteSuccessCount; - private final Counter secondaryRouteSuccessCount; - private final Counter logRouteSuccessCount; - private final Counter errorRouteSuccessCount; - private final Counter auditRouteSuccessCount; - private final Counter assessRouteSuccessCount; - private final Counter shareEventRouteSuccessCount; - private final Counter cacheHitCount; - private final Counter cacheMissCount; - private final Counter cacheErrorCount; - private final Counter cacheEmptyValuesCount; - private final Counter processedMessageCount; - private final Counter unprocessedMessageCount; - private final Counter dbHitCount; - private final Counter userCacheHitCount; - private final Counter expiredEventCount; - private final Counter duplicateEventCount; - private final Counter deviceDBUpdateCount; - private final Counter deviceCacheUpdateCount; - private final Counter userDeclaredHitCount; - private final Counter ipLocationHitCount; - - private final Counter dbInsertCount; - private final Counter dbUpdateCount; - private final Counter dialCodesCount; - private final Counter dialCodesFromApiCount; - private final Counter dialCodesFromCacheCount; - - private TaskContext context; - private int partition; - - private Map metricCounterMap = new HashMap<>(); - - public JobMetrics(TaskContext context) { - this(context, null); - } - - public JobMetrics(TaskContext context, String jName) { - MetricsRegistry metricsRegistry = context.getMetricsRegistry(); - successMessageCount = metricsRegistry.newCounter(getClass().getName(), "success-message-count"); - failedMessageCount = metricsRegistry.newCounter(getClass().getName(), "failed-message-count"); - skippedMessageCount = metricsRegistry.newCounter(getClass().getName(), "skipped-message-count"); - errorMessageCount = metricsRegistry.newCounter(getClass().getName(), "error-message-count"); - batchSuccessCount = metricsRegistry.newCounter(getClass().getName(), "batch-success-count"); - batchErrorCount = metricsRegistry.newCounter(getClass().getName(), "batch-error-count"); - primaryRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "primary-route-success-count"); - secondaryRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "secondary-route-success-count"); - logRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "log-route-success-count"); - errorRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "error-route-success-count");; - auditRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "audit-route-success-count"); - assessRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "assess-route-success-count"); - shareEventRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "share-route-success-count"); - cacheHitCount = metricsRegistry.newCounter(getClass().getName(), "cache-hit-count"); - cacheMissCount = metricsRegistry.newCounter(getClass().getName(), "cache-miss-count"); - cacheEmptyValuesCount = metricsRegistry.newCounter(getClass().getName(), "cache-empty-values-count"); - cacheErrorCount = metricsRegistry.newCounter(getClass().getName(), "cache-error-count"); - processedMessageCount = metricsRegistry.newCounter(getClass().getName(), "processed-message-count"); - unprocessedMessageCount = metricsRegistry.newCounter(getClass().getName(), "unprocessed-message-count"); - dbHitCount = metricsRegistry.newCounter(getClass().getName(), "db-hit-count"); - userCacheHitCount = metricsRegistry.newCounter(getClass().getName(), "user-cache-hit-count"); - expiredEventCount = metricsRegistry.newCounter(getClass().getName(), "expired-event-count"); - duplicateEventCount = metricsRegistry.newCounter(getClass().getName(), "duplicate-event-count"); - deviceDBUpdateCount = metricsRegistry.newCounter(getClass().getName(), "device-db-update-count"); - deviceCacheUpdateCount = metricsRegistry.newCounter(getClass().getName(), "device-cache-update-count"); - userDeclaredHitCount = metricsRegistry.newCounter(getClass().getName(), "user-declared-hit-count"); - ipLocationHitCount = metricsRegistry.newCounter(getClass().getName(), "ip-location-hit-count"); - dbInsertCount = metricsRegistry.newCounter(getClass().getName(), "db-insert-count"); - dbUpdateCount = metricsRegistry.newCounter(getClass().getName(), "db-update-count"); - dialCodesCount = metricsRegistry.newCounter(getClass().getName(),"dial-codes-count"); - dialCodesFromApiCount = metricsRegistry.newCounter(getClass().getName(),"dial-codes-from-api-count"); - dialCodesFromCacheCount = metricsRegistry.newCounter(getClass().getName(),"dial-codes-from-cache-count"); - - metricCounterMap.put("success-message-count", successMessageCount); - metricCounterMap.put("failed-message-count", failedMessageCount); - metricCounterMap.put("skipped-message-count", skippedMessageCount); - metricCounterMap.put("error-message-count", errorMessageCount); - metricCounterMap.put("batch-success-count", batchSuccessCount); - metricCounterMap.put("batch-error-count", batchErrorCount); - metricCounterMap.put("primary-route-success-count", primaryRouteSuccessCount); - metricCounterMap.put("secondary-route-success-count", secondaryRouteSuccessCount); - metricCounterMap.put("log-route-success-count", logRouteSuccessCount); - metricCounterMap.put("error-route-success-count", errorRouteSuccessCount); - metricCounterMap.put("audit-route-success-count", auditRouteSuccessCount); - metricCounterMap.put("assess-route-success-count", assessRouteSuccessCount); - metricCounterMap.put("cache-hit-count", cacheHitCount); - metricCounterMap.put("cache-miss-count", cacheMissCount); - metricCounterMap.put("cache-empty-values-count", cacheEmptyValuesCount); - metricCounterMap.put("cache-error-count", cacheErrorCount); - metricCounterMap.put("processed-message-count", processedMessageCount); - metricCounterMap.put("unprocessed-message-count", unprocessedMessageCount); - metricCounterMap.put("user-cache-hit-count", userCacheHitCount); - metricCounterMap.put("db-hit-count", dbHitCount); - metricCounterMap.put("expired-event-count", expiredEventCount); - metricCounterMap.put("duplicate-event-count", duplicateEventCount); - metricCounterMap.put("device-db-update-count", deviceDBUpdateCount); - metricCounterMap.put("device-cache-update-count", deviceCacheUpdateCount); - metricCounterMap.put("user-declared-hit-count", userDeclaredHitCount); - metricCounterMap.put("ip-location-hit-count", ipLocationHitCount); - metricCounterMap.put("db-insert-count", dbInsertCount); - metricCounterMap.put("db-update-count", dbUpdateCount); - metricCounterMap.put("dialcodes-count", dialCodesCount); - metricCounterMap.put("dialcodes-api-hit", dialCodesFromApiCount); - metricCounterMap.put("dialcodes-cache-hit", dialCodesFromCacheCount); - jobName = jName; - this.context = context; - } - - public void clear() { - successMessageCount.clear(); - failedMessageCount.clear(); - skippedMessageCount.clear(); - errorMessageCount.clear(); - batchSuccessCount.clear(); - batchErrorCount.clear(); - cacheEmptyValuesCount.clear(); - cacheHitCount.clear(); - cacheMissCount.clear(); - cacheErrorCount.clear(); - processedMessageCount.clear(); - unprocessedMessageCount.clear(); - dbHitCount.clear(); - userCacheHitCount.clear(); - expiredEventCount.clear(); - duplicateEventCount.clear(); - deviceDBUpdateCount.clear(); - deviceCacheUpdateCount.clear(); - userDeclaredHitCount.clear(); - ipLocationHitCount.clear(); - primaryRouteSuccessCount.clear(); - secondaryRouteSuccessCount.clear(); - logRouteSuccessCount.clear(); - errorRouteSuccessCount.clear(); - auditRouteSuccessCount.clear(); - assessRouteSuccessCount.clear(); - shareEventRouteSuccessCount.clear(); - dbInsertCount.clear(); - dbUpdateCount.clear(); - dialCodesCount.clear(); - dialCodesFromApiCount.clear(); - dialCodesFromCacheCount.clear(); - } - - public void incSuccessCounter() { - successMessageCount.inc(); - } - - public void deviceDBUpdateSuccess() { - deviceDBUpdateCount.inc(); - } - - public void deviceCacheUpdateSuccess() { - deviceCacheUpdateCount.inc(); - } - - public void incFailedCounter() { - failedMessageCount.inc(); - } - - public void incSkippedCounter() { - skippedMessageCount.inc(); - } - - public void incErrorCounter() { - errorMessageCount.inc(); - } - - public void incBatchSuccessCounter() { - batchSuccessCount.inc(); - } - - public void incBatchErrorCounter() { - batchErrorCount.inc(); - } - - public void incPrimaryRouteSuccessCounter() { - primaryRouteSuccessCount.inc(); - } - - public void incSecondaryRouteSuccessCounter() { - secondaryRouteSuccessCount.inc(); - } - - public void incLogRouteSuccessCounter() { - logRouteSuccessCount.inc(); - } - - public void incErrorRouteSuccessCounter() { - errorRouteSuccessCount.inc(); - } - - public void incAuditRouteSuccessCounter() { - auditRouteSuccessCount.inc(); - } - - public void incAssessRouteSuccessCounter() { - assessRouteSuccessCount.inc(); - } - - public void incShareEventRouteSuccessCounter() { - shareEventRouteSuccessCount.inc(); - } - - public void incDuplicateCounter() { - duplicateEventCount.inc(); - } - - public void incCacheHitCounter() { - cacheHitCount.inc(); - } - - public void incCacheMissCounter() { - cacheMissCount.inc(); - } - - public void incCacheErrorCounter() { - cacheErrorCount.inc(); - } - - public void incEmptyCacheValueCounter() { - cacheEmptyValuesCount.inc(); - } - - public void incProcessedMessageCount() { - processedMessageCount.inc(); - } - - public void incUnprocessedMessageCount() { - unprocessedMessageCount.inc(); - } - - public void incDBHitCount() { - dbHitCount.inc(); - } - - public void incUserCacheHitCount() { - userCacheHitCount.inc(); - } - - public void incExpiredEventCount() { - expiredEventCount.inc(); - } - - public void incUserDeclaredHitCount() { - userDeclaredHitCount.inc(); - } - - public void incIpLocationHitCount() { - ipLocationHitCount.inc(); - } - - public void incDBInsertCount() { - dbInsertCount.inc(); - } - - public void incDBUpdateCount() { - dbUpdateCount.inc(); - } - - public void incDialCodesCount() { dialCodesCount.inc(); } - - public void incDialCodesFromApiCount() { dialCodesFromApiCount.inc(); } - - public void incDialCodesFromCacheCount() { dialCodesFromCacheCount.inc(); } - - public long consumerLag(Map> containerMetricsRegistry) { - long consumerLag = 0; - try { - for (SystemStreamPartition sysPartition : context.getSystemStreamPartitions()) { - long highWatermarkOffsetForPartition = - Long.valueOf(containerMetricsRegistry.get("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics") - .get(getSamzaMetricKey(sysPartition, "high-watermark")).toString()); - long checkpointedOffsetForPartition = Long.valueOf(containerMetricsRegistry.get("org.apache.samza.checkpoint.OffsetManagerMetrics") - .get(getSamzaMetricKey(sysPartition, "checkpointed-offset")).toString()); - consumerLag += highWatermarkOffsetForPartition - checkpointedOffsetForPartition; - this.partition = sysPartition.getPartition().getPartitionId(); - } - - } catch (Exception e) { - LOGGER.error(null, "EXCEPTION. WHEN COMPUTING CONSUMER LAG METRIC", e); - } - return consumerLag; - } - - private String getSamzaMetricKey(SystemStreamPartition partition, String samzaMetricName) { - return String.format("%s-%s-%s-%s", - partition.getSystem(), partition.getStream(), partition.getPartition().getPartitionId(), samzaMetricName); - } - - public String collect() { - Map metricsEvent = new HashMap<>(); - metricsEvent.put("job-name", jobName); - metricsEvent.put("partition", partition); - metricsEvent.put("success-message-count", successMessageCount.getCount()); - metricsEvent.put("failed-message-count", failedMessageCount.getCount()); - metricsEvent.put("error-message-count", errorMessageCount.getCount()); - metricsEvent.put("batch-success-count", batchSuccessCount.getCount()); - metricsEvent.put("batch-error-count", batchErrorCount.getCount()); - metricsEvent.put("primary-route-success-count", primaryRouteSuccessCount.getCount()); - metricsEvent.put("secondary-route-success-count", secondaryRouteSuccessCount.getCount()); - metricsEvent.put("log-route-success-count", logRouteSuccessCount.getCount()); - metricsEvent.put("error-route-success-count", errorRouteSuccessCount.getCount()); - metricsEvent.put("skipped-message-count", skippedMessageCount.getCount()); - metricsEvent.put("cache-hit-count", cacheHitCount.getCount()); - metricsEvent.put("cache-miss-count", cacheMissCount.getCount()); - metricsEvent.put("cache-error-count", cacheErrorCount.getCount()); - metricsEvent.put("cache-empty-values-count", cacheEmptyValuesCount.getCount()); - metricsEvent.put("processed-message-count", processedMessageCount.getCount()); - metricsEvent.put("unprocessed-message-count", unprocessedMessageCount.getCount()); - metricsEvent.put("db-hit-count", dbHitCount.getCount()); - metricsEvent.put("user-cache-hit-count", userCacheHitCount.getCount()); - metricsEvent.put("expired-event-count", expiredEventCount.getCount()); - metricsEvent.put("duplicate-event-count", duplicateEventCount.getCount()); - - metricsEvent.put("device-db-update-count", deviceDBUpdateCount.getCount()); - metricsEvent.put("device-cache-update-count", deviceCacheUpdateCount.getCount()); - metricsEvent.put("user-declared-hit-count", userDeclaredHitCount.getCount()); - metricsEvent.put("ip-location-hit-count", ipLocationHitCount.getCount()); - metricsEvent.put("audit-route-success-count", auditRouteSuccessCount.getCount()); - metricsEvent.put("share-route-success-count", shareEventRouteSuccessCount.getCount()); - metricsEvent.put("db-insert-count", dbInsertCount.getCount()); - metricsEvent.put("db-update-count", dbUpdateCount.getCount()); - metricsEvent.put("dialcodes-count", dialCodesCount.getCount()); - metricsEvent.put("dialcodes-api-hit", dialCodesCount.getCount()); - metricsEvent.put("dialcodes-cache-hit", dialCodesCount.getCount()); - - metricsEvent.put("consumer-lag", - consumerLag(((MetricsRegistryMap) context.getSamzaContainerContext().metricsRegistry).metrics())); - metricsEvent.put("metricts", new DateTime().getMillis()); - return new Gson().toJson(metricsEvent); - } - - public String collect(List metrics) { - Map metricsEvent = new HashMap<>(); - metricsEvent.put("job-name", jobName); - metricsEvent.put("partition", partition); - metricsEvent.put("consumer-lag", - consumerLag(((MetricsRegistryMap) context.getSamzaContainerContext().metricsRegistry).metrics())); - metricsEvent.put("metricts", new DateTime().getMillis()); - for (String metric: metrics) { - metricsEvent.put(metric, metricCounterMap.get(metric).getCount()); - } - return new Gson().toJson(metricsEvent); - } - - public String generateMetrics(List metrics) { - Map metricsEvent = new HashMap<>(); - metricsEvent.put("system", "samza"); - metricsEvent.put("subsystem", "pipeline-metrics"); - metricsEvent.put("metricts", new DateTime().getMillis()); - List> dimsList = new ArrayList<>(); - dimsList.add(createMap("job-name", jobName)); - dimsList.add(createMap("partition", partition)); - - List> metricsList = new ArrayList<>(); - for (String metric: metrics) { - Map mMap = new HashMap<>(); - mMap.put("id", metric); - mMap.put("value", metricCounterMap.get(metric).getCount()); - metricsList.add(mMap); - } - - metricsList.add(createMap("consumer-lag", - consumerLag(((MetricsRegistryMap) context.getSamzaContainerContext().metricsRegistry).metrics()))); - - metricsEvent.put("dimensions", dimsList); - metricsEvent.put("metrics", metricsList); - return new Gson().toJson(metricsEvent); - } - - private Map createMap(String id, Object value) { - Map map = new HashMap<>(); - map.put("id", id); - map.put("value", value); - return map; - } -} diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/Logger.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/Logger.java deleted file mode 100644 index 53aa3bfea5..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/Logger.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.ekstep.ep.samza.core; - -import org.slf4j.LoggerFactory; - -import static java.text.MessageFormat.format; - -public class Logger { - - private final org.slf4j.Logger logger; - - public Logger(Class clazz) { - logger = LoggerFactory.getLogger(clazz); - } - - public void trace(String eventId, String s) { - logger.trace(logPrependedWithMetadata(eventId, s)); - } - - public void trace(String eventId, String s, Object o) { - logger.trace(logPrependedWithMetadata(eventId, s), o); - } - - public void debug(String eventId, String s) { - logger.debug(logPrependedWithMetadata(eventId, s)); - } - - public void debug(String eventId, String s, Object o) { - logger.debug(logPrependedWithMetadata(eventId, s), o); - } - - public void info(String eventId, String s) { - logger.info(logPrependedWithMetadata(eventId, s)); - } - - public void info(String eventId, String s, Object o) { - logger.info(logPrependedWithMetadata(eventId, s), o); - } - - public void warn(String eventId, String s) { - logger.warn(logPrependedWithMetadata(eventId, s)); - } - - public void warn(String eventId, String s, Object o) { - logger.warn(logPrependedWithMetadata(eventId, s), o); - } - - public void error(String eventId, String s) { - logger.error(logPrependedWithMetadata(eventId, s)); - } - - public void error(String eventId, String s, Object o) { - logger.error(logPrependedWithMetadata(eventId, s), o); - } - - public void error(String eventId, String s, Throwable throwable) { - logger.error(logPrependedWithMetadata(eventId, s), throwable); - throwable.printStackTrace(); - } - - private String logPrependedWithMetadata(String eventId, String s) { - return eventId == null ? s : format("{0} {1}", eventId, s); - } -} diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/task/BaseSamzaTask.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/task/BaseSamzaTask.java deleted file mode 100644 index a72266f032..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/task/BaseSamzaTask.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.task.InitableTask; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.StreamTask; -import org.apache.samza.task.TaskCoordinator; -import org.apache.samza.task.WindowableTask; -import org.ekstep.ep.samza.core.JobMetrics; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -abstract class BaseSamzaTask implements StreamTask, InitableTask, WindowableTask { - - protected JobMetrics metrics; - private String metricsTopic; - private List metricsList; - private String prometheusMetricsTopic; - private List defaultPipelineMetrics = - new ArrayList<>(Arrays.asList("success-message-count", "failed-message-count", "error-message-count", "skipped-message-count")); - - public BaseSamzaTask() { - - } - - public void initTask(Config config, JobMetrics metrics) { - this.metrics = metrics; - this.metricsTopic = config.get("output.metrics.topic.name", "telemetry.pipeline_metrics"); - this.prometheusMetricsTopic = config.get("output.prometheus.metrics.topic.name", "telemetry.metrics"); - this.metricsList = config.getList("pipeline.metrics.list", defaultPipelineMetrics); - } - - - @Override - public void window(MessageCollector collector, TaskCoordinator coordinator) throws Exception { - - String mEvent = metrics.collect(metricsList); - String prometheusMetricEvent = metrics.generateMetrics(metricsList); - collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", metricsTopic), mEvent)); - collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", prometheusMetricsTopic), prometheusMetricEvent)); - this.metrics.clear(); - } -} diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/CassandraConnect.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/CassandraConnect.java deleted file mode 100644 index 29bb94c6c5..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/CassandraConnect.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.ekstep.ep.samza.util; - -import com.datastax.driver.core.*; - -import java.util.List; - -import com.datastax.driver.core.querybuilder.Insert; -import org.apache.samza.config.Config; - -public class CassandraConnect { - private Session session; - private Config config; - private List clusterHosts; - private int port; - - public CassandraConnect(Config config) { - this.config = config; - String host = config.get("cassandra.host", "127.0.0.1"); - Integer port = config.getInt("cassandra.port", 9042); - Cluster cluster = Cluster.builder().addContactPoint(host).withPort(port).build(); - this.session = cluster.connect(); - } - - public CassandraConnect(String host, Integer port) { - Cluster cluster = Cluster.builder().addContactPoints(host).withPort(port).build(); - this.session = cluster.connect(); - } - - public CassandraConnect(List hosts, Integer port) { - this.clusterHosts = hosts; - this.port = port; - Cluster.Builder builder = Cluster.builder(); - for (String host : hosts) { - builder.addContactPoint(host); - } - // TODO: We need to check if the consistency level can be downgraded to ONE - Cluster cluster = builder.withPort(port).withQueryOptions(new QueryOptions() - .setConsistencyLevel(ConsistencyLevel.QUORUM)).build(); - this.session = cluster.connect(); - } - - public Row findOne(String query) { - ResultSet rs = session.execute(query); - return rs.one(); - } - - public List find(String query) { - ResultSet rs = session.execute(query); - return rs.all(); - } - - public boolean upsert(String query) { - ResultSet rs = session.execute(query); - return rs.wasApplied(); - } - - public boolean upsert(Insert query) { - ResultSet rs = session.execute(query); - return rs.wasApplied(); - } - - public UserType getUDTType(String keyspace, String typeName) { - return session.getCluster().getMetadata().getKeyspace(keyspace).getUserType(typeName); - } - - public void reconnect() { - this.session.close(); - String host = config.get("cassandra.host", "127.0.0.1"); - Integer port = config.getInt("cassandra.port", 9042); - Cluster cluster = Cluster.builder().addContactPoint(host).withPort(port).build(); - this.session = cluster.connect(); - } - - public void reconnectCluster() { - this.session.close(); - Cluster.Builder builder = Cluster.builder(); - for (String host : clusterHosts) { - builder.addContactPoint(host); - } - Cluster cluster = builder.withPort(port).withQueryOptions(new QueryOptions() - .setConsistencyLevel(ConsistencyLevel.QUORUM)).build(); - this.session = cluster.connect(); - } -} diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/DeDupEngine.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/DeDupEngine.java deleted file mode 100644 index 8fb88478cf..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/DeDupEngine.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.ekstep.ep.samza.util; - - -import redis.clients.jedis.Jedis; -import redis.clients.jedis.exceptions.JedisException; - -public class DeDupEngine { - - private RedisConnect redisConnect; - private Jedis redisConnection; - private int store; - private int expirySeconds; - - public DeDupEngine(RedisConnect redisConnect, int store, int expirySeconds) { - - this.redisConnect = redisConnect; - this.redisConnection = redisConnect.getConnection(); - this.store = store; - this.redisConnection.select(store); - this.expirySeconds = expirySeconds; - } - - public boolean isUniqueEvent(String checksum) throws JedisException { - - boolean unique = false; - try { - unique = !redisConnection.exists(checksum); - } catch (JedisException ex) { - this.redisConnection.close(); - this.redisConnection = redisConnect.getConnection(store, 10000); - unique = !redisConnection.exists(checksum); - } - return unique; - } - - public void storeChecksum(String checksum) throws JedisException { - - try { - redisConnection.setex(checksum, expirySeconds, ""); - } catch (JedisException ex) { - this.redisConnection.close(); - this.redisConnection = redisConnect.getConnection(10000); - this.redisConnection.select(store); - redisConnection.setex(checksum, expirySeconds, ""); - } - } - - public Jedis getRedisConnection() { - return redisConnection; - } - -} diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/PostgresConnect.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/PostgresConnect.java deleted file mode 100644 index 36774328b7..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/PostgresConnect.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.ekstep.ep.samza.util; - -import org.apache.samza.config.Config; -import org.postgresql.ds.PGPoolingDataSource; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; - -public class PostgresConnect { - private Config config; - private Connection connection; - private Statement statement; - private String user; - private String password; - private String db; - private String host; - private Integer port; - private Integer maxConnections; - private PGPoolingDataSource source; - - public PostgresConnect(Config config) throws Exception { - this.config = config; - user = config.get("postgres.user"); - password = config.get("sensitive.postgres.password"); - db = config.get("postgres.db"); - host = config.get("postgres.host","127.0.0.1"); - port = config.getInt("postgres.port", 5432); - maxConnections = config.getInt("postgres.maxConnections", 2); - buildPoolConfig(); - connection = source.getConnection(); - statement = connection.createStatement(); - } - - public void buildPoolConfig() throws Exception { - Class.forName("org.postgresql.Driver"); - source = new PGPoolingDataSource(); - source.setServerName(host); - source.setPortNumber(port); - source.setUser(user); - source.setPassword(password); - source.setDatabaseName(db); - source.setMaxConnections(maxConnections); - } - - public Connection getConnection() { - return this.connection; - } - - public boolean execute(String query) throws Exception { - try { - return statement.execute(query); - } catch (SQLException ex) { - resetConnection(); - return statement.execute(query); - } - } - - public Connection resetConnection() throws Exception { - closeConnection(); - buildPoolConfig(); - connection = source.getConnection(); - statement = connection.createStatement(); - return connection; - } - - public void closeConnection() throws Exception { - connection.close(); - source.close(); - } -} diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RedisConnect.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RedisConnect.java deleted file mode 100644 index 259bbb14e2..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RedisConnect.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.ekstep.ep.samza.util; - -import org.apache.samza.config.Config; -import redis.clients.jedis.Jedis; - - -public class RedisConnect { - - private Config config; - - public RedisConnect(Config config) { - this.config = config; - } - - private Jedis getConnection(long backoffTimeInMillis) { - String redisHost = config.get("redis.host", "localhost"); - Integer redisPort = config.getInt("redis.port", 6379); - if(backoffTimeInMillis > 0) { - try { - Thread.sleep(backoffTimeInMillis); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - return new Jedis(redisHost, redisPort, 30000); - } - - public Jedis getConnection(int db, long backoffTimeInMillis) { - - Jedis jedis = getConnection(backoffTimeInMillis); - jedis.select(db); - return jedis; - } - - public Jedis getConnection(int db) { - - Jedis jedis = getConnection(db, 0); - jedis.select(db); - return jedis; - } - - public Jedis getConnection() { - - return getConnection(0); - } - -} diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RestUtil.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RestUtil.java deleted file mode 100644 index 89a35d8948..0000000000 --- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RestUtil.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.ekstep.ep.samza.util; - - -import kong.unirest.GetRequest; -import kong.unirest.Unirest; -import kong.unirest.UnirestException; - -import java.util.Map; - -public class RestUtil { - - /** - * @param apiURL - API Url - * @param requestHeaders - Map Request header parameters - * @return - Unirest.ResponseBody - * @throws UnirestException - */ - public String get(String apiURL, Map requestHeaders) throws UnirestException { - GetRequest request = Unirest.get(apiURL); - request.headers(requestHeaders); - return request.asString().getBody(); - } -} \ No newline at end of file diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/fixtures/MetricsFixture.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/fixtures/MetricsFixture.java deleted file mode 100644 index 39332ce2f6..0000000000 --- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/fixtures/MetricsFixture.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.Metric; - -import java.lang.reflect.Type; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class MetricsFixture { - - public static final String METRIC_EVENT_STREAM1 = "{\n" + - " \"org.apache.samza.system.kafka.KafkaSystemConsumerMetrics\": {\n" + - " \"kafka-inputtopic-0-high-watermark\": {\n" + - " \"name\": \"kafka-inputtopic-0-high-watermark\",\n" + - " \"count\": {\n" + - " \"value\": 1000\n" + - " }\n" + - " }\n" + - " }, \n" + - " \"org.apache.samza.checkpoint.OffsetManagerMetrics\": {\n" + - " \"kafka-inputtopic-0-checkpointed-offset\": {\n" + - " \"name\": \"kafka-inputtopic-0-checkpointed-offset\",\n" + - " \"count\": {\n" + - " \"value\": 200\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; - - public static final String METRIC_EVENT_STREAM2 = "{\n" + - " \"org.apache.samza.system.kafka.KafkaSystemConsumerMetrics\": {\n" + - " \"kafka-inputtopic-0-high-watermark\": {\n" + - " \"name\": \"kafka-inputtopic-0-high-watermark\",\n" + - " \"count\": {\n" + - " \"value\": 1000\n" + - " }\n" + - " }\n" + - " }, \n" + - " \"org.apache.samza.checkpoint.OffsetManagerMetrics\": {\n" + - " \"kafka-inputtopic-0-checkpointed-offset\": {\n" + - " \"name\": \"kafka-inputtopic-0-checkpointed-offset\",\n" + - " \"count\": {\n" + - " \"value\": 1000\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; - - - public static Map> getMetricMap(String message) { - Type type = new TypeToken>>() {}.getType(); - return (Map>) new Gson().fromJson(message, type); - } - - -} diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/Element.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/Element.java deleted file mode 100644 index 0237ae17a4..0000000000 --- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/Element.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.ekstep.ep.samza.schema; - -public class Element { - private String id; - private Object value; - - public Element(String id, Object value) { - this.id = id; - this.value = value; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public Object getValue() { - return value; - } - - public void setValue(Object value) { - this.value = value; - } - - @Override - public int hashCode() { - return id.hashCode(); - } - - @Override - public boolean equals(Object obj) { - Element other = (Element) obj; - return id.equalsIgnoreCase(other.id) && value.equals(other.value); - } -} diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/MetricEvent.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/MetricEvent.java deleted file mode 100644 index d25b2e0fe7..0000000000 --- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/MetricEvent.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.ekstep.ep.samza.schema; - -import org.joda.time.DateTime; - -import java.util.List; - -public class MetricEvent { - - private String system; - private String subsystem; - private Long metricts; - private List metrics; - private List dimensions; - - public String getSystem() { - return system; - } - - public void setSystem(String system) { - this.system = system; - } - - public String getSubsystem() { - return subsystem; - } - - public void setSubsystem(String subsystem) { - this.subsystem = subsystem; - } - - public Long getMetricts() { - return metricts; - } - - public void setMetricts(Long metricts) { - this.metricts = metricts; - } - - public List getMetrics() { - return metrics; - } - - public void setMetrics(List metrics) { - this.metrics = metrics; - } - - public List getDimensions() { - return dimensions; - } - - public void setDimensions(List dimensions) { - this.dimensions = dimensions; - } -} diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/BaseUpdaterTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/BaseUpdaterTest.java deleted file mode 100644 index ed2b716702..0000000000 --- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/BaseUpdaterTest.java +++ /dev/null @@ -1,126 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.datastax.driver.core.Row; -import com.datastax.driver.core.exceptions.DriverException; -import com.datastax.driver.core.querybuilder.Clause; -import com.datastax.driver.core.querybuilder.QueryBuilder; -import com.fiftyonred.mock_jedis.MockJedis; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.ekstep.ep.samza.core.BaseCacheUpdaterService; -import org.ekstep.ep.samza.util.CassandraConnect; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import redis.clients.jedis.Jedis; - -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.*; - - -public class BaseUpdaterTest { - - private RedisConnect redisConnectMock; - private CassandraConnect cassandraConnectMock; - private BaseCacheUpdaterService baseUpdater; - private Jedis jedisMock = new MockJedis("test"); - private static int storeId = 5; - Gson gson = new Gson(); - Type type = new TypeToken>() { - }.getType(); - - @Before - public void setUp() { - redisConnectMock = mock(RedisConnect.class); - cassandraConnectMock = mock(CassandraConnect.class); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - new BaseCacheUpdaterService(redisConnectMock); - baseUpdater = new BaseCacheUpdaterService(redisConnectMock, cassandraConnectMock); - - stub(redisConnectMock.getConnection(anyInt())).toReturn(jedisMock); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - } - - @Test - public void shouldAddToCache() { - baseUpdater.addToCache("4569876545678", "{\"role\":\"student\",\"type\":\"User\"}", storeId); - String value = jedisMock.get("4569876545678"); - Map parsedData = gson.fromJson(value, type); - - assertEquals("User", parsedData.get("type")); - assertEquals("student", parsedData.get("role")); - - } - - @Test - public void shouldReadFromCache() { - jedisMock.select(storeId); - jedisMock.set("4569876545678", "{\"role\":\"teacher\",\"type\":\"Request\"}"); - String value = baseUpdater.readFromCache("4569876545678", storeId); - Map parsedData = gson.fromJson(value, type); - - assertEquals("Request", parsedData.get("type")); - assertEquals("teacher", parsedData.get("role")); - } - - @Test(expected = DriverException.class) - public void shouldHandleCassandraException() throws Exception { - Clause userDataClause = QueryBuilder.eq("id", "id"); - - when(cassandraConnectMock.find(anyString())).thenThrow(new DriverException("Cassandra Exception")); - baseUpdater.readFromCassandra("sunbird", "user", userDataClause); - } - - @Test(expected = DriverException.class) - public void shouldHandleCassandraExceptionForLocationQuery() throws Exception { - List locationIds = new ArrayList<>(); - locationIds.add("1f56a8458d78df90"); - Clause locationDataClause = QueryBuilder.in("id", locationIds); - - when(cassandraConnectMock.find(anyString())).thenThrow(new DriverException("Cassandra Exception")); - baseUpdater.readFromCassandra("sunbird", "location", locationDataClause); - } - - @Test - public void shouldHandleNullOrEmptyClause() { - List emptyClauseValue = baseUpdater.readFromCassandra("sunbird", "user", QueryBuilder.eq("", "")); - assertEquals(true, emptyClauseValue.isEmpty()); - } - - @Test - public void shouldGetLocationDetailsFromDB() { - List locationIds = new ArrayList<>(); - locationIds.add("1f56a8458d78df90"); - ArrayList location = new ArrayList(3); - location.add(0, "state-name"); - location.add(1, "district-name"); - Clause locationDataClause = QueryBuilder.in("id", locationIds); - stub(baseUpdater.readFromCassandra("sunbird", "location", locationDataClause)).toReturn(location); - - List row = baseUpdater.readFromCassandra("sunbird", "location", locationDataClause); - assertEquals("state-name", row.get(0)); - assertEquals("district-name", row.get(1)); - } - - @Test - public void shouldUpdateToCassandra() { - Map map = new HashMap<>(); - map.put("device_id", "5975394fjh9543978987593"); - ArrayList location = new ArrayList(3); - location.add(0, "device_id"); - baseUpdater.updateToCassandra("sunbird", "location", map); - Clause locationDataClause = QueryBuilder.in("id", map); - stub(baseUpdater.readFromCassandra("sunbird", "location", locationDataClause)).toReturn(location); - List row = baseUpdater.readFromCassandra("sunbird", "location", locationDataClause); - Assert.assertNotNull(row.get(0)); - } -} diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/JobMetricsTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/JobMetricsTest.java deleted file mode 100644 index 7557ccd0ad..0000000000 --- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/JobMetricsTest.java +++ /dev/null @@ -1,262 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.gson.Gson; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.container.SamzaContainerContext; -import org.apache.samza.container.TaskName; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.Metric; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.metrics.MetricsRegistryMap; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.TaskContext; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.fixtures.MetricsFixture; -import org.ekstep.ep.samza.schema.Element; -import org.ekstep.ep.samza.schema.MetricEvent; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; - -import static org.junit.Assert.assertEquals; -import static junit.framework.TestCase.assertTrue; -import static org.mockito.Mockito.*; - -public class JobMetricsTest { - - private TaskContext contextMock; - private JobMetrics jobMetricsMock; - - @Before - public void setUp() { - contextMock = mock(TaskContext.class); - MetricsRegistry metricsRegistry = mock(MetricsRegistry.class); - Counter counter = mock(Counter.class); - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - } - - @Test - public void shouldReturnConsumerLag() { - - jobMetricsMock = new JobMetrics(contextMock, "test-job"); - jobMetricsMock.clear(); - Set systemStreamPartitions = new HashSet<>(); - SystemStreamPartition systemStreamPartition = - new SystemStreamPartition("kafka", "inputtopic", new Partition(0)); - systemStreamPartitions.add(systemStreamPartition); - - Map> concurrentHashMap = - MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM1); - when(contextMock.getSystemStreamPartitions()).thenReturn(systemStreamPartitions); - long consumer_lag = jobMetricsMock.consumerLag(concurrentHashMap); - Assert.assertEquals(800, consumer_lag); - - } - - @Test - public void shouldReturnZeroConsumerLagWhenAllMessagesAreProcessed() { - - jobMetricsMock = new JobMetrics(contextMock, null); - - Set systemStreamPartitions = new HashSet<>(); - SystemStreamPartition systemStreamPartition = - new SystemStreamPartition("kafka", "inputtopic", new Partition(0)); - systemStreamPartitions.add(systemStreamPartition); - - Map> concurrentHashMap = - MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM2); - when(contextMock.getSystemStreamPartitions()).thenReturn(systemStreamPartitions); - long consumer_lag = jobMetricsMock.consumerLag(concurrentHashMap); - Assert.assertEquals(0, consumer_lag); - } - - /* - @Test - public void shouldCollectTheMetrics() { - Map> concurrentHashMap = - MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM1); - SystemStreamPartition systemStreamPartition = new SystemStreamPartition("kafka", "inputtopic", new Partition(0)); - Set partitionsSet = new HashSet<>(); - partitionsSet.add(systemStreamPartition); - MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap("id"); - metricsRegistryMap.metrics().put("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics", concurrentHashMap.get("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics")); - metricsRegistryMap.metrics().put("org.apache.samza.checkpoint.OffsetManagerMetrics", concurrentHashMap.get("org.apache.samza.checkpoint.OffsetManagerMetrics")); - - TaskName taskName = new TaskName("task"); - ArrayList list = new ArrayList(); - list.add(taskName); - SamzaContainerContext samzaContainerContext = new SamzaContainerContext("id", mock(Config.class), list, metricsRegistryMap); - when(contextMock.getSamzaContainerContext()).thenReturn(samzaContainerContext); - when(contextMock.getSystemStreamPartitions()).thenReturn(partitionsSet); - when(contextMock.getMetricsRegistry()).thenReturn(samzaContainerContext.metricsRegistry); - - jobMetricsMock = new JobMetrics(contextMock, "test-job"); - jobMetricsMock.incSuccessCounter(); - jobMetricsMock.deviceDBUpdateSuccess(); - jobMetricsMock.deviceCacheUpdateSuccess(); - jobMetricsMock.incFailedCounter(); - jobMetricsMock.incSkippedCounter(); - jobMetricsMock.incErrorCounter(); - jobMetricsMock.incBatchSuccessCounter(); - jobMetricsMock.incBatchErrorCounter(); - jobMetricsMock.incPrimaryRouteSuccessCounter(); - jobMetricsMock.incSecondaryRouteSuccessCounter(); - jobMetricsMock.incDuplicateCounter(); - jobMetricsMock.incCacheHitCounter(); - jobMetricsMock.incCacheMissCounter(); - jobMetricsMock.incCacheErrorCounter(); - jobMetricsMock.incEmptyCacheValueCounter(); - jobMetricsMock.incProcessedMessageCount(); - jobMetricsMock.incUnprocessedMessageCount(); - jobMetricsMock.incDBHitCount(); - jobMetricsMock.incUserCacheHitCount(); - jobMetricsMock.incExpiredEventCount(); - jobMetricsMock.incUserDeclaredHitCount(); - jobMetricsMock.incIpLocationHitCount(); - jobMetricsMock.incDialCodesCount(); - jobMetricsMock.incDialCodesFromApiCount(); - jobMetricsMock.incDialCodesFromCacheCount(); - - String metricsJson = jobMetricsMock.collect(); - System.out.println(metricsJson); - Gson gson = new Gson(); - @SuppressWarnings("unchecked") - Map metrics = (Map) gson.fromJson(metricsJson, Map.class); - assertEquals("test-job", metrics.get("job-name")); - assertEquals(1.0, metrics.get("cache-hit-count")); - assertEquals(1.0, metrics.get("db-hit-count")); - assertEquals(1.0, metrics.get("batch-error-count")); - assertEquals(1.0, metrics.get("success-message-count")); - assertEquals(1.0, metrics.get("cache-empty-values-count")); - assertEquals(1.0, metrics.get("failed-message-count")); - assertEquals(1.0, metrics.get("unprocessed-message-count")); - assertEquals(1.0, metrics.get("expired-event-count")); - assertEquals(1.0, metrics.get("duplicate-event-count")); - assertEquals(1.0, metrics.get("processed-message-count")); - assertEquals(1.0, metrics.get("primary-route-success-count")); - assertEquals(1.0, metrics.get("batch-success-count")); - assertEquals(1.0, metrics.get("secondary-route-success-count")); - assertEquals(1.0, metrics.get("device-cache-update-count")); - assertEquals(1.0, metrics.get("cache-error-count")); - assertEquals(1.0, metrics.get("error-message-count")); - assertEquals(1.0, metrics.get("dialcodes-count")); - assertEquals(1.0, metrics.get("dialcodes-api-hit")); - assertEquals(1.0, metrics.get("dialcodes-cache-hit")); - assertEquals(0.0, metrics.get("partition")); - assertEquals(800.0, metrics.get("consumer-lag")); - } - */ - - - @Test - public void validateCollectedMetrics() { - Map> metricsMap = - MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM1); - MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap("id"); - SystemStreamPartition systemStreamPartition = new SystemStreamPartition("kafka", "inputtopic", new Partition(0)); - Set partitionsSet = new HashSet<>(); - partitionsSet.add(systemStreamPartition); - metricsRegistryMap.metrics().put("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics", - metricsMap.get("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics")); - metricsRegistryMap.metrics().put("org.apache.samza.checkpoint.OffsetManagerMetrics", - metricsMap.get("org.apache.samza.checkpoint.OffsetManagerMetrics")); - TaskName taskName = new TaskName("task"); - ArrayList taskList = new ArrayList<>(); - taskList.add(taskName); - SamzaContainerContext samzaContainerContext = new SamzaContainerContext("id", mock(Config.class), taskList, metricsRegistryMap); - when(contextMock.getSamzaContainerContext()).thenReturn(samzaContainerContext); - when(contextMock.getSystemStreamPartitions()).thenReturn(partitionsSet); - when(contextMock.getMetricsRegistry()).thenReturn(samzaContainerContext.metricsRegistry); - - jobMetricsMock = new JobMetrics(contextMock, "test-job"); - List jobMetricList = new ArrayList<>(Arrays.asList("success-message-count","skipped-message-count","error-message-count", - "batch-success-count","batch-error-count","duplicate-event-count")); - jobMetricsMock.incSuccessCounter(); - jobMetricsMock.incSkippedCounter(); - jobMetricsMock.incErrorCounter(); - jobMetricsMock.incBatchSuccessCounter(); - jobMetricsMock.incBatchErrorCounter(); - jobMetricsMock.incDuplicateCounter(); - String metricsOutput = jobMetricsMock.collect(jobMetricList); - System.out.println(metricsOutput); - Gson gson = new Gson(); - @SuppressWarnings("unchecked") - Map metrics = (Map) gson.fromJson(metricsOutput, Map.class); - assertEquals("test-job", metrics.get("job-name")); - assertEquals(1.0, metrics.get("success-message-count")); - assertEquals(1.0, metrics.get("skipped-message-count")); - assertEquals(1.0, metrics.get("error-message-count")); - assertEquals(1.0, metrics.get("batch-success-count")); - assertEquals(1.0, metrics.get("batch-error-count")); - assertEquals(1.0, metrics.get("duplicate-event-count")); - assertEquals(0.0, metrics.get("partition")); - assertEquals(800.0, metrics.get("consumer-lag")); - assertTrue(metrics.containsKey("metricts") && metrics.get("metricts") != null); - } - - @Test - public void generatePrometheusMetricStructure() { - Map> concurrentHashMap = - MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM1); - - MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap("metric-registry"); - metricsRegistryMap.metrics().put("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics", - concurrentHashMap.get("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics")); - metricsRegistryMap.metrics().put("org.apache.samza.checkpoint.OffsetManagerMetrics", - concurrentHashMap.get("org.apache.samza.checkpoint.OffsetManagerMetrics")); - - SystemStreamPartition systemStreamPartition = new SystemStreamPartition("kafka", "inputtopic", new Partition(0)); - Set partitionsSet = new HashSet<>(); - partitionsSet.add(systemStreamPartition); - - TaskName taskName = new TaskName("generate-prometheus-metrics-task"); - ArrayList taskList = new ArrayList<>(Collections.singletonList(taskName)); - SamzaContainerContext samzaContainerContext = new SamzaContainerContext("id", - mock(Config.class), taskList, metricsRegistryMap); - when(contextMock.getSamzaContainerContext()).thenReturn(samzaContainerContext); - when(contextMock.getSystemStreamPartitions()).thenReturn(partitionsSet); - when(contextMock.getMetricsRegistry()).thenReturn(samzaContainerContext.metricsRegistry); - - jobMetricsMock = new JobMetrics(contextMock, "test-job"); - - List jobMetricList = new ArrayList<>(Arrays.asList("success-message-count","skipped-message-count","error-message-count", - "batch-success-count","batch-error-count","duplicate-event-count")); - jobMetricsMock.incSuccessCounter(); - jobMetricsMock.incSkippedCounter(); - jobMetricsMock.incErrorCounter(); - jobMetricsMock.incBatchSuccessCounter(); - jobMetricsMock.incBatchErrorCounter(); - jobMetricsMock.incDuplicateCounter(); - String metricsOutput = jobMetricsMock.generateMetrics(jobMetricList); - - SimpleDateFormat dt = new SimpleDateFormat("yyyy-MM-dd"); - Gson gson = new Gson(); - @SuppressWarnings("unchecked") - MetricEvent metrics = gson.fromJson(metricsOutput, MetricEvent.class); - assertEquals("samza", metrics.getSystem()); - assertEquals("pipeline-metrics", metrics.getSubsystem()); - assertEquals(dt.format(new Date()), dt.format(new Date(metrics.getMetricts()))); - assertEquals(7, metrics.getMetrics().size()); - assertEquals(2, metrics.getDimensions().size()); - - - List metricList = metrics.getMetrics(); - assertTrue(metricList.contains(new Element("success-message-count", 1.0))); - assertTrue(metricList.contains(new Element("skipped-message-count", 1.0))); - assertTrue(metricList.contains(new Element("error-message-count", 1.0))); - assertTrue(metricList.contains(new Element("batch-success-count", 1.0))); - assertTrue(metricList.contains(new Element("batch-error-count", 1.0))); - assertTrue(metricList.contains(new Element("duplicate-event-count", 1.0))); - assertTrue(metricList.contains(new Element("consumer-lag", 800.0))); - - List dimsList = metrics.getDimensions(); - assertTrue(dimsList.contains(new Element("job-name", "test-job"))); - assertTrue(dimsList.contains(new Element("partition", 0.0))); - } -} diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/PostgresConnectTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/PostgresConnectTest.java deleted file mode 100644 index e627c8fa8f..0000000000 --- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/PostgresConnectTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.opentable.db.postgres.embedded.EmbeddedPostgres; -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.util.PostgresConnect; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; - -import static org.junit.Assert.*; - -import static org.mockito.Mockito.*; - -public class PostgresConnectTest { - - private Config configMock; - private PostgresConnect postgresConnect; - private Connection connection; - private Connection resetConnection; - - @Before - public void setUp() throws Exception { - configMock = mock(Config.class); - - EmbeddedPostgres.builder().setPort(5430).start(); - - stub(configMock.get("postgres.user")).toReturn("postgres"); - stub(configMock.get("postgres.password")).toReturn("postgres"); - stub(configMock.get("postgres.db")).toReturn("postgres"); - stub(configMock.get("postgres.host","127.0.0.1")).toReturn("localhost"); - stub(configMock.getInt("postgres.port", 5432)).toReturn(5430); - - postgresConnect = new PostgresConnect(configMock); - } - - @Test - public void shouldVerifyPostgresConnect() throws Exception { - connection = postgresConnect.getConnection(); - assertNotNull(connection); - - postgresConnect.execute("CREATE TABLE device_table(id text PRIMARY KEY, channel text);"); - postgresConnect.execute("INSERT INTO device_table(id,channel) VALUES('12345','custchannel');"); - - Statement st = connection.createStatement(); - ResultSet rs = st.executeQuery("SELECT * FROM device_table where id='12345';"); - while(rs.next()) { - assertEquals("12345", rs.getString("id")); - assertEquals("custchannel", rs.getString("channel")); - } - - resetConnection = postgresConnect.resetConnection(); - assertNotNull(resetConnection); - - postgresConnect.closeConnection(); - } - -} diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/RestUtilTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/RestUtilTest.java deleted file mode 100644 index 2604049037..0000000000 --- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/RestUtilTest.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.ekstep.ep.samza.task; - -import kong.unirest.UnirestException; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import org.ekstep.ep.samza.util.RestUtil; -import org.junit.Test; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertNotNull; - -public class RestUtilTest { - @Test - public void shouldInvokeHTTPCall() throws IOException { - MockWebServer server = new MockWebServer(); - server.enqueue(new MockResponse().setBody("hello, world!")); - server.enqueue(new MockResponse().setHeader("Authorization", "")); - try { - server.start(3000); - } catch (IOException e) { - System.out.println("Exception" + e); - } - server.url("http://127.0.0.1:3000/api/dialcode/v3/read/"); - Map headers = new HashMap<>(); - headers.put("Authorization", ""); - try { - String response = new RestUtil().get("http://127.0.0.1:3000/api/dialcode/v3/read/", headers); - // TODO: Assertion are failing need to add - System.out.println("response is" + response); - assertNotNull(response); - } catch (UnirestException e) { - System.out.println("Exception is" + e); - } finally { - server.shutdown(); - } - - - } -} diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/DeDupEngineTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/DeDupEngineTest.java deleted file mode 100644 index 77e8a28820..0000000000 --- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/DeDupEngineTest.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.ekstep.ep.samza.task.util; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.when; - -import java.io.IOException; - -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import redis.clients.jedis.Jedis; -import redis.embedded.RedisServer; - -public class DeDupEngineTest { - - private RedisServer redisServer; - private Config configMock; - private RedisConnect redisConnect; - - @Before - public void setUp() throws IOException { - redisServer = new RedisServer(6379); - redisServer.start(); - configMock = Mockito.mock(Config.class); - when(configMock.get("redis.host", "localhost")).thenReturn("localhost"); - when(configMock.getInt("redis.port", 6379)).thenReturn(6379); - when(configMock.getInt("redis.connection.max", 2)).thenReturn(2); - when(configMock.getInt("redis.connection.idle.max", 2)).thenReturn(2); - when(configMock.getInt("redis.connection.idle.min", 1)).thenReturn(1); - when(configMock.getInt("redis.connection.minEvictableIdleTimeSeconds", 120)).thenReturn(120); - when(configMock.getInt("redis.connection.timeBetweenEvictionRunsSeconds", 300)).thenReturn(300); - redisConnect = new RedisConnect(configMock); - } - - @After - public void tearDown() { - redisServer.stop(); - } - - @Test - public void shouldVerifyDedupEngine() { - - DeDupEngine engine = new DeDupEngine(redisConnect, 1, 100); - - engine.storeChecksum("1234"); - - assertTrue(engine.isUniqueEvent("2345")); - assertTrue(!engine.isUniqueEvent("1234")); - - Jedis jedis = engine.getRedisConnection(); - assertNotNull(jedis); - assertEquals(Long.valueOf(1), jedis.getDB()); - - redisServer.stop(); - try { - engine.storeChecksum("1234"); - } catch(Exception ex) { - assertNotNull(ex); - } - - try { - assertTrue(engine.isUniqueEvent("2345")); - } catch(Exception ex) { - assertNotNull(ex); - } finally { - redisServer.start(); - engine.storeChecksum("1234"); - assertTrue(engine.isUniqueEvent("2345")); - assertTrue(!engine.isUniqueEvent("1234")); - } - - } - -} diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/RedisConnectTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/RedisConnectTest.java deleted file mode 100644 index 93e2870778..0000000000 --- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/RedisConnectTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.ekstep.ep.samza.task.util; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.when; - -import java.io.IOException; - -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import redis.clients.jedis.Jedis; -import redis.embedded.RedisServer; - -public class RedisConnectTest { - - private RedisServer redisServer; - private Config configMock; - private RedisConnect redisConnect; - - @Before - public void setUp() throws IOException { - redisServer = new RedisServer(6379); - redisServer.start(); - configMock = Mockito.mock(Config.class); - when(configMock.get("redis.host", "localhost")).thenReturn("localhost"); - when(configMock.getInt("redis.port", 6379)).thenReturn(6379); - when(configMock.getInt("redis.connection.max", 2)).thenReturn(2); - when(configMock.getInt("redis.connection.idle.max", 2)).thenReturn(2); - when(configMock.getInt("redis.connection.idle.min", 1)).thenReturn(1); - when(configMock.getInt("redis.connection.minEvictableIdleTimeSeconds", 120)).thenReturn(120); - when(configMock.getInt("redis.connection.timeBetweenEvictionRunsSeconds", 300)).thenReturn(300); - redisConnect = new RedisConnect(configMock); - } - - @After - public void tearDown() { - redisServer.stop(); - } - - @Test - public void shouldVerifyRedisConnect() { - Jedis jedis = redisConnect.getConnection(0); - assertEquals(Long.valueOf(0), jedis.getDB()); - assertNotNull(jedis); - jedis.close(); - - jedis = redisConnect.getConnection(1, 0); - assertNotNull(jedis); - assertEquals(Long.valueOf(1), jedis.getDB()); - jedis.set("key", "value"); - assertEquals("value", jedis.get("key")); - jedis.close(); - - } - -} diff --git a/data-pipeline/ep-telemetry-reader/pom.xml b/data-pipeline/ep-telemetry-reader/pom.xml deleted file mode 100644 index c46811475e..0000000000 --- a/data-pipeline/ep-telemetry-reader/pom.xml +++ /dev/null @@ -1,90 +0,0 @@ - - - - jobs - org.ekstep.ecosystem - 0.0.1 - - 4.0.0 - - ep-telemetry-reader - 0.0.3 - jar - ep-telemetry-reader - - - - org.slf4j - slf4j-api - 1.6.2 - - - org.slf4j - slf4j-log4j12 - 1.6.2 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - - \ No newline at end of file diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Events.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Events.java deleted file mode 100644 index bff396e749..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Events.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.ekstep.ep.samza.events.domain; - - -import com.google.gson.Gson; -import org.ekstep.ep.samza.reader.NullableValue; -import org.ekstep.ep.samza.reader.Telemetry; - -import java.util.List; -import java.util.Map; - -public abstract class Events { - - protected final Telemetry telemetry; - - protected Path path; - - public Events(Map map) { - this.telemetry = new Telemetry(map); - path = new Path(); - } - - public String getChecksum() { - String checksum = id(); - if (checksum != null) - return checksum; - return mid(); - } - - public String id() { - NullableValue checksum = telemetry.read("metadata.checksum"); - return checksum.value(); - } - - public Map getMap() { - return telemetry.getMap(); - } - - public String getJson() { - Gson gson = new Gson(); - String json = gson.toJson(getMap()); - return json; - } - - public String mid() { - NullableValue checksum = telemetry.read("mid"); - return checksum.value(); - } - - public String did() { - NullableValue did = telemetry.read("dimensions.did"); - return did.isNull() ? telemetry.read("context.did").value() : did.value(); - } - - public String eid() { - NullableValue eid = telemetry.read("eid"); - return eid.value(); - } - - @Override - public String toString() { - return "Event{" + "telemetry=" + telemetry + '}'; - } - - - public void updateTs(String value) { - telemetry.add("@timestamp", value); - } - - public String pid() { - NullableValue pid = telemetry.read("context.pdata.pid"); - return pid.value(); - } - - public String version() { - return (String) telemetry.read("ver").value(); - } - - public String producerId() { - NullableValue producerId = telemetry.read("context.pdata.id"); - return producerId.value(); - } - - public final String producerPid() { - NullableValue producerPid = telemetry.read("context.pdata.pid"); - return producerPid.value(); - } - - - public Long ets() { - NullableValue ets = telemetry.read("ets"); - if (ets.value().getClass().equals(Double.class)) { - return ((Double) ets.value()).longValue(); - } - return ((Long) ets.value()); - } - - public String channel() { - NullableValue channel = telemetry.read("dimensions.channel"); - return channel.isNull() ? telemetry.read("context.channel").value() : channel.value(); - } - - public String actorId() { - NullableValue actorid = telemetry.read("uid"); - return actorid.isNull() ? telemetry.read("actor.id").value() : actorid.value(); - } - - - public String actorType() { - NullableValue actortype = telemetry.read("actor.type"); - return actortype.value(); - } - - public String objectID() { - if (objectFieldsPresent()) { - return telemetry.read("object.id").value(); - } else return null; - } - - public String objectType() { - if (objectFieldsPresent()) { - return telemetry.read("object.type").value(); - } else return null; - } - - public boolean objectFieldsPresent() { - String objectId = telemetry.read("object.id").value(); - String objectType = telemetry.read("object.type").value(); - return null != objectId && null != objectType && !objectId.isEmpty() && !objectType.isEmpty(); - } - - public String edataType() { - return telemetry.read("edata.type").value(); - } - - public List> edataItems() { - return telemetry.>>read("edata.items").value(); - } - - - -} diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Path.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Path.java deleted file mode 100644 index b12ef5fb94..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Path.java +++ /dev/null @@ -1,94 +0,0 @@ -package org.ekstep.ep.samza.events.domain; - -public class Path { - - public String metadata() { - return "metadata"; - } - - public String flags() { - return "flags"; - } - - public String dimensionsDid() { - return "dimensions.did"; - } - - public String contextDid() { - return "context.did"; - } - - public String ets() { - return "ets"; - } - - public String ts() { - return "ts"; - } - - public String mid() { - return "mid"; - } - - public String eid() { - return "eid"; - } - - public String deviceData() { - return "devicedata"; - } - - public String loc() { - return "edata.loc"; - } - - public String userData() { - return "userdata"; - } - - public String contentData() { - return "contentdata"; - } - - public String dialCodeData() { - return "dialcodedata"; - } - - public String collectionData() { - return "collectiondata"; - } - - public String ver() { - return "ver"; - } - - public String edata() { - return "edata"; - } - - public String checksum() { - return "metadata.checksum"; - } - - public String channel() { - return "channel"; - } - - public String derivedLocationData() { - return "derivedlocationdata"; - } - - public String stateKey() { - return "state"; - } - - public String districtKey() { - return "district"; - } - - public String locDerivedFromKey() { - return "from"; - } - - -} diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullParent.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullParent.java deleted file mode 100644 index 2b7d612957..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullParent.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.ekstep.ep.samza.reader; - -import java.text.MessageFormat; - -import org.ekstep.ep.samza.core.Logger; - -class NullParent implements ParentType { - static Logger LOGGER = new Logger(NullParent.class); - Object parent; - String childKey; - - NullParent(Object parent, String childKey) { - this.parent = parent; - this.childKey = childKey; - } - - @Override - public T readChild() { - LOGGER.warn(null, MessageFormat.format("NULL PARENT READ CHILD INVOKED FOR PARENT: {0}, CHILD KEY: {1}", parent, childKey)); - return null; - } - - @Override - public void addChild(Object value) { - LOGGER.warn(null, MessageFormat.format("NULL PARENT ADD CHILD INVOKED FOR PARENT: {0}, CHILD KEY: {1}", parent, childKey)); - } -} diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullableValue.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullableValue.java deleted file mode 100644 index 679e9bf241..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullableValue.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.ekstep.ep.samza.reader; - -public class NullableValue { - private T value; - - public NullableValue(T value) { - this.value = value; - } - - public T value() { - return value; - } - - public boolean isNull() { - return value == null; - } - - public T valueOrDefault(T defaultValue) { - if (isNull()) - return defaultValue; - - return value(); - } - - @Override - public String toString() { - return "NullableValue{" + - "value=" + value + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - NullableValue that = (NullableValue) o; - - return value != null ? value.equals(that.value) : that.value == null; - } - - @Override - public int hashCode() { - return value != null ? value.hashCode() : 0; - } -} diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentListOfMap.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentListOfMap.java deleted file mode 100644 index 85cccd36c6..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentListOfMap.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.ekstep.ep.samza.reader; - -import sun.reflect.generics.reflectiveObjects.NotImplementedException; - -import java.util.List; -import java.util.Map; - -import org.ekstep.ep.samza.core.Logger; - -class ParentListOfMap implements ParentType { - //TODO#: Make this class more genic. - static Logger LOGGER = new Logger(ParentListOfMap.class); - List> list; - String childKey; - - ParentListOfMap(List> list, String childKey) { - this.list = list; - this.childKey = childKey; - } - - @Override - public T readChild() { - if (list == null) { - return null; - } - for (Object itemsObject : list) { - if (!(itemsObject instanceof Map)) { - continue; - } - Map items = (Map) itemsObject; - if (items.containsKey(childKey)) { - Object o = items.get(childKey); - if (o instanceof List && ((List) o).size() > 0) { - return (T) ((List) o).get(0); - } - } - } - return null; - } - - @Override - public void addChild(Object value) { - throw new NotImplementedException(); - } -} diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentMap.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentMap.java deleted file mode 100644 index f34bfe6206..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentMap.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.ekstep.ep.samza.reader; - -import java.util.Map; - -import org.ekstep.ep.samza.core.Logger; - -class ParentMap implements ParentType { - static Logger LOGGER = new Logger(ParentMap.class); - Map map; - String childKey; - - ParentMap(Map map, String childKey) { - this.map = map; - this.childKey = childKey; - } - - @Override - public T readChild() { - if (map != null && map.containsKey(childKey) && map.get(childKey) != null) { - Object child = map.get(childKey); - return (T) child; - } - return null; - } - - @Override - public void addChild(Object value) { - if (map != null) - map.put(childKey, value); - } -} diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentType.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentType.java deleted file mode 100644 index 1424b629a7..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentType.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.ekstep.ep.samza.reader; - -public interface ParentType { - T readChild(); - - void addChild(Object value); -} diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/Telemetry.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/Telemetry.java deleted file mode 100644 index 3980a66737..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/Telemetry.java +++ /dev/null @@ -1,150 +0,0 @@ -package org.ekstep.ep.samza.reader; - - -import org.ekstep.ep.samza.core.Logger; - -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.Map; - -import static java.text.MessageFormat.format; - -public class Telemetry { - private Map map; - private Logger logger = new Logger(this.getClass()); - - public Telemetry(Map map) { - this.map = map; - } - - public boolean add(String keyPath, Object value) { - try { - ParentType lastParent = lastParentMap(map, keyPath); - lastParent.addChild(value); - return true; - } catch (Exception e) { - logger.error("", format("Couldn't add value:{0} at key: {0} for event:{1}", value, keyPath, map), e); - } - return false; - } - - public Map getMap() { - return map; - } - - public NullableValue read(String keyPath) { - try { - ParentType parentMap = lastParentMap(map, keyPath); - return new NullableValue(parentMap.readChild()); - } catch (Exception e) { - logger.error("", format("Couldn't get key: {0} from event:{1}", keyPath, map), e); - return new NullableValue(null); - } - } - - public NullableValue readOrDefault(String keyPath, T defaultValue) { - return read(keyPath).isNull() ? new NullableValue(defaultValue) : read(keyPath); - } - - public T mustReadValue(String keyPath) throws TelemetryReaderException { - NullableValue val = read(keyPath); - if (val.isNull()) { - NullableValue eid = read("eid"); - String message = keyPath + " is not available in the event"; - if (!eid.isNull()) { - message = keyPath + " is not available in " + eid.value(); - } - throw new TelemetryReaderException(message); - } - - return val.value(); - } - - private ParentType lastParentMap(Map map, String keyPath) { - Object parent = map; - String[] keys = keyPath.split("\\."); - int lastIndex = keys.length - 1; - if (keys.length > 1) { - for (int i = 0; i < lastIndex && parent != null; i++) { - Object o; - if (parent instanceof Map) { - o = new ParentMap((Map) parent, keys[i]).readChild(); - } else if (parent instanceof List) { - o = new ParentListOfMap((List>) parent, keys[i]).readChild(); - } else { - o = new NullParent(parent, keys[i]).readChild(); - } - parent = o; - } - } - String lastKeyInPath = keys[lastIndex]; - if (parent instanceof Map) { - return new ParentMap((Map) parent, lastKeyInPath); - } else if (parent instanceof List) { - return new ParentListOfMap((List>) parent, lastKeyInPath); - } else { - return new NullParent(parent, lastKeyInPath); - } - } - - @Override - public String toString() { - return "Telemetry{" + - "map=" + map + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Telemetry telemetry = (Telemetry) o; - - return map != null ? map.equals(telemetry.map) : telemetry.map == null; - } - - @Override - public int hashCode() { - return map != null ? map.hashCode() : 0; - } - - public String id() { - return this.read("metadata.checksum").value(); - } - - public void addFieldIfAbsent(String fieldName, T value) { - if (read(fieldName).isNull()) { - add(fieldName, value); - } - } - - public long getEts() throws TelemetryReaderException { - double ets = this.mustReadValue("ets"); - return (long) ets; - } - - public String getAtTimestamp(){ - Object timestamp = this.read("@timestamp").value(); - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - if ( timestamp instanceof Number){ - Date date = new Date(((Number) timestamp).longValue()); - return simpleDateFormat.format(date); - } else { - return this.read("@timestamp").value(); - } - } - - public String getSyncts(){ - Object timestamp = this.read("syncts").value(); - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - if ( timestamp instanceof Number){ - Date date = new Date(((Number) timestamp).longValue()); - return simpleDateFormat.format(date); - } else { - return this.read("syncts").value(); - } - } - -} diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/TelemetryReaderException.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/TelemetryReaderException.java deleted file mode 100644 index 527c1e6e02..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/TelemetryReaderException.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.ekstep.ep.samza.reader; - -public class TelemetryReaderException extends Exception { - public TelemetryReaderException(String message) { - super(message); - } -} diff --git a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventFixture.java b/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventFixture.java deleted file mode 100644 index 8bacf6df7e..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventFixture.java +++ /dev/null @@ -1,102 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import java.util.Map; - -public class EventFixture { - - public static final String IMPRESSION_EVENT_MISSING_FIELDS = "{\n" + - " \"eid\": \"IMPRESSION\",\n" + - " \"ets\": 1574945199426,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"IMPRESSION:0093c96434557b2ead169c7156e95770\",\n" + - " \"actor\": {\n" + - " \"id\": \"f1a99b5cf111be23bd0e8d48a50458c6\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.portal\",\n" + - " \"ver\": \"2.0.0\",\n" + - " \"pid\": \"sunbird-portal\"\n" + - " },\n" + - " \"env\": \"public\",\n" + - " \"sid\": \"1f077b10-11dd-11ea-bbdc-fbe7180a137e\",\n" + - " \"did\": \"f1a99b5cf111be23bd0e8d48a50458c6\",\n" + - " \"cdata\": [],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " },\n" + - " \"uid\": \"anonymous\"\n" + - " },\n" + - " \"object\": {},\n" + - " \"tags\": [\n" + - " \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"type\": \"view\",\n" + - " \"pageid\": \"landing\",\n" + - " \"subtype\": \"init\",\n" + - " \"uri\": \"https://diksha.gov.in/\",\n" + - " \"visits\": []\n" + - " }\n" + - "}"; - - public static final String IMPRESSION_EVENT = "{\n" + - " \"eid\": \"IMPRESSION\",\n" + - " \"ets\": 1574945199426,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"IMPRESSION:0093c96434557b2ead169c7156e95770\",\n" + - " \"actor\": {\n" + - " \"id\": \"f1a99b5cf111be23bd0e8d48a50458c6\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.portal\",\n" + - " \"ver\": \"2.0.0\",\n" + - " \"pid\": \"sunbird-portal\"\n" + - " },\n" + - " \"env\": \"public\",\n" + - " \"sid\": \"1f077b10-11dd-11ea-bbdc-fbe7180a137e\",\n" + - " \"did\": \"f1a99b5cf111be23bd0e8d48a50458c6\",\n" + - " \"cdata\": [],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " },\n" + - " \"uid\": \"anonymous\"\n" + - " },\n" + - " \"object\": {\n" + - " \"type\": \"content\",\n" + - " \"id\": \"658374_49785\"\n" + - " },\n" + - " \"tags\": [\n" + - " \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"type\": \"view\",\n" + - " \"pageid\": \"landing\",\n" + - " \"subtype\": \"init\",\n" + - " \"uri\": \"https://diksha.gov.in/\",\n" + - " \"visits\": []\n" + - " },\n" + - " \"metadata\": {\n" + - " \"checksum\": \"55437853\"\n" + - " },\n" + - " \"@timestamp\": 1575265622000,\n" + - " \"syncts\": 56543875684\n" + - "}"; - - - - - - - public static Map getMap(String message) { - return (Map) new Gson().fromJson(message, Map.class); - } - -} diff --git a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventTest.java b/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventTest.java deleted file mode 100644 index 8e909fb7e9..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventTest.java +++ /dev/null @@ -1,168 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.reader.NullableValue; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Map; - -public class EventTest { - - @Test - public void shouldReturnMid() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("IMPRESSION:0093c96434557b2ead169c7156e95770", event.mid()); - } - - @Test - public void shouldReturnDid() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("f1a99b5cf111be23bd0e8d48a50458c6", event.did()); - } - - @Test - public void shouldReturnEid() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("IMPRESSION", event.eid()); - } - - @Test - public void shouldReturnPid() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("sunbird-portal", event.pid()); - } - - @Test - public void shouldReturnVersion() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("3.0", event.version()); - } - - @Test - public void shouldReturunContextPID() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("sunbird-portal", event.producerPid()); - } - - @Test - public void shouldReturunEts() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Long ets = 1574945199426L; - Assert.assertEquals(ets, event.ets()); - } - - @Test - public void shouldReturnChannel() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("505c7c48ac6dc1edc9b08f21db5a571d", event.channel()); - } - - @Test - public void shouldReturnActorId() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("f1a99b5cf111be23bd0e8d48a50458c6", event.actorId()); - } - - @Test - public void shouldReturnActorType() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("User", event.actorType()); - } - - @Test - public void shouldReturnObjectId() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("658374_49785", event.objectID()); - } - - @Test - public void shoudlReturnNullWhenObjectIdIsNotPresent() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT_MISSING_FIELDS)); - Assert.assertNull(event.objectID()); - } - - @Test - public void shouldReturnProducerPID() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("prod.diksha.portal", event.producerId()); - } - - @Test - public void shouldReturnObjectType() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("content", event.objectType()); - } - - @Test - public void shouldReturnNullWhenObjectTypeIsNotPresent() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT_MISSING_FIELDS)); - Assert.assertNull(event.objectType()); - } - - @Test - public void shouldReturnEdataType() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals("view", event.edataType()); - } - - @Test - public void shouldGetEventJsonObject() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNotNull(event.getJson()); - } - - @Test - public void shouldGetTheMetaDataId() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNotNull(event.id()); - } - - @Test - public void shouldGetTheCheckSumValue() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNotNull(event.getChecksum()); - } - - @Test - public void shouldGetTheStringObject() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNotNull(event.toString()); - } - - @Test - public void shouldUpdateTheTSValue() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - event.updateTs("5468376530"); - Assert.assertEquals(event.getData().value(), "5468376530"); - } - - @Test - public void shouldGetMidValueIfTheCheckSumIsNotPresent() { - Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT_MISSING_FIELDS)); - - Assert.assertEquals(event.getChecksum(), "IMPRESSION:0093c96434557b2ead169c7156e95770"); - } - - -} - - -/** - * Creating a dummy event class to to test the base abstract class object - */ - -class Event extends Events { - - public Event(Map map) { - super(map); - } - - public NullableValue getData() { - return telemetry.read("@timestamp"); - } - -} - - - diff --git a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/PathTest.java b/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/PathTest.java deleted file mode 100644 index 4fbeb5934a..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/PathTest.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.events.domain.Path; -import org.junit.Assert; -import org.junit.Test; - -public class PathTest { - private Path path = new Path(); - - @Test - public void shouldGetMetadataName() { - Assert.assertEquals("metadata", path.metadata()); - } - @Test - public void shouldGetFlagsName() { - Assert.assertEquals("flags", path.flags()); - } - @Test - public void shouldGetDimensionDid() { - Assert.assertEquals("dimensions.did", path.dimensionsDid()); - } - @Test - public void shouldGetContextDid() { - Assert.assertEquals("context.did", path.contextDid()); - } - @Test - public void shouldGetTheEts() { - Assert.assertEquals("ets", path.ets()); - } - @Test - public void shouldGetCheckSumKey() { - Assert.assertEquals("metadata.checksum", path.checksum()); - } - @Test - public void shouldGetTimeStampKey() { - Assert.assertEquals("ts", path.ts()); - } - @Test - public void shouldGetMidKey() { - Assert.assertEquals("mid", path.mid()); - } - @Test - public void shouldGetEidKey() { - Assert.assertEquals("eid", path.eid()); - } - @Test - public void shouldGetDeviceDataKey() { - Assert.assertEquals("devicedata", path.deviceData()); - } - @Test - public void shouldGetLocationKey() { - Assert.assertEquals("edata.loc", path.loc()); - } - @Test - public void shouldGetUserDataKey() { - Assert.assertEquals("userdata", path.userData()); - } - @Test - public void shouldGetContentDataKey() { - Assert.assertEquals("contentdata", path.contentData()); - } - @Test - public void shouldGetDialCodeDataKey() { - Assert.assertEquals("dialcodedata", path.dialCodeData()); - } - @Test - public void shouldGetCollectionDataKey() { - Assert.assertEquals("collectiondata", path.collectionData()); - } - @Test - public void shouldGetDerivedLocationDataKey() { - Assert.assertEquals("derivedlocationdata", path.derivedLocationData()); - } - @Test - public void shouldGetStateKey() { - Assert.assertEquals("state", path.stateKey()); - } - @Test - public void shouldGetDistrictKey() { - Assert.assertEquals("district", path.districtKey()); - } - @Test - public void shouldGetLocationDerivedFromKey() { - Assert.assertEquals("from", path.locDerivedFromKey()); - } - @Test - public void shouldGetEdataKey() { - Assert.assertEquals("edata", path.edata()); - } - @Test - public void shouldGetChannelKey() { - Assert.assertEquals("channel", path.channel()); - } - @Test - public void shouldGetVersionKey() { - Assert.assertEquals("ver", path.ver()); - } - -} diff --git a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/reader/TelemetryTest.java b/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/reader/TelemetryTest.java deleted file mode 100644 index 5f4b225222..0000000000 --- a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/reader/TelemetryTest.java +++ /dev/null @@ -1,266 +0,0 @@ -package org.ekstep.ep.samza.reader; - -import org.ekstep.ep.samza.domain.EventFixture; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.*; - -public class TelemetryTest { - - - @Test - public void shouldReadTheDefaultValue() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT_MISSING_FIELDS)); - Assert.assertEquals(telemetry.readOrDefault("context.channel", "in.sunbird").value(), "in.sunbird"); - } - - @Test - public void shouldReadTheActualValue() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertEquals(telemetry.readOrDefault("context.channel", "in.sunbird").value(), "505c7c48ac6dc1edc9b08f21db5a571d"); - } - - - @Test - public void shouldGetFirstStringValue() { - Map hashMap = new HashMap(); - hashMap.put("key1", "value1"); - Telemetry telemetry = new Telemetry(hashMap); - NullableValue nullableValue = telemetry.read("key1"); - assertEquals("value1", nullableValue.value()); - assertFalse(nullableValue.isNull()); - } - - @Test - public void shouldGetFirstMapValue() { - Map hashMap = new HashMap(); - HashMap nestedObject = new HashMap(); - nestedObject.put("key2", "get"); - hashMap.put("key1", nestedObject); - Telemetry telemetry = new Telemetry(hashMap); - NullableValue nullableValue = telemetry.read("key1"); - assertEquals(nestedObject, nullableValue.value()); - assertFalse(nullableValue.isNull()); - } - - @Test - public void shouldGetOneNestedStringValue() { - Map hashMap = new HashMap(); - HashMap nestedObject = new HashMap(); - nestedObject.put("key2", "get"); - hashMap.put("key1", nestedObject); - Telemetry telemetry = new Telemetry(hashMap); - NullableValue nullableValue = telemetry.read("key1.key2"); - assertEquals("get", nullableValue.value()); - assertFalse(nullableValue.isNull()); - } - - @Test - public void shouldGetOneNestedMapValue() { - Map hashMap = new HashMap(); - HashMap nestedObject = new HashMap(); - HashMap otherNestedObject = new HashMap(); - otherNestedObject.put("key3", "get"); - nestedObject.put("key2", otherNestedObject); - hashMap.put("key1", nestedObject); - Telemetry telemetry = new Telemetry(hashMap); - NullableValue nullableValue = telemetry.read("key1.key2"); - assertEquals(otherNestedObject, nullableValue.value()); - assertFalse(nullableValue.isNull()); - } - - @Test - public void shouldGetTwoNestedValue() { - Map hashMap = new HashMap(); - HashMap nestedObject = new HashMap(); - HashMap otherNestedObject = new HashMap(); - otherNestedObject.put("key3", "get"); - nestedObject.put("key2", otherNestedObject); - hashMap.put("key1", nestedObject); - Telemetry telemetry = new Telemetry(hashMap); - NullableValue nullableValue = telemetry.read("key1.key2.key3"); - assertEquals("get", nullableValue.value()); - assertFalse(nullableValue.isNull()); - } - - @Test - public void shouldGetNullValueForMissingKey() { - Map hashMap = new HashMap(); - hashMap.put("key1", "get"); - Telemetry telemetry = new Telemetry(hashMap); - NullableValue nullableValue = telemetry.read("invalidKey"); - assertEquals(null, nullableValue.value()); - assertTrue(nullableValue.isNull()); - } - - @Test - public void shouldNotFailWhenReadingWrongNesting() { - Map hashMap = new HashMap(); - hashMap.put("key1", "get"); - Telemetry telemetry = new Telemetry(hashMap); - NullableValue nullableValue = telemetry.read("key1.invalidKey"); - assertEquals(null, nullableValue.value()); - assertTrue(nullableValue.isNull()); - } - - @Test - public void shouldGetNullValueForBothMissingNestedKey() { - Map hashMap = new HashMap(); - hashMap.put("key1", "get"); - Telemetry telemetry = new Telemetry(hashMap); - NullableValue nullableValue = telemetry.read("invalidKey1.invalidKey2"); - assertEquals(null, nullableValue.value()); - assertTrue(nullableValue.isNull()); - } - - @Test - public void shouldAddValue() { - Map hashMap = new HashMap(); - HashMap nestedMap = new HashMap(); - Telemetry telemetry = new Telemetry(hashMap); - telemetry.add("key", nestedMap); - assertEquals(nestedMap, telemetry.getMap().get("key")); - } - - @Test - public void shouldOverrideValue() { - Map hashMap = new HashMap(); - HashMap nestedMap = new HashMap(); - HashMap overrideNestedMap = new HashMap(); - hashMap.put("key", nestedMap); - Telemetry telemetry = new Telemetry(hashMap); - telemetry.add("key", overrideNestedMap); - assertEquals(overrideNestedMap, telemetry.getMap().get("key")); - } - - @Test - public void shouldAddNestedValue() { - Map hashMap = new HashMap(); - HashMap nestedMap = new HashMap(); - hashMap.put("key", nestedMap); - HashMap otherNestedMap = new HashMap(); - Telemetry telemetry = new Telemetry(hashMap); - telemetry.add("key.nested", otherNestedMap); - assertEquals(otherNestedMap, telemetry.read("key.nested").value()); - } - - @Test - public void shouldNotAddWhenKeysDoesNotExists() { - Map hashMap = new HashMap(); - HashMap nestedMap = new HashMap(); - Telemetry telemetry = new Telemetry(hashMap); - telemetry.add("invalidKey.nested", nestedMap); - assertFalse(telemetry.getMap().containsKey("invalidKey")); - } - - @Test - public void shouldGetTheEts() { - double ets = 12334534; - Map hashMap = new HashMap(); - hashMap.put("ets", ets); - HashMap nestedMap = new HashMap(); - Telemetry telemetry = new Telemetry(hashMap); - try { - assertEquals(12334534, telemetry.getEts()); - } catch (TelemetryReaderException e) { - - } - - assertFalse(telemetry.getMap().containsKey("invalidKey")); - } - - @Test - public void shouldGetTheTimeStamp() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNotNull(telemetry.getAtTimestamp()); - } - - @Test - public void shouldGetTheSyncTS() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNotNull(telemetry.getSyncts()); - } - - @Test - public void shouldGetHashCodeValue() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNotNull(telemetry.hashCode()); - } - - @Test - public void shouldGetStringObject() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNotNull(telemetry.toString()); - } - - @Test - public void shouldGetTheMetaDataValue() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNotNull(telemetry.id()); - } - - - @Test - public void CheckObjectsAreEqualOrNot() { - Telemetry telemetry1 = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Telemetry telemetry2 = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - String stringObj = new String(); - Assert.assertTrue(telemetry1.equals(telemetry2)); - Assert.assertFalse(telemetry1.equals(stringObj)); - } - - @Test - public void ShouldAddFieldIfNotPresent() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - telemetry.addFieldIfAbsent("error", "Invalid Key"); - Assert.assertEquals(telemetry.read("error").value(), "Invalid Key"); - } - - @Test - public void ShouldReturnNullIfTheKeyIsNotPresent() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Assert.assertNull(telemetry.read("invalidKey").value()); - } - - @Test - public void shouldThrowError() { - Telemetry telemetry = new Telemetry(null); - - try { - Assert.assertNull(telemetry.mustReadValue("invalidKey")); - } catch (Exception e) { - Assert.assertNotNull(e.getMessage()); - } - } - - @Test - public void shouldValidateTheNullableObject() { - Telemetry telemetry1 = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - Telemetry telemetry2 = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - NullableValue eid1 = telemetry1.read("eid"); - NullableValue eid2 = telemetry2.read("eid"); - Assert.assertTrue(eid1.equals(eid2)); - System.out.println(eid1.equals(eid2)); - Assert.assertNotNull(eid1.hashCode()); - Assert.assertNotNull(eid1.toString()); - } - - @Test - public void shouldGetTheDefaultValue() { - Telemetry telemetry = new Telemetry(null); - NullableValue eid1 = telemetry.read("eid"); - Assert.assertEquals(eid1.valueOrDefault("START"), "START"); - } - - @Test - public void shouldGetTheActualValue() { - Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT)); - NullableValue eid1 = telemetry.read("eid"); - Assert.assertEquals(eid1.valueOrDefault("START"), "IMPRESSION"); - } - -} \ No newline at end of file diff --git a/data-pipeline/events-router/pom.xml b/data-pipeline/events-router/pom.xml deleted file mode 100644 index 392b2f1233..0000000000 --- a/data-pipeline/events-router/pom.xml +++ /dev/null @@ -1,190 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - - org.ekstep.ecosystem.jobs - events-router - 0.1.7 - jar - EventsRouter - - Router job to route telemetry and summary events to separate topics - - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - - com.google.guava - guava - 18.0 - - - com.fiftyonred - mock-jedis - 0.4.0 - test - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - diff --git a/data-pipeline/events-router/src/main/assembly/src.xml b/data-pipeline/events-router/src/main/assembly/src.xml deleted file mode 100644 index e31a82cf30..0000000000 --- a/data-pipeline/events-router/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/events-router.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-kv-rocksdb_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:events-router - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/events-router/src/main/config/events-router.properties b/data-pipeline/events-router/src/main/config/events-router.properties deleted file mode 100644 index d122d557b4..0000000000 --- a/data-pipeline/events-router/src/main/config/events-router.properties +++ /dev/null @@ -1,91 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.EventsRouter - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__events_router_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.EventsRouterTask -task.inputs=kafka.__env__.telemetry.denorm.valid -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -router.events.telemetry.route.topic=__env__.events.telemetry -router.events.summary.route.topic=__env__.events.summary -router.events.summary.route.events=ME_WORKFLOW_SUMMARY -output.failed.topic.name=__env__.telemetry.failed -output.duplicate.topic.name=__env__.telemetry.duplicate -output.malformed.topic.name=__env__.telemetry.malformed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,skipped-message-count,error-message-count,duplicate-event-count - -# dedup configurations - -dedup.enabled=true -# redis -#redis.host=localhost -redis.host=__redis_host__ -#redis.port=6379 -redis.port=__redis_port__ -redis.connection.max=2 -redis.connection.idle.max=2 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.database.duplicationstore.id=8 -redis.database.key.expiry.seconds=86400 - -dedup.exclude.eids=__dedup_exclude_eids__ - diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index 8c40be6dab..0000000000 --- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.task.EventsRouterConfig; - -import java.util.HashMap; -import java.util.Map; - -public class Event extends Events { - - public Event(Map map) { - super(map); - } - - public void markFailure(String error, EventsRouterConfig config) { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.tr_processed", false); - - telemetry.addFieldIfAbsent("metadata", new HashMap()); - telemetry.add("metadata.tr_error", error); - telemetry.add("metadata.src", config.jobName()); - } - - public void markSkipped() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.dd_dr_processed", false); - telemetry.add("flags.dd_dr_checksum_present", false); - } - - public void markDuplicate() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.dd_dr_processed", false); - telemetry.add("flags.dd_dr_duplicate_event", true); - } - -} diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/service/EventsRouterService.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/service/EventsRouterService.java deleted file mode 100644 index f63cdabc53..0000000000 --- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/service/EventsRouterService.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.JsonSyntaxException; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.EventsRouterConfig; -import org.ekstep.ep.samza.task.EventsRouterSink; -import org.ekstep.ep.samza.task.EventsRouterSource; -import org.ekstep.ep.samza.util.DeDupEngine; -import redis.clients.jedis.exceptions.JedisException; - -import java.text.SimpleDateFormat; -import java.util.Date; - -import static java.text.MessageFormat.format; - -public class EventsRouterService { - - private static Logger LOGGER = new Logger(EventsRouterService.class); - private final DeDupEngine deDupEngine; - private final EventsRouterConfig config; - - public EventsRouterService(DeDupEngine deDupEngine, EventsRouterConfig config) { - - this.config = config; - this.deDupEngine = deDupEngine; - } - - public void process(EventsRouterSource source, EventsRouterSink sink) { - Event event = null; - try { - event = source.getEvent(); - if (config.isDedupEnabled() && isDupCheckRequired(event.eid())) { - String checksum = event.getChecksum(); - if (!deDupEngine.isUniqueEvent(checksum)) { - LOGGER.info(event.id(), "DUPLICATE EVENT, CHECKSUM: {}", checksum); - event.markDuplicate(); - sink.toDuplicateTopic(event); - return; - } - LOGGER.info(event.id(), "ADDING EVENT CHECKSUM TO STORE"); - deDupEngine.storeChecksum(checksum); - } else { - LOGGER.info(event.id(), "SKIPPING THE DEDUP CHECK"); - } - String eid = event.eid(); - if(event.mid().contains("TRACE")){ - SimpleDateFormat simple = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - String timeStamp = simple.format(new Date()); - event.updateTs(timeStamp); - } - String summaryRouteEventPrefix = this.config.getSummaryRouteEvents(); - if (eid.startsWith(summaryRouteEventPrefix)) { - sink.toSummaryEventsTopic(event); - } else if (eid.startsWith("ME_")) { - sink.incrementSkippedCount(event); - } else { - sink.toTelemetryEventsTopic(event); - } - } catch (JedisException e) { - e.printStackTrace(); - LOGGER.error(null, "Exception when retrieving data from redis: ", e); - deDupEngine.getRedisConnection().close(); - throw e; - } catch (JsonSyntaxException e) { - e.printStackTrace(); - LOGGER.error(null, "INVALID EVENT: " + source.getMessage()); - sink.toMalformedTopic(source.getMessage()); - } catch (Exception e) { - e.printStackTrace(); - LOGGER.error(null, - format("EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:", - event), - e); - sink.toErrorTopic(event, e.getMessage()); - } - } - - private boolean isDupCheckRequired(String eid) { - return (config.excludedEids().isEmpty() || (null != eid && !(config.excludedEids().contains(eid)))); - } -} diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterConfig.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterConfig.java deleted file mode 100644 index 1a69d0cfc9..0000000000 --- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterConfig.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; - -import java.util.ArrayList; -import java.util.List; - -public class EventsRouterConfig { - - private final String JOB_NAME = "EventsRouter"; - - private String failedTopic; - private String duplicateTopic; - private String telemetryEventsRouteTopic; - private String summaryEventsRouteTopic; - private String summaryRouteEvents; - private String malformedTopic; - private boolean dedupEnabled; - private final int dupStore; - private int expirySeconds; - private List excludedEids = new ArrayList<>(); - - public EventsRouterConfig(Config config) { - failedTopic = config.get("output.failed.topic.name", "telemetry.failed"); - duplicateTopic = config.get("output.duplicate.topic.name", "telemetry.duplicate"); - telemetryEventsRouteTopic = config.get("router.events.telemetry.route.topic", "events.telemetry"); - summaryRouteEvents = config.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY"); - summaryEventsRouteTopic = config.get("router.events.summary.route.topic", "events.summary"); - malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed"); - dedupEnabled = config.getBoolean("dedup.enabled", true); - dupStore = config.getInt("redis.database.duplicationstore.id", 8); - expirySeconds = config.getInt("redis.database.key.expiry.seconds", 28800); - if (!config.get("dedup.exclude.eids", "").isEmpty()) { - excludedEids = config.getList("dedup.exclude.eids", new ArrayList<>()); - } - else { - excludedEids = new ArrayList<>(); - } - } - - public String getTelemetryEventsRouteTopic() { - return telemetryEventsRouteTopic; - } - - - public String getSummaryEventsRouteTopic() { - return summaryEventsRouteTopic; - } - - - public String getSummaryRouteEvents() { - return this.summaryRouteEvents; - } - - public String failedTopic() { - return failedTopic; - } - - public String duplicateTopic() { - return duplicateTopic; - } - - public String malformedTopic() { - return malformedTopic; - } - - public String jobName() { - return JOB_NAME; - } - - public Boolean isDedupEnabled() { - return dedupEnabled; - } - - public int dupStore() { - return dupStore; - } - - public int expirySeconds() { - return expirySeconds; - } - - public List excludedEids() { - return excludedEids; - } -} \ No newline at end of file diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSink.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSink.java deleted file mode 100644 index fa34845910..0000000000 --- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSink.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - -public class EventsRouterSink extends BaseSink { - - // private JobMetrics metrics; - private EventsRouterConfig config; - - public EventsRouterSink(MessageCollector collector, JobMetrics metrics, EventsRouterConfig config) { - super(collector, metrics); - // this.metrics = metrics; - this.config = config; - } - - public void toTelemetryEventsTopic(Event event) { - toTopic(config.getTelemetryEventsRouteTopic(), event.did(), event.getJson()); - metrics.incSuccessCounter(); - } - - public void toErrorTopic(Event event, String errorMessage) { - event.markFailure(errorMessage, config); - toTopic(config.failedTopic(), event.did(), event.getJson()); - metrics.incErrorCounter(); - } - - public void toDuplicateTopic(Event event) { - toTopic(config.duplicateTopic(), event.did(), event.getJson()); - metrics.incDuplicateCounter(); - } - - public void toMalformedTopic(String message) { - toTopic(config.malformedTopic(), null, message); - metrics.incErrorCounter(); - } - - public void toSummaryEventsTopic(Event event) { - toTopic(config.getSummaryEventsRouteTopic(), event.did(), event.getJson()); - metrics.incSuccessCounter(); - } - - public void incrementSkippedCount(Event event) { - metrics.incSkippedCounter(); - } - -} diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSource.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSource.java deleted file mode 100644 index ad150c7f5e..0000000000 --- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSource.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ekstep.ep.samza.task; - -import java.util.Map; - -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; - -import com.google.gson.Gson; - -public class EventsRouterSource { - static Logger LOGGER = new Logger(EventsRouterSource.class); - - private IncomingMessageEnvelope envelope; - - public EventsRouterSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - return new Event(getMap()); - } - - @SuppressWarnings("unchecked") - private Map getMap() { - String message = (String) envelope.getMessage(); - return (Map) new Gson().fromJson(message, Map.class); - } - - public String getMessage() { - return envelope.toString(); - } - - -} diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterTask.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterTask.java deleted file mode 100644 index 2c137fb5bf..0000000000 --- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterTask.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.service.EventsRouterService; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.ekstep.ep.samza.util.RedisConnect; - -public class EventsRouterTask extends BaseSamzaTask { - - static Logger LOGGER = new Logger(EventsRouterTask.class); - private EventsRouterConfig config; - private JobMetrics metrics; - private EventsRouterService service; - - public EventsRouterTask(DeDupEngine deDupEngine, Config config, TaskContext context) { - init(config, context, deDupEngine); - } - - public EventsRouterTask() { - - } - - @Override - public void init(Config config, TaskContext context) { - - this.config = new EventsRouterConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - init(config, context, null); - } - - private void init(Config config, TaskContext context, DeDupEngine deDupEngine) { - this.config = new EventsRouterConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - deDupEngine = deDupEngine == null ? - new DeDupEngine(new RedisConnect(config), this.config.dupStore(), this.config.expirySeconds()) : deDupEngine; - service = new EventsRouterService(deDupEngine, this.config); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) - throws Exception { - - EventsRouterSink sink = new EventsRouterSink(collector, metrics, config); - EventsRouterSource source = new EventsRouterSource(envelope); - service.process(source, sink); - } - -} diff --git a/data-pipeline/events-router/src/main/resources/log4j.xml b/data-pipeline/events-router/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/events-router/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index d783838071..0000000000 --- a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,327 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; - -import java.util.Map; - -public class EventFixture { - - public static final String WORKFLOW_SUMMARY_EVENT = "{\n" + - " \"eid\": \"ME_WORKFLOW_SUMMARY\",\n" + - " \"ets\": 1548040103586,\n" + - " \"syncts\": 1547991371195,\n" + - " \"mid\": \"837C68F76FB02D338E39DFBEE1095E3B\",\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"id\": \"AnalyticsDataPipeline\",\n" + - " \"ver\": \"1.0\",\n" + - " \"model\": \"WorkflowSummarizer\"\n" + - " },\n" + - " \"granularity\": \"SESSION\",\n" + - " \"date_range\": {\n" + - " \"from\": 1547991074190,\n" + - " \"to\": 1547991170529\n" + - " }\n" + - " },\n" + - " \"dimensions\": {\n" + - " \"did\": \"e57185808603966cb58e18647bfc5f0de4e4d820\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.app\",\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\"\n" + - " },\n" + - " \"sid\": \"f962116f-3f84-446b-964c-fae92a15fefd\",\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"type\": \"session\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"interact_events_per_min\": 3.74,\n" + - " \"start_time\": 1547991074190,\n" + - " \"end_time\": 1547991170529,\n" + - " \"events_summary\": [\n" + - " { \"id\": \"START\", \"count\": 3 },\n" + - " { \"id\": \"IMPRESSION\", \"count\": 5 },\n" + - " { \"id\": \"END\", \"count\": 3 }\n" + - " ],\n" + - " \"page_summary\": [\n" + - " {\n" + - " \"id\": \"collection-detail\",\n" + - " \"type\": \"detail\",\n" + - " \"env\": \"home\",\n" + - " \"time_spent\": 13.45,\n" + - " \"visit_count\": 1\n" + - " },\n" + - " {\n" + - " \"id\": \"library\",\n" + - " \"type\": \"search\",\n" + - " \"env\": \"home\",\n" + - " \"time_spent\": 28.09,\n" + - " \"visit_count\": 1\n" + - " }\n" + - " ],\n" + - " \"time_diff\": 96.34,\n" + - " \"telemetry_version\": \"3.0\",\n" + - " \"env_summary\": [\n" + - " {\n" + - " \"env\": \"home\",\n" + - " \"time_spent\": 41.54,\n" + - " \"count\": 1\n" + - " }\n" + - " ],\n" + - " \"time_spent\": 96.36\n" + - " }\n" + - " }\n" + - "}"; - - public static final String ERROR_EVENT = "{\n" + - " \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" + - " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + - " \"eid\": \"ERROR\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genie.android\",\n" + - " \"ver\": \"2.2.15\"\n" + - " },\n" + - " \"pdata\": {\n" + - " \"id\": \"genie\",\n" + - " \"ver\": \"2.0\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"ets\": 1454064092546,\n" + - " \"uid\": \"\",\n" + - " \"ver\": \"2.2\",\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"correlationid\",\n" + - " \"type\": \"correlationtype\"\n" + - " }\n" + - " ]\n" + - "}"; - public static final String UNPARSABLE_START_EVENT = "{\n" + - " \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n" + - " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"mode\": \"WIFI\",\n" + - " \"ver\": \"12\",\n" + - " \"size\": 12.67,\n" + - " \"err\": \"\",\n" + - " \"referrer\": [\n" + - " {\n" + - " \"action\": \"INSTALL\",\n" + - " \"utmsource\": \"Ekstep\",\n" + - " \"utmmedium\": \"Portal\",\n" + - " \"utmterm\": \"December 2016\",\n" + - " \"utmcontent\": \"Ramayana\",\n" + - " \"utmcampaign\": \"Epics of India\"\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " \"eid\": \"START\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genie.android\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"ets\": 1454064092546,\n" + - " \"uid\": \"\",\n" + - " \"ver\": \"2.0\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"correlationid\",\n" + - " \"type\": \"correlationtype\"\n" + - " "; - public static final String START_EVENT = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"3ae5dc29-17ea-4465-800e-a4361b9d9604\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"type\": \"app\"\n" + - " },\n" + - " \"eid\": \"START\",\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1548720004773,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.30\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"home\",\n" + - " \"did\": \"28b3576420784da6a7b556c1966c759fc793ac0d\",\n" + - " \"sid\": \"042d7f61-225b-4e3f-9fe4-93f373368d8f\",\n" + - " \"cdata\": []\n" + - " },\n" + - " \"mid\": \"b8b0f2ca-ce0a-427d-9c47-195eb5424071\",\n" + - " \"object\": {\n" + - " \"id\": \"\",\n" + - " \"type\": \"\",\n" + - " \"version\": \"\"\n" + - " },\n" + - " \"tags\": [],\n" + - " \"syncts\": 1548720010819,\n" + - " \"@timestamp\": \"2019-01-29T00:00:10.819Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - public static final String EVENT_WITH_NULL_EID = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"3ae5dc29-17ea-4465-800e-a4361b9d9604\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"type\": \"app\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1548720004773,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.30\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"home\",\n" + - " \"did\": \"28b3576420784da6a7b556c1966c759fc793ac0d\",\n" + - " \"sid\": \"042d7f61-225b-4e3f-9fe4-93f373368d8f\",\n" + - " \"cdata\": []\n" + - " },\n" + - " \"mid\": \"b8b0f2ca-ce0a-427d-9c47-195eb5424071\",\n" + - " \"object\": {\n" + - " \"id\": \"\",\n" + - " \"type\": \"\",\n" + - " \"version\": \"\"\n" + - " },\n" + - " \"tags\": [],\n" + - " \"syncts\": 1548720010819,\n" + - " \"@timestamp\": \"2019-01-29T00:00:10.819Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String ANY_STRING = "Hey Samza, Whats Up?"; - public static final String EMPTY_JSON = "{}"; - - public static final String LOG_EVENT = "{\n"+ - " \"@timestamp\":\"2019-03-20T00:00:01.176Z\",\n"+ - "\"actor\":{\n"+ - "\"id\":\"0b251080-3230-415e-a593-ab7c1fac7ae3\",\n"+ - "\"type\":\"User\"\n"+ - "},\n"+ - "\"context\":{\n"+ - "\"cdata\":[\n"+ - - "],\n"+ - "\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\n"+ - /*"\"did\":\"923c675274cfbf19fd0402fe4d2c37afd597f0ab\",\n"+*/ - "\"env\":\"home\",\n"+ - "\"pdata\":{\n"+ - "\"id\":\"prod.diksha.app\",\n"+ - "\"pid\":\"sunbird.app\",\n"+ - "\"ver\":\"2.1.45\"\n"+ - "},\n"+ - "\"sid\":\"57b5b7ea-93c5-48d6-ba51-f5f9a3570ffe\"\n"+ - "},\n"+ - "\"edata\":{\n"+ - "\"level\":\"INFO\",\n"+ - "\"message\":\"content-detail\",\n"+ - "\"params\":[\n"+ - "{\n"+ - "\"PopupType\":\"automatic\"\n"+ - "}\n"+ - "],\n"+ - "\"type\":\"view\"\n"+ - "},\n"+ - "\"eid\":\"LOG\",\n"+ - "\"ets\":1.553039987481E12,\n"+ - "\"flags\":{\n"+ - "\"dd_processed\":true,\n"+ - "\"tv_processed\":true\n"+ - "},\n"+ - "\"mid\":\"ca17e5bd-71d4-487a-92cd-0fb377e7a591\",\n"+ - "\"syncts\":1.553040001176E12,\n"+ - "\"tags\":[\n"+ - "],\n"+ - "\"type\":\"events\",\n"+ - "\"ver\":\"3.0\"\n"+ - " }"; - public static final String TELEMETRY_ERROR_EVENT = "{"+ - " \"eid\":\"ERROR\","+ - " \"ets\":%s,"+ - " \"ver\":\"3.0\","+ - " \"mid\":\"LP.1553040097857.bf0e4e15-014e-4a22-ba00-e02ff3c38784\","+ - " \"actor\":{"+ - " \"id\":\"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\","+ - " \"type\":\"System\""+ - " },"+ - " \"context\":{"+ - " \"channel\":\"01235953109336064029450\","+ - " \"pdata\":{"+ - " \"id\":\"prod.ntp.learning.platform\","+ - " \"pid\":\"learning-service\","+ - " \"ver\":\"1.0\""+ - " },"+ - " \"env\":\"framework\""+ - " },"+ - " \"edata\":{"+ - " \"err\":\"ERR_DATA_NOT_FOUND\","+ - " \"stacktrace\":\"ERR_DATA_NOT_FOUND: Data not found with id : null\n\tat\","+ - " \"errtype\":\"system\""+ - " },"+ - " \"flags\":{"+ - " \"tv_processed\":true,"+ - " \"dd_processed\":true"+ - " },"+ - " \"type\":\"events\","+ - " \"syncts\":1.553040098435E12,"+ - " \"@timestamp\":\"2019-03-20T00:01:38.435Z\""+ - "}"; - - public static final String TRACE_EVENT = "{"+ - " \"eid\":\"TRACE\","+ - " \"ets\":%s,"+ - " \"ver\":\"3.0\","+ - " \"mid\":\"TRACE.bf0e4e15-014e-4a22-ba00-e02ff3c38784\","+ - " \"actor\":{"+ - " \"id\":\"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\","+ - " \"type\":\"System\""+ - " },"+ - " \"context\":{"+ - " \"channel\":\"01235953109336064029450\","+ - " \"pdata\":{"+ - " \"id\":\"prod.ntp.learning.platform\","+ - " \"pid\":\"learning-service\","+ - " \"ver\":\"1.0\""+ - " },"+ - " \"env\":\"framework\""+ - " },"+ - " \"edata\":{"+ - " \"err\":\"ERR_DATA_NOT_FOUND\","+ - " \"stacktrace\":\"ERR_DATA_NOT_FOUND: Data not found with id : null\n\tat\","+ - " \"errtype\":\"system\""+ - " },"+ - " \"flags\":{"+ - " \"tv_processed\":true,"+ - " \"dd_processed\":true"+ - " },"+ - " \"type\":\"events\","+ - " \"syncts\":1.553040098435E12,"+ - " \"@timestamp\":\"2019-03-20T00:01:38.435Z\""+ - "}"; - - public static final String WORKFLOW_USAGE_EVENT = "{\"eid\":\"ME_WORKFLOW_USAGE_SUMMARY\",\"ets\":1580174594135,\"syncts\":1580112438526,\"ver\":\"2.1\",\"mid\":\"7FA1DEB21833F00EE8AD66FF8AFF4589\",\"uid\":\"\",\"channel\":\"\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"WorkFlowUsageSummarizer\"},\"granularity\":\"DAY\",\"date_range\":{\"from\":1580108039040,\"to\":1580112356069}},\"dimensions\":{\"uid\":\"04b88827-87b1-40fd-a662-b1b97068d127\",\"did\":\"all\",\"pdata\":{\"id\":\"prod.diksha.app\",\"ver\":\"2.3.162\",\"pid\":\"sunbird.app\"},\"tag\":\"all\",\"period\":20200127,\"content_id\":\"all\",\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"type\":\"resource\",\"mode\":\"play\",\"content_type\":\"all\"},\"edata\":{\"eks\":{\"total_users_count\":0,\"total_devices_count\":1,\"total_content_count\":4,\"avg_ts_session\":4.88,\"total_sessions\":5,\"avg_interactions_min\":24.58,\"total_interactions\":10,\"avg_pageviews\":0.0,\"total_ts\":24.41,\"total_pageviews_count\":0}}}"; - public static Map getMap(String message) { - return (Map) new Gson().fromJson(message, Map.class); - } - -} \ No newline at end of file diff --git a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java deleted file mode 100644 index a22a07d379..0000000000 --- a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.ekstep.ep.samza.system; - -import java.util.Map; - -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.junit.Assert; -import org.junit.Test; - -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; -import com.google.gson.reflect.TypeToken; - -public class EventTest { - - @Test - public void shouldReturnMid() { - - Event event = new Event(EventFixture.getMap(EventFixture.ERROR_EVENT)); - Assert.assertEquals("43288930-e54a-230b-b56e-876gnm8712ok", event.mid()); - } - - @Test - public void shouldReturnEid() { - - Event event = new Event(EventFixture.getMap(EventFixture.WORKFLOW_SUMMARY_EVENT)); - Assert.assertEquals("ME_WORKFLOW_SUMMARY", event.eid()); - } - - @Test(expected = JsonSyntaxException.class) - public void shouldThrowException() { - new Gson().fromJson(EventFixture.UNPARSABLE_START_EVENT, - new TypeToken>() { - }.getType()); - } - - @Test - public void shouldAddSkipFlags() { - String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}"; - Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class)); - event.markSkipped(); - Map flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class); - Assert.assertNotNull(flagData); - Assert.assertEquals(flagData.get("dd_dr_processed"), false); - Assert.assertEquals(flagData.get("dd_dr_checksum_present"), false); - } -} diff --git a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/task/EventsRouterTaskTest.java b/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/task/EventsRouterTaskTest.java deleted file mode 100644 index f10d5d4a45..0000000000 --- a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/task/EventsRouterTaskTest.java +++ /dev/null @@ -1,231 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.fiftyonred.mock_jedis.MockJedis; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.exceptions.JedisException; - -import java.util.ArrayList; -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -public class EventsRouterTaskTest { - - private static final String TELEMETRY_EVENTS_TOPIC = "events.telemetry"; - private static final String SUMMARY_EVENTS_TOPIC = "events.summary"; - private static final String FAILED_TOPIC = "telemetry.failed"; - private static final String MALFORMED_TOPIC = "telemetry.malformed"; - private static final String DUPLICATE_TOPIC = "telemetry.duplicate"; - - private MessageCollector collectorMock; - private TaskContext contextMock; - private MetricsRegistry metricsRegistry; - private Counter counter; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private Config configMock; - private EventsRouterTask eventsRouterTask; - private DeDupEngine deDupEngineMock; - private RedisConnect redisConnectMock; - private Jedis jedisMock = new MockJedis("duplicationtest"); - private int dupStoreId = 1; - - @Before - public void setUp() { - collectorMock = mock(MessageCollector.class); - contextMock = Mockito.mock(TaskContext.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = Mockito.mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - configMock = Mockito.mock(Config.class); - - redisConnectMock = mock(RedisConnect.class); - deDupEngineMock = mock(DeDupEngine.class); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - stub(configMock.get("router.events.telemetry.route.topic", TELEMETRY_EVENTS_TOPIC)).toReturn(TELEMETRY_EVENTS_TOPIC); - stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC); - stub(configMock.get("router.events.summary.route.topic", SUMMARY_EVENTS_TOPIC)).toReturn(SUMMARY_EVENTS_TOPIC); - stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC); - stub(configMock.get("output.duplicate.topic.name", DUPLICATE_TOPIC)).toReturn(DUPLICATE_TOPIC); - stub(configMock.getBoolean("dedup.enabled", true)).toReturn(true); - when(configMock.get("dedup.exclude.eids", "")).thenReturn("LOG"); - stub(configMock.getList("dedup.exclude.eids", new ArrayList<>())).toReturn(Arrays.asList("LOG","ERROR")); - - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - stub(envelopeMock.getOffset()).toReturn("2"); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "telemetry.denorm.valid", new Partition(1))); - - } - - @Test - public void shouldRouteSummaryEventsToSummaryTopic() throws Exception { - - stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY"); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.WORKFLOW_SUMMARY_EVENT); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUMMARY_EVENTS_TOPIC))); - } - - @Test - public void shouldRouteTelemetryEventsToTelemetryTopic() throws Exception { - - stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY"); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), TELEMETRY_EVENTS_TOPIC))); - - } - - @Test - public void shouldStoreChecksumForUniqueEvents() throws Exception { - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - DeDupEngine deDupEngine = new DeDupEngine(redisConnectMock, dupStoreId,60); - boolean isUnique = deDupEngine.isUniqueEvent("677009782"); - deDupEngine.getRedisConnection(); - deDupEngine.storeChecksum("678998676"); - - assertEquals(isUnique, true); - } - - @Test - public void shouldMarkEventFailureIfNullEid() throws Exception { - stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY"); - stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITH_NULL_EID); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - } - - @Test - public void shouldSendEventToFailedTopicIfEventIsNotParseable() throws Exception { - when(configMock.get("dedup.exclude.eids", "")).thenReturn(""); - stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSABLE_START_EVENT); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - @Test - public void shouldSendEventToMalformedTopicIfEventIsAnyRandomString() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - @Test - public void shouldSendEventToDuplicateTopic() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(false); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(deDupEngineMock, times(1)).isUniqueEvent(any()); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), DUPLICATE_TOPIC))); - } - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - return true; - } - }; - } - - @Test - public void shouldRouteLogEventsToTelemetryEventsTopic() throws Exception { - - stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY"); - stub(envelopeMock.getMessage()).toReturn(EventFixture.LOG_EVENT); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(deDupEngineMock, times(0)).isUniqueEvent(any()); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), TELEMETRY_EVENTS_TOPIC))); - } - - @Test - public void shouldRouteErrorEventsToTelemetryEventsTopic() throws Exception { - - stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY"); - stub(envelopeMock.getMessage()).toReturn(EventFixture.TELEMETRY_ERROR_EVENT); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(deDupEngineMock, times(0)).isUniqueEvent(any()); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), TELEMETRY_EVENTS_TOPIC))); - } - - @Test - public void shouldStampTimeStampforTraceEvent() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.TRACE_EVENT); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - } - - @Test - public void shouldSkipsSummaryEvents() throws Exception { - - stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY"); - stub(envelopeMock.getMessage()).toReturn(EventFixture.WORKFLOW_USAGE_EVENT); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - } - - @Test(expected = JedisException.class) - public void shouldCatchRedisException() throws Exception{ - - stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY"); - stub(envelopeMock.getMessage()).toReturn(EventFixture.WORKFLOW_USAGE_EVENT); - Jedis jedismock = mock(Jedis.class); - when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true); - when(deDupEngineMock.getRedisConnection()).thenReturn(jedismock); - eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock); - when(deDupEngineMock.isUniqueEvent(any())).thenThrow(JedisException.class); - eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - - } -} diff --git a/data-pipeline/pom.xml b/data-pipeline/pom.xml deleted file mode 100644 index 2da519fb5f..0000000000 --- a/data-pipeline/pom.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - 4.0.0 - - - 3.0.0 - - - org.ekstep.ecosystem - jobs - 0.0.1 - pom - Ekstep Samza - - EkStep Samza for stream processing within Ecosystem - - https://ekstep.org/ - - - ep-core - distribution - ep-telemetry-reader - telemetry-validator - telemetry-extractor - de-duplication - derived-de-duplication - telemetry-router - telemetry-redacter - telemetry-location-updater - de-normalization - druid-events-validator - events-router - device-profile-updater - assessment-aggregator - user-cache-updater - content-cache-updater - share-events-flattener - - - - - UTF-8 - 0.14.1 - 2.11 - 1.1.0 - 1.9.13 - /tmp - test-container - - - - - my-local-repo - file://${user.home}/.m2/repository - - - apache-releases - https://repository.apache.org/content/groups/public - - - scala-tools.org - Scala-tools Maven2 Repository - https://oss.sonatype.org/content/groups/scala-tools - - - - - - scala-tools.org - Scala-tools Maven2 Repository - http://scala-tools.org/repo-releases - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - - - - - - - diff --git a/data-pipeline/share-events-flattener/pom.xml b/data-pipeline/share-events-flattener/pom.xml deleted file mode 100644 index 89bd5385a0..0000000000 --- a/data-pipeline/share-events-flattener/pom.xml +++ /dev/null @@ -1,172 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - org.ekstep.ecosystem - jobs - 0.0.1 - - org.ekstep.ecosystem.jobs - share-events-flattener - 0.0.1 - jar - Share Events Flattener - - Events Flattener job to flatten the share events. - - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - com.github.java-json-tools - json-schema-validator - 2.2.8 - - - com.google.guava - guava - 18.0 - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - diff --git a/data-pipeline/share-events-flattener/src/main/assembly/src.xml b/data-pipeline/share-events-flattener/src/main/assembly/src.xml deleted file mode 100644 index 8335b290c5..0000000000 --- a/data-pipeline/share-events-flattener/src/main/assembly/src.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/share-events-flattener.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:share-events-flattener - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/share-events-flattener/src/main/config/share-events-flattener.properties b/data-pipeline/share-events-flattener/src/main/config/share-events-flattener.properties deleted file mode 100644 index 4a67cb58d9..0000000000 --- a/data-pipeline/share-events-flattener/src/main/config/share-events-flattener.properties +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.ShareEventsFlattener - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__events_flattener_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.EventsFlattenerTask -task.inputs=kafka.__env__.telemetry.share -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -output.success.topic.name=__env__.telemetry.sink -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,error-message-count diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index 4ea73764e4..0000000000 --- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.events.domain.Events; - -import java.util.HashMap; -import java.util.Map; - -public class Event extends Events { - - public Event(Map map) { - super(map); - } - - public void markSuccess() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.ef_processed", true); - telemetry.add("type", "events"); - } - - public void updateEventObjectKey(String objId, String type, String version, String rollupLevel1) { - telemetry.addFieldIfAbsent("object", new HashMap()); - telemetry.add("object.id", objId); - telemetry.add("object.type", type); - telemetry.add("object.version", version); - telemetry.add("object.rollup.l1", rollupLevel1); - } - - public void updatedEventEdata(String edataType, Long size) { - telemetry.add("edata.type", edataType); - telemetry.add("edata.size", size); - } - - public void updateMid(String mid){ - telemetry.add("mid", mid); - } - - public void removeItems() { - telemetry.add("edata.items", null); - } - - public void renameEventIdTo(String eid) { - telemetry.add("eid", eid); - } - -} diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/service/EventsFlattenerService.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/service/EventsFlattenerService.java deleted file mode 100644 index 474e94ea08..0000000000 --- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/service/EventsFlattenerService.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.EventsFlattenerConfig; -import org.ekstep.ep.samza.task.EventsFlattenerSink; -import org.ekstep.ep.samza.task.EventsFlattenerSource; - -import java.lang.reflect.Type; -import java.math.BigDecimal; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; - -public class EventsFlattenerService { - - private static Logger LOGGER = new Logger(EventsFlattenerService.class); - private final EventsFlattenerConfig config; - private static String IMPORT_KEY = "import"; - private static String DOWNLOAD_KEY = "download"; - private static String FLATTEN_EVENT_NAME = "SHARE_ITEM"; - - /** - * Constructor of the EventFlattenService - * - * @param config - Configurations of the EventsFlatten Samza job. - */ - public EventsFlattenerService(EventsFlattenerConfig config) { - this.config = config; - } - - /** - * Listening to "telemetry.share" topic, Assuming only share events should produce - * From this "telemetry.share" topic. - * - * @param source - Producer - "telemetry.share" topic - * @param sink - Consumer - "telemetry.sink" - */ - public void process(EventsFlattenerSource source, EventsFlattenerSink sink) { - Event event = source.getEvent(); - // Flattening the "SHARE" Event to Multiple "SHARE_ITEM" Events - this.toFlatten(event, getClonedEventObject(event), sink); - // Adding Original "SHARE" Events to success topic - sink.toSuccessTopic(event); - } - - /** - * Method to flatten the SHARE EVENT Object to multiple share event. - * Events flattening constraints - *

- * ================= Constraints=============================== - * 1. If the Item list object as params.transfers = 0 then edata.type should be "download" else do not modify the edata.type value. - * 2. If the Item list object as params.transfers > 0 then edata.type should be "import" else do not modify the edta.type value. - * 3. If the Item list object as params.transfers = null then do not update the edata.type (Keep the orginal value ie., dir, type) - * 4. If the share event has object then move the object data to rollup l1, share event item.id, item.typ and item.ver should be in object properties - * =============================================================== - * - * @param orginalEvent - SHARE Event object. - * @param sink - Object to push the event to kafka sink - */ - private void toFlatten(Event orginalEvent, Event clonedEvent, EventsFlattenerSink sink) { - String objectId = null; - String eDataType = clonedEvent.edataType(); - Gson gson = new Gson(); - Type type = new TypeToken>>() { - }.getType(); - - List> items = clonedEvent.edataItems(); - if (clonedEvent.objectFieldsPresent()) { - objectId = clonedEvent.objectID(); - } - String itemId = null; - String itemType = null; - String itemVerion = null; - - for (Map item : items) { - if (item.get("id") != null) { - itemId = item.get("id").toString(); - } - if (item.get("type") != null) { - itemType = item.get("type").toString(); - } - if (item.get("ver") != null) { - itemVerion = item.get("ver").toString(); - } - clonedEvent.updateEventObjectKey(itemId, itemType, itemVerion, objectId); - Object itemParams = item.get("params"); - if (itemParams != null) { - List> param = gson.fromJson(gson.toJson(itemParams), type); - String paramTransfer = this.getValue(param, "transfers"); - Long paramSize = null; - if (this.getValue(param, "size") != null && !this.getValue(param, "size").isEmpty()) { - paramSize = new BigDecimal(Objects.requireNonNull(this.getValue(param, "size"))).longValue(); - } - if (paramTransfer != null) { - if (Double.parseDouble(paramTransfer) == 0) { - eDataType = DOWNLOAD_KEY; - } - if (Double.parseDouble(paramTransfer) > 0) { - eDataType = IMPORT_KEY; - } - clonedEvent.updatedEventEdata(eDataType, paramSize); - } else { - clonedEvent.updatedEventEdata(orginalEvent.edataType(), paramSize); - } - } else { - clonedEvent.updatedEventEdata(orginalEvent.edataType(), null); - } - clonedEvent.renameEventIdTo(FLATTEN_EVENT_NAME); - clonedEvent.removeItems(); - // Adding "SHARE_ITEM" Events to success topic - clonedEvent.updateMid(this.getMid()); - sink.toSuccessTopic(clonedEvent); - } - } - - /** - * Method to clone the telemetry event object data. - * - * @param event Event object - * @return Duplicated Event object. - */ - private Event getClonedEventObject(Event event) { - String message = event.getJson(); - return new Event((Map) new Gson().fromJson(message, Map.class)); - } - - - private String getValue(final List> paramsList, final String name) { - for (Map p : paramsList) { - if (p.get(name) != null) { - return p.get(name); - } - } - return null; - } - - private String getMid() { - return "SHARE_ITEM:" + UUID.randomUUID(); - } - -} diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerConfig.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerConfig.java deleted file mode 100644 index 0573e6522d..0000000000 --- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerConfig.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; - -public class EventsFlattenerConfig { - - private final String JOB_NAME = "SHARE_EVENTS_FLATTENER"; - - private String successEventTopic; - - public EventsFlattenerConfig(Config config) { - successEventTopic = config.get("output.success.topic.name", "telemetry.sink"); - } - - public String getSuccessTopic() { - return successEventTopic; - } - - public String jobName() { - return JOB_NAME; - } - -} \ No newline at end of file diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSink.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSink.java deleted file mode 100644 index 68135a44b5..0000000000 --- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSink.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - -public class EventsFlattenerSink extends BaseSink { - - private EventsFlattenerConfig config; - - public EventsFlattenerSink(MessageCollector collector, JobMetrics metrics, EventsFlattenerConfig config) { - super(collector, metrics); - this.config = config; - } - - public void toSuccessTopic(Event event) { - event.markSuccess(); - toTopic(config.getSuccessTopic(), event.did(), event.getJson()); - metrics.incSuccessCounter(); - } - -} diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSource.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSource.java deleted file mode 100644 index 61645f616a..0000000000 --- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSource.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.gson.Gson; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.ekstep.ep.samza.domain.Event; - -import java.util.Map; - -public class EventsFlattenerSource { - - private IncomingMessageEnvelope envelope; - - public EventsFlattenerSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - return new Event(getMap()); - } - - @SuppressWarnings("unchecked") - private Map getMap() { - String message = (String) envelope.getMessage(); - return (Map) new Gson().fromJson(message, Map.class); - } - -} diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerTask.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerTask.java deleted file mode 100644 index b52d767fb0..0000000000 --- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerTask.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.EventsFlattenerService; - -public class EventsFlattenerTask extends BaseSamzaTask { - - private EventsFlattenerConfig config; - private JobMetrics metrics; - private EventsFlattenerService service; - - public EventsFlattenerTask(Config config, TaskContext context) throws Exception { - init(config, context); - } - - public EventsFlattenerTask() { - } - - @Override - public void init(Config config, TaskContext context) { - this.config = new EventsFlattenerConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - service = new EventsFlattenerService(this.config); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) { - EventsFlattenerSink sink = new EventsFlattenerSink(collector, metrics, config); - EventsFlattenerSource source = new EventsFlattenerSource(envelope); - service.process(source, sink); - } - -} diff --git a/data-pipeline/share-events-flattener/src/main/resources/log4j.xml b/data-pipeline/share-events-flattener/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/share-events-flattener/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index e80f3a80f3..0000000000 --- a/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,451 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; - -import java.util.Map; - -public class EventFixture { - - public static final String VALID_SHARE_EVENT = "{\n" + - " \"ver\": \"3.0\",\n" + - " \"eid\": \"SHARE\",\n" + - " \"ets\": 1577278681178,\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"\n" + - " },\n" + - " \"context\": {\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\n" + - " \"type\": \"API\"\n" + - " },\n" + - " {\n" + - " \"id\": \"SearchResult\",\n" + - " \"type\": \"Section\"\n" + - " }\n" + - " ],\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.app\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"ver\": \"2.3.162\"\n" + - " },\n" + - " \"env\": \"app\",\n" + - " \"sid\": \"82e41d87-e33f-4269-aeae-d56394985599\",\n" + - " \"did\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"dir\": \"In\",\n" + - " \"type\": \"File\",\n" + - " \"items\": [\n" + - " {\n" + - " \"origin\": {\n" + - " \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" + - " \"type\": \"Device\"\n" + - " },\n" + - " \"id\": \"do_312785709424099328114191\",\n" + - " \"type\": \"CONTENT\",\n" + - " \"ver\": \"1\",\n" + - " \"params\": [\n" + - " {\n" + - " \"transfers\": 0,\n" + - " \"size\": 21084308\n" + - " }\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"origin\": {\n" + - " \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" + - " \"type\": \"Device\"\n" + - " },\n" + - " \"id\": \"do_31277435209002188818711\",\n" + - " \"type\": \"CONTENT\",\n" + - " \"ver\": \"18\",\n" + - " \"params\": [\n" + - " {\n" + - " \"transfers\": 12,\n" + - " \"size\": \"123\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"origin\": {\n" + - " \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" + - " \"type\": \"Device\"\n" + - " },\n" + - " \"id\": \"do_31278794857559654411554\",\n" + - " \"type\": \"TextBook\",\n" + - " \"ver\": \"1\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312528116260749312248818\",\n" + - " \"type\": \"TextBook\",\n" + - " \"version\": \"10\",\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"mid\": \"02ba33e5-15fe-4ec5-b32.1084308E760-3d03429fae84\",\n" + - " \"syncts\": 1577278682630,\n" + - " \"@timestamp\": \"2019-12-25T12:58:02.630Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - public static final String VALID_SHARE_EVENT_WHEN_EMPTY_SIZE = "{\n" + - " \"eid\": \"SHARE\",\n" + - " \"ets\": 1578979395120,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"SHARE:6a68490ac9c8ffcb8f8b8e1e6da1f09c\",\n" + - " \"actor\": {\n" + - " \"id\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"01231711180382208027\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.desktop\",\n" + - " \"ver\": \"1.0.3\",\n" + - " \"pid\": \"desktop.app\"\n" + - " },\n" + - " \"env\": \"Content\",\n" + - " \"sid\": \"d17137fd-224a-49bd-834c-0e9d7a21ca18\",\n" + - " \"did\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" + - " \"cdata\": [],\n" + - " \"rollup\": {\n" + - " \"l1\": \"01231711180382208027\"\n" + - " }\n" + - " },\n" + - " \"object\": {},\n" + - " \"tags\": [\n" + - " \"01231711180382208027\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"dir\": \"Out\",\n" + - " \"type\": \"File\",\n" + - " \"items\": [\n" + - " {\n" + - " \"type\": \"TextBook\",\n" + - " \"ver\": \"1\",\n" + - " \"params\": [\n" + - " {\n" + - " \"transfers\": \"53\"\n" + - " },\n" + - " {\n" + - " \"size\": \"\"\n" + - " }\n" + - " ],\n" + - " \"origin\": {\n" + - " \"id\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" + - " \"type\": \"Device\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"syncts\": 1578979437523,\n" + - " \"@timestamp\": \"2020-01-14T05:23:57.523Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - - public static final String VALID_SHARE_EVENT_WITHOU_SIZE = "{\n" + - " \"eid\": \"SHARE\",\n" + - " \"ets\": 1578979334609,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"SHARE:336bd87993a20d2dc14a56d2ff197111\",\n" + - " \"actor\": {\n" + - " \"id\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"01231711180382208027\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.desktop\",\n" + - " \"ver\": \"1.0.3\",\n" + - " \"pid\": \"desktop.app\"\n" + - " },\n" + - " \"env\": \"Content\",\n" + - " \"sid\": \"d17137fd-224a-49bd-834c-0e9d7a21ca18\",\n" + - " \"did\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" + - " \"cdata\": [],\n" + - " \"rollup\": {\n" + - " \"l1\": \"01231711180382208027\"\n" + - " }\n" + - " },\n" + - " \"object\": {},\n" + - " \"tags\": [\n" + - " \"01231711180382208027\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"dir\": \"Out\",\n" + - " \"type\": \"File\",\n" + - " \"items\": [\n" + - " {\n" + - " \"type\": \"TextBook\",\n" + - " \"ver\": \"2\",\n" + - " \"params\": [\n" + - " {\n" + - " \"transfers\": \"2\"\n" + - " },\n" + - " {\n" + - " \"size\": \"66438\"\n" + - " }\n" + - " ],\n" + - " \"origin\": {\n" + - " \"id\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" + - " \"type\": \"Device\"\n" + - " }\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"syncts\": 1578979371914,\n" + - " \"@timestamp\": \"2020-01-14T05:22:51.914Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String VALID_START_EVENT = "{\n" + - " \"ver\": \"3.0\",\n" + - " \"eid\": \"START\",\n" + - " \"ets\": 1577278681178,\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"\n" + - " },\n" + - " \"context\": {\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\n" + - " \"type\": \"API\"\n" + - " },\n" + - " {\n" + - " \"id\": \"SearchResult\",\n" + - " \"type\": \"Section\"\n" + - " }\n" + - " ],\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.app\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"ver\": \"2.3.162\"\n" + - " },\n" + - " \"env\": \"app\",\n" + - " \"sid\": \"82e41d87-e33f-4269-aeae-d56394985599\",\n" + - " \"did\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"dir\": \"In\",\n" + - " \"type\": \"File\",\n" + - " \"items\": [\n" + - " {\n" + - " \"origin\": {\n" + - " \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" + - " \"type\": \"Device\"\n" + - " },\n" + - " \"id\": \"do_312785709424099328114191\",\n" + - " \"type\": \"CONTENT\",\n" + - " \"ver\": \"1\",\n" + - " \"params\": [\n" + - " {\n" + - " \"transfers\": 0,\n" + - " \"size\": 21084308\n" + - " }\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"origin\": {\n" + - " \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" + - " \"type\": \"Device\"\n" + - " },\n" + - " \"id\": \"do_31277435209002188818711\",\n" + - " \"type\": \"CONTENT\",\n" + - " \"ver\": \"18\",\n" + - " \"params\": [\n" + - " {\n" + - " \"transfers\": 12,\n" + - " \"size\": \"123\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"origin\": {\n" + - " \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" + - " \"type\": \"Device\"\n" + - " },\n" + - " \"id\": \"do_31278794857559654411554\",\n" + - " \"type\": \"TextBook\",\n" + - " \"ver\": \"1\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312528116260749312248818\",\n" + - " \"type\": \"TextBook\",\n" + - " \"version\": \"10\",\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"mid\": \"02ba33e5-15fe-4ec5-b360-3d03429fae84\",\n" + - " \"syncts\": 1577278682630,\n" + - " \"@timestamp\": \"2019-12-25T12:58:02.630Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - - public static final String INVALID_SHARE_EVENT = "{\n" + - " \"ver\": \"3.0\",\n" + - " \"eid\": \"SHARE\",\n" + - " \"ets\": 1577278681178,\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"\n" + - " },\n" + - " \"context\": {\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\n" + - " \"type\": \"API\"\n" + - " },\n" + - " {\n" + - " \"id\": \"SearchResult\",\n" + - " \"type\": \"Section\"\n" + - " }\n" + - " ],\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.app\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"ver\": \"2.3.162\"\n" + - " },\n" + - " \"env\": \"app\",\n" + - " \"sid\": \"82e41d87-e33f-4269-aeae-d56394985599\",\n" + - " \"did\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"dir\": \"In\",\n" + - " \"type\": \"File\",\n" + - " \"items\": \"test\"\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312528116260749312248818\",\n" + - " \"type\": \"TextBook\",\n" + - " \"version\": \"10\",\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"mid\": \"02ba33e5-15fe-4ec5-b360-3d03429fae84\",\n" + - " \"syncts\": 1577278682630,\n" + - " \"@timestamp\": \"2019-12-25T12:58:02.630Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String INVALID_EVENT = "{\n" + - " \"ver\": \"3.0\",\n" + - " \"eid\": \"SHARE\",\n" + - " \"ets\": 1577278681178,\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"\n" + - " },\n" + - " \"context\": {\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\n" + - " \"type\": \"API\"\n" + - " },\n" + - " {\n" + - " \"id\": \"SearchResult\",\n" + - " \"type\": \"Section\"\n" + - " }\n" + - " ],\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.app\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"ver\": \"2.3.162\"\n" + - " },\n" + - " \"env\": \"app\",\n" + - " \"sid\": \"82e41d87-e33f-4269-aeae-d56394985599\",\n" + - " \"did\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"dir\": \"In\",\n" + - " \"type\": \"File\",\n" + - " \"items\": [\n" + - " {\n" + - " \"origin\": {\n" + - " \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" + - " \"type\": \"Device\"\n" + - " },\n" + - " \"id\": \"do_312785709424099328114191\",\n" + - " \"type\": \"CONTENT\",\n" + - " \"ver\": \"1\",\n" + - " \"params\": [\n" + - " {\n" + - " \"size\": 21084308\n" + - " }\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"origin\": {\n" + - " \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" + - " \"type\": \"Device\"\n" + - " },\n" + - " \"id\": \"do_31277435209002188818711\",\n" + - " \"type\": \"CONTENT\",\n" + - " \"ver\": \"18\",\n" + - " \"params\": [\n" + - " {\n" + - " \"transfers\": 12,\n" + - " \"size\": \"123\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"origin\": {\n" + - " \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" + - " \"type\": \"Device\"\n" + - " },\n" + - " \"id\": \"do_31278794857559654411554\",\n" + - " \"type\": \"TextBook\",\n" + - " \"ver\": \"1\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312528116260749312248818\",\n" + - " \"type\": \"TextBook\",\n" + - " \"version\": \"10\",\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"mid\": \"02ba33e5-15fe-4ec5-b360-3d03429fae84\",\n" + - " \"syncts\": 1577278682630,\n" + - " \"@timestamp\": \"2019-12-25T12:58:02.630Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static Map getMap(String message) { - return (Map) new Gson().fromJson(message, Map.class); - } - -} diff --git a/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/task/EventsFlattenerTaskTest.java b/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/task/EventsFlattenerTaskTest.java deleted file mode 100644 index 6fdd6d8ae4..0000000000 --- a/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/task/EventsFlattenerTaskTest.java +++ /dev/null @@ -1,165 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.gson.Gson; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Matchers; -import org.mockito.Mockito; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static junit.framework.Assert.assertNotNull; -import static junit.framework.Assert.assertNull; -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.*; - -public class EventsFlattenerTaskTest { - - private static final String TELEMETRY_EVENTS_TOPIC = "telemetry.sink"; - private static final String FAILED_TOPIC = "telemetry.failed"; - - private MessageCollector collectorMock; - private TaskContext contextMock; - private MetricsRegistry metricsRegistry; - private Counter counter; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private Config configMock; - private EventsFlattenerTask eventsFlattenerTask; - - @Before - public void setUp() { - collectorMock = mock(MessageCollector.class); - contextMock = Mockito.mock(TaskContext.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = Mockito.mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - configMock = Mockito.mock(Config.class); - - stub(configMock.get("output.success.topic.name", TELEMETRY_EVENTS_TOPIC)).toReturn(TELEMETRY_EVENTS_TOPIC); - stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC); - - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - } - - /** - * When share event having multiple edata items - */ - @Test - public void shouldFlattenTheShareEvent() { - try { - eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_SHARE_EVENT_WITHOU_SIZE); - eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock); - Mockito.verify(collectorMock, times(2)).send(Matchers.argThat(validateEventObject(Arrays.asList("File", "import", "download"), true))); - } catch (Exception e) { - System.out.println(e.getMessage()); - } - } - - @Test - public void shouldFlattenTheShareEventWhenParamsHavingMultipleObject() { - try { - eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_SHARE_EVENT); - eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock); - Mockito.verify(collectorMock, times(4)).send(Matchers.argThat(validateEventObject(Arrays.asList("File", "import", "download"), true))); - } catch (Exception e) { - e.printStackTrace(); - System.out.println(e.getMessage()); - } - } - - @Test - public void shouldFlattenTheEventWhenTransferIsNul() { - try { - eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_SHARE_EVENT_WITHOU_SIZE); - eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock); - Mockito.verify(collectorMock, times(2)).send(Matchers.argThat(validateEventObject(Arrays.asList("File", "import", "download"), true))); - } catch (Exception e) { - System.out.println(e.getMessage()); - } - } - - @Test - public void shouldFlattenTheEventWhenSizeIsEmpty() { - try { - eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_SHARE_EVENT_WHEN_EMPTY_SIZE); - eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock); - Mockito.verify(collectorMock, times(2)).send(Matchers.argThat(validateEventObject(Arrays.asList("File", "import", "download"), true))); - } catch (Exception e) { - System.out.println(e.getMessage()); - } - } - - /** - * When Event is other than "SHARE" Then it should route to success topic - */ - @Test - public void shouldRouteToSuccessTopic() { - try { - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_START_EVENT); - eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock); - eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock, times(4)).send(argThat(validateOutputTopic(envelopeMock.getMessage(), TELEMETRY_EVENTS_TOPIC))); - } catch (Exception e) { - System.out.println(e.getMessage()); - } - } - - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertNotNull(outgoingMessageEnvelope.getMessage()); - return true; - } - }; - } - - public ArgumentMatcher validateEventObject(List edataType, Boolean ef_processed) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - Map shareEvent; - Map shareEventEdata; - OutgoingMessageEnvelope flattenEvent = (OutgoingMessageEnvelope) o; - String message = (String) flattenEvent.getMessage(); - System.out.println(message); - shareEvent = new Gson().fromJson(message, Map.class); - assertEquals(ef_processed, new Gson().fromJson(shareEvent.get("flags").toString(), Map.class).get("ef_processed")); - if (shareEvent.get("eid").equals("SHARE_ITEM")) { - shareEventEdata = new Gson().fromJson(new Gson().toJson(shareEvent.get("edata")), Map.class); - assertEquals(true, edataType.contains(shareEventEdata.get("type"))); - assertNull(shareEventEdata.get("items")); - } else { - shareEventEdata = new Gson().fromJson(new Gson().toJson(shareEvent.get("edata")), Map.class); - assertNotNull(shareEventEdata.get("items")); - } - return true; - } - }; - } -} diff --git a/data-pipeline/telemetry-extractor/pom.xml b/data-pipeline/telemetry-extractor/pom.xml deleted file mode 100644 index a5a7a6e83e..0000000000 --- a/data-pipeline/telemetry-extractor/pom.xml +++ /dev/null @@ -1,175 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - org.ekstep.ecosystem.jobs - telemetry-extractor - 0.1.5 - jar - TelemetryExtractor - - Stamp @timestmap in telemetry - - - - - org.apache.samza - samza-api - ${samza.version} - - - - org.scala-lang - scala-reflect - 2.11.0 - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - com.google.code.gson - gson - 2.8.0 - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - - diff --git a/data-pipeline/telemetry-extractor/src/main/assembly/src.xml b/data-pipeline/telemetry-extractor/src/main/assembly/src.xml deleted file mode 100644 index 6e0aa8d169..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/assembly/src.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/telemetry-extractor.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:telemetry-extractor - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/telemetry-extractor/src/main/config/telemetry-extractor.properties b/data-pipeline/telemetry-extractor/src/main/config/telemetry-extractor.properties deleted file mode 100644 index 03b4ca2a0e..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/config/telemetry-extractor.properties +++ /dev/null @@ -1,116 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -####Environment - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.TelemetryExtractor - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__telemetry_extractor_yarn_container_count__ -yarn.container.memory.mb=__telemetry_extractor_container_memory_mb__ - -# Task -task.class=org.ekstep.ep.samza.task.TelemetryExtractorTask -task.inputs=kafka.__env__.telemetry.ingest -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -# Normally, this would be 3, but we have only one broker. -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot,jmx -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics -metrics.reporter.jmx.class=org.apache.samza.metrics.reporter.JmxReporterFactory - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.msg.serde=string -systems.kafka.samza.key.serde=string -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.consumer.fetch.message.max.bytes=__telemetry_extractor_consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__telemetry_extractor_consumer_fetch_max_bytes__ -systems.kafka.samza.fetch.threshold=__telemetry_extractor_messages_fetch_threshold__ - - -## Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -output.success.topic.name=__env__.telemetry.raw -output.assess.topic.name=__env__.telemetry.assess.redact -output.error.topic.name=__env__.telemetry.extractor.failed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,skipped-message-count,error-message-count,batch-success-count,batch-error-count,duplicate-event-count,assess-route-success-count -default.channel=__default_channel__ - -# redis -#redis.host=localhost -redis.host=__redis_host__ -#redis.port=6379 -redis.port=__redis_port__ -redis.connection.max=2 -redis.connection.idle.max=2 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.database.duplicationstore.id=1 -redis.database.key.expiry.seconds=432000 -raw.individual.event.maxsize=996148 - -####Local -## Job -#job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -#job.name=TelemetryExtractor - -## YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz - -## Task -#task.inputs=kafka.telemetry.ingest - -## Metrics -#metrics.reporter.snapshot.stream=kafka.metrics - -##Key-Value store -#stores.telemetry-extractor.changelog=kafka.telemetry-extractor-changelog - -## Systems -#systems.kafka.consumer.zookeeper.connect=localhost:2181/ -#systems.kafka.producer.bootstrap.servers=localhost:9092 - -## Job Coordinator -#output.success.topic.name=telemetry.raw -#output.error.topic.name=telemetry.extractor.failed -#output.metrics.topic.name=pipeline_metrics \ No newline at end of file diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Actor.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Actor.java deleted file mode 100755 index 5aacd0308b..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Actor.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import java.util.Map; - -public class Actor { - private String id = ""; - private String type = "telemetry-sync"; - - public Actor() { - - } - - public Actor(Map eventSpec) { - this.id = (String) eventSpec.get("id"); - } - -} diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/CData.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/CData.java deleted file mode 100755 index 85d6e0def1..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/CData.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import java.util.Map; - -public class CData { - private String type; - private String id; - - public CData(Map cdata) { - this.id = (String) cdata.get("id"); - this.type = (String) cdata.get("type"); - } - - public CData(String type, String id) { - this.type = type; - this.id = id; - } - - public String getType() { - return type; - } - - public String getId() { - return id; - } -} diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Context.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Context.java deleted file mode 100755 index 3a65b488e2..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Context.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.annotations.SerializedName; - -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.task.TelemetryExtractorConfig; - -import java.util.*; - -public class Context { - private String channel = ""; - private String env = ""; - private String sid = ""; - private String did = ""; - static Logger LOGGER = new Logger(Context.class); - - @SerializedName("pdata") - private Map pData = new HashMap(); - - @SerializedName("cdata") - private ArrayList cData = new ArrayList<>(); - - - public Context(Map eventSpec, String defaultChannel) { - - this.channel = defaultChannel; - this.pData.put("id", "pipeline"); - this.pData.put("pid", ""); - this.pData.put("ver", ""); - - try { - List> events = (List>) eventSpec.get("events"); - Map event = events.get(0); - Map eventContext = (Map) event.get("context"); - env = "telemetry-sync"; - did = (String) eventContext.get("did"); - String channel = (String) eventContext.get("channel"); - if (channel != null && !"".equals(channel.trim())) { - this.channel = channel; - } - Map pdata = (Map) eventContext.get("pdata"); - if (pdata != null && pdata.containsKey("id")) { - this.pData = pdata; - } - - } catch (Exception e) { - LOGGER.info("", "Failed to initialize context: " + e.getMessage()); - } - - } - -} diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/TObject.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/TObject.java deleted file mode 100755 index af26bbafa1..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/TObject.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.annotations.SerializedName; - -import java.util.Map; - -public class TObject { - private String id; - private String type; - private String ver; - - @SerializedName("subtype") - private String subType; - - public TObject(Map eventSpec) { - - this.id = (String) eventSpec.get("mid"); - this.type = "event"; - - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - -} diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Telemetry.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Telemetry.java deleted file mode 100755 index e04d9a0709..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Telemetry.java +++ /dev/null @@ -1,123 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; -import org.ekstep.ep.samza.core.Logger; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class Telemetry { - - static Logger LOGGER = new Logger(Telemetry.class); - - private final String ver = "3.0"; - private String eid; - private long ets; - private String mid; - private Actor actor; - private Context context; - private TObject object; - private HashMap edata; - private List tags = new ArrayList<>(); - private HashMap metadata; - private long syncts; - private String syncTimestamp; - - public Telemetry() { - - } - - public Telemetry(Map batchEvent, long syncts, String syncTimestamp, String defaultChannel) { - - try { - @SuppressWarnings("unchecked") - List> events = (List>) batchEvent.get("events"); - - // TODO - Handle NPE if the batchEvent doesn't have 'ets'. Default 'ets' to System.currentTimeInMillis() - this.ets = ((Number) batchEvent.get("ets")).longValue(); - this.syncts = syncts; - this.syncTimestamp = syncTimestamp; - - String mid = (String) batchEvent.get("mid"); - - int eventCount = events.size(); - String status = (String) batchEvent.get("sync_status"); - String ver = (String) batchEvent.get("ver"); - if (null == status) { - status = "SUCCESS"; - } - String consumerId = (String) batchEvent.get("consumer_id"); - if (null == consumerId) { - consumerId = ""; - } - - HashMap edata = new HashMap(); - edata.put("type", "telemetry_audit"); - edata.put("level", "INFO"); - edata.put("message", "telemetry sync"); - edata.put("pageid", "data-pipeline"); - - List> params = new ArrayList>(); - Map param = new HashMap<>(); - param.put("sync_status", status); - param.put("consumer_id", consumerId); - param.put("events_count", eventCount); - param.put("ver", ver); - params.add(param); - edata.put("params", params); - - this.eid = "LOG"; - this.edata = edata; - this.mid = computeMid(this.eid, mid); - this.metadata = new HashMap<>(); - this.actor = new Actor(batchEvent); - this.context = new Context(batchEvent, defaultChannel); - this.object = new TObject(batchEvent); - } catch (Exception e) { - - LOGGER.info("", "Failed to initialize telemetry spec data: " + e.getMessage()); - } - - } - - public String computeMid(String eid, String mid) { - // Because v2->v3 is one to many, mids have to be changed - // We just prefix the LOG EID with telemetry spec mid - return String.format("%s:%s", eid, mid); - } - - - public TObject getObject() { - return object; - } - - public void setObject(TObject object) { - this.object = object; - } - - - public Map toMap() { - Map v3map = new HashMap<>(); - v3map.put("eid", this.eid); - v3map.put("ets", this.ets); - v3map.put("ver", this.ver); - v3map.put("mid", this.mid); - v3map.put("actor", this.actor); - v3map.put("context", this.context); - v3map.put("object", this.object); - v3map.put("metadata", this.metadata); - v3map.put("edata", this.edata); - v3map.put("tags", this.tags); - v3map.put("syncts", this.syncts); - v3map.put("@timestamp", this.syncTimestamp); - return v3map; - } - - public String toJson() { - Map map = toMap(); - return new Gson().toJson(map); - } - -} diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/service/TelemetryExtractorService.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/service/TelemetryExtractorService.java deleted file mode 100644 index 4dffe171d1..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/service/TelemetryExtractorService.java +++ /dev/null @@ -1,149 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.Gson; - -import redis.clients.jedis.exceptions.JedisException; - -import org.apache.commons.lang.StringUtils; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Telemetry; -import org.ekstep.ep.samza.task.TelemetryExtractorConfig; -import org.ekstep.ep.samza.task.TelemetryExtractorSink; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class TelemetryExtractorService { - - static Logger LOGGER = new Logger(TelemetryExtractorService.class); - - private DateTimeFormatter df = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZoneUTC(); - private JobMetrics metrics; - private DeDupEngine deDupEngine; - private String defaultChannel = ""; - private int rawIndividualEventMaxSize; - private List assessEvents = Arrays.asList("ASSESS", "RESPONSE"); - - public TelemetryExtractorService(TelemetryExtractorConfig config, JobMetrics metrics, DeDupEngine deDupEngine) { - this.metrics = metrics; - this.deDupEngine = deDupEngine; - this.defaultChannel = config.defaultChannel(); - this.rawIndividualEventMaxSize = config.rawIndividualEventMaxSize(); - } - - @SuppressWarnings("unchecked") - public void process(String message, TelemetryExtractorSink sink) { - - try { - Map batchEvent = (Map) new Gson().fromJson(message, Map.class); - long syncts = getSyncTS(batchEvent); - String syncTimestamp = df.print(syncts); - - if (batchEvent.containsKey("params") && null != batchEvent.get("params")) { - String msgid = ""; - Map params = (Map) batchEvent.get("params"); - if (params.containsKey("msgid") && null != params.get("msgid")) { - msgid = params.get("msgid").toString(); - if (!deDupEngine.isUniqueEvent(msgid)) { - LOGGER.info("", String.format("msgid: %s: DUPLICATE EVENT", msgid)); - sink.toDuplicateTopic(addDuplicateFlag(batchEvent)); - return; - } - deDupEngine.storeChecksum(msgid); - } - - } - - List> events = (List>) batchEvent.get("events"); - - for (Map event : events) { - String json = ""; - try { - event.put("syncts", syncts); - event.put("@timestamp", syncTimestamp); - Map context = (Map) event.get("context"); - String channel = (String) context.get("channel"); - String eid = (String) event.get("eid"); - if (StringUtils.isEmpty(channel)) { - event.put("context", context); - } - json = new Gson().toJson(event); - int eventSizeInBytes = json.getBytes("UTF-8").length; - if (eventSizeInBytes > rawIndividualEventMaxSize) { - LOGGER.info("", - String.format("Event with mid %s of size %d bytes is greater than %d. " + "Sending to error topic", - event.get("mid"), eventSizeInBytes, rawIndividualEventMaxSize)); - sink.toErrorTopic(json); - } else { - if (assessEvents.contains(eid)) { - sink.toAssessTopic(json); - } else { - sink.toSuccessTopic(json); - } - } - } catch (Throwable t) { - LOGGER.info("", "Failed to send extracted event to success topic: " + t.getMessage()); - sink.toErrorTopic(json); - } - } - metrics.incBatchSuccessCounter(); - generateAuditEvent(batchEvent, syncts, syncTimestamp, sink, defaultChannel); - } catch (JedisException e) { - e.printStackTrace(); - LOGGER.error(null, "Exception when retrieving data from redis: ", e); - deDupEngine.getRedisConnection().close(); - throw e; - } catch (Exception ex) { - ex.printStackTrace(); - LOGGER.error("", "Failed to extract the event batch: ", ex); - sink.sinkBatchErrorEvents(message); - } - - } - - private long getSyncTS(Map batchEvent) { - - if (batchEvent.containsKey("syncts")) { - Object obj = batchEvent.get("syncts"); - if (obj instanceof Number) { - return ((Number) obj).longValue(); - } - } - - return System.currentTimeMillis(); - } - - /** - * Create LOG event to audit telemetry sync - * - * @param eventSpec - * @param syncts - * @param syncTimestamp - * @param sink - */ - private void generateAuditEvent(Map eventSpec, long syncts, String syncTimestamp, - TelemetryExtractorSink sink, String defaultChannel) { - try { - Telemetry v3spec = new Telemetry(eventSpec, syncts, syncTimestamp, defaultChannel); - String auditEvent = v3spec.toJson(); - sink.toSuccessTopic(auditEvent); - } catch (Exception e) { - e.printStackTrace(); - LOGGER.debug("", "Failed to generate LOG event: " + e.getMessage()); - } - } - - public String addDuplicateFlag(Map batchEvent) { - Map flags = new HashMap<>(); - flags.put("extractor_duplicate", "true"); - batchEvent.put("flags", flags); - return new Gson().toJson(batchEvent); - } - -} diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorConfig.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorConfig.java deleted file mode 100644 index 78f413b0d6..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorConfig.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; - -public class TelemetryExtractorConfig { - - private final String JOB_NAME = "TelemetryExtractor"; - private final String errorTopic; - private String successTopic; - private String assessTopic; - private String defaultChannel; - private final int dupStore; - private int expirySeconds; - private int rawIndividualEventMaxSize; - - public TelemetryExtractorConfig(Config config) { - successTopic = config.get("output.success.topic.name", "telemetry.raw"); - assessTopic = config.get("output.assess.topic.name", "telemetry.assess.redact"); - errorTopic = config.get("output.error.topic.name", "telemetry.extractor.failed"); - defaultChannel = config.get("default.channel", "01250894314817126443"); - dupStore = config.getInt("redis.database.duplicationstore.id", 1); - expirySeconds = config.getInt("redis.database.key.expiry.seconds", 432000); - rawIndividualEventMaxSize = config.getInt("raw.individual.event.maxsize", 996148); - } - - public String successTopic() { - return successTopic; - } - - public String assessTopic() { - return assessTopic; - } - - public String errorTopic() { - return errorTopic; - } - - public String jobName() { - return JOB_NAME; - } - - public String defaultChannel() { - return defaultChannel; - } - - public int dupStore() { - return dupStore; - } - - public int expirySeconds() { - return expirySeconds; - } - - public int rawIndividualEventMaxSize() { - return rawIndividualEventMaxSize; - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorSink.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorSink.java deleted file mode 100644 index 40d9be5f8b..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorSink.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; - -public class TelemetryExtractorSink extends BaseSink { - - private TelemetryExtractorConfig config; - - public TelemetryExtractorSink(MessageCollector collector, JobMetrics metrics, TelemetryExtractorConfig config) { - super(collector, metrics); - this.config = config; - } - - public void toSuccessTopic(String message) { - metrics.incSuccessCounter(); - toTopic(config.successTopic(), null, message); - } - - public void toAssessTopic(String message) { - metrics.incAssessRouteSuccessCounter(); - toTopic(config.assessTopic(), null, message); - } - - public void toErrorTopic(String message) { - metrics.incErrorCounter(); - toTopic(config.errorTopic(), null, message); - } - - public void sinkBatchErrorEvents(String message) { - metrics.incBatchErrorCounter(); - toTopic(config.errorTopic(), null, message); - } - - public void toDuplicateTopic(String message) - { - metrics.incDuplicateCounter(); - toTopic(config.errorTopic(), null, message); - } - -} diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorTask.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorTask.java deleted file mode 100644 index 3b35945eb2..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorTask.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.TelemetryExtractorService; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.ekstep.ep.samza.util.RedisConnect; - -public class TelemetryExtractorTask extends BaseSamzaTask { - - private TelemetryExtractorConfig config; - private TelemetryExtractorService service; - private JobMetrics metrics; - - public TelemetryExtractorTask(Config config, TaskContext context, DeDupEngine deDupEngine) { - init(config, context, deDupEngine); - } - - public TelemetryExtractorTask() { - - } - - @Override - public void init(Config config, TaskContext context) { - init(config, context, null); - } - - - public void init(Config config, TaskContext context, DeDupEngine deDupEngine) { - this.config = new TelemetryExtractorConfig(config); - this.metrics = new JobMetrics(context, this.config.jobName()); - deDupEngine = deDupEngine == null ? - new DeDupEngine(new RedisConnect(config), this.config.dupStore(), this.config.expirySeconds()) : deDupEngine; - this.service = new TelemetryExtractorService(this.config, this.metrics,deDupEngine); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) - throws Exception { - - String message = (String) envelope.getMessage(); - TelemetryExtractorSink sink = new TelemetryExtractorSink(collector, metrics, config); - service.process(message, sink); - } - -} \ No newline at end of file diff --git a/data-pipeline/telemetry-extractor/src/main/resources/log4j.xml b/data-pipeline/telemetry-extractor/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/telemetry-extractor/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/domain/TestCdata.java b/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/domain/TestCdata.java deleted file mode 100644 index 2c832f8006..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/domain/TestCdata.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertTrue; - -public class TestCdata { - - @Test - public void shouldIntializeTheCdataValues() { - CData cData = new CData("contentSession", "c803439b26d433fa920200067ebe4408"); - assertTrue(cData.getType() == "contentSession"); - assertTrue(cData.getId() == "c803439b26d433fa920200067ebe4408"); - - } - - @Test - public void shouldGetTheCdataValues() { - Map cdata = new HashMap<>(); - cdata.put("type", "contentSession"); - cdata.put("id", "c803439b26d433fa920200067ebe4408"); - - CData cData = new CData(cdata); - assertTrue(cData.getType() == "contentSession"); - assertTrue(cData.getId() == "c803439b26d433fa920200067ebe4408"); - - } - -} diff --git a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index c517273eb0..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.lang.reflect.Type; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Map; - -public class EventFixture { - public static Map getEvent(String eventId) throws FileNotFoundException { - URL url = Thread.currentThread().getContextClassLoader().getResource(eventId + ".json"); - File file = new File(url.getPath()); - Type type = new TypeToken>(){}.getType(); - return new Gson().fromJson(new FileReader(file), type); - } - - public static String getEventAsString(String eventId) throws IOException, URISyntaxException { - URL url = Thread.currentThread().getContextClassLoader().getResource(eventId + ".json"); - return new String(Files.readAllBytes(Paths.get(url.toURI()))); - } -} diff --git a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/task/TelemetryExtractorTaskTest.java b/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/task/TelemetryExtractorTaskTest.java deleted file mode 100644 index 333328e25a..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/task/TelemetryExtractorTaskTest.java +++ /dev/null @@ -1,264 +0,0 @@ -package org.ekstep.ep.samza.task; - -import static junit.framework.Assert.assertNotNull; -import static junit.framework.Assert.assertNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -import java.lang.reflect.Type; -import java.util.List; -import java.util.Map; - -import com.google.gson.reflect.TypeToken; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.util.DeDupEngine; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; - -import com.google.gson.Gson; -import redis.clients.jedis.Jedis; - -public class TelemetryExtractorTaskTest { - - class TestMessageCollector implements MessageCollector { - - public OutgoingMessageEnvelope outgoingEnvelope; - - @Override - public void send(OutgoingMessageEnvelope outgoingMessageEnvelope) { - outgoingEnvelope = outgoingMessageEnvelope; - } - } - - private TaskContext context; - private Config config; - private IncomingMessageEnvelope envelope; - private MessageCollector collector; - private TaskCoordinator coordinator; - private Counter counter; - private MetricsRegistry metricsRegistry; - private final String successTopic = "telemetry.raw"; - private final String errorTopic = "telemetry.extractor.failed"; - private final String defaultChannel = "01250894314817126443"; - private Type mapType = new TypeToken>() { - }.getType(); - TelemetryExtractorTask task; - private DeDupEngine deDupEngineMock; - - @Before - public void setup() { - - context = mock(TaskContext.class); - config = mock(Config.class); - envelope = mock(IncomingMessageEnvelope.class); - collector = new TestMessageCollector(); - coordinator = mock(TaskCoordinator.class); - counter = mock(Counter.class); - metricsRegistry = mock(MetricsRegistry.class); - stub(context.getMetricsRegistry()).toReturn(metricsRegistry); - stub(metricsRegistry.newCounter(anyString(), anyString())) - .toReturn(counter); - stub(envelope.getOffset()).toReturn("2"); - stub(envelope.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1))); - stub(config.get("output.success.topic.name", "telemetry.raw")).toReturn(successTopic); - stub(config.get("output.assess.topic.name", "telemetry.assess.redact")).toReturn("telemetry.assess.redact"); - stub(config.get("output.error.topic.name", "telemetry.extractor.failed")).toReturn(errorTopic); - stub(config.get("default.channel", "01250894314817126443")).toReturn(defaultChannel); - stub(config.get("default.channel", "01250894314817126443")).toReturn(defaultChannel); - stub(config.getInt("raw.individual.event.maxsize", 996148)).toReturn(996148); - deDupEngineMock = mock(DeDupEngine.class); - task = new TelemetryExtractorTask(config, context, deDupEngineMock); - - } - - public ArgumentMatcher validateEventObject() { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope envelope = (OutgoingMessageEnvelope) o; - String message = (String) envelope.getMessage(); - SystemStream stream = envelope.getSystemStream(); - Map event = new Gson().fromJson(message, Map.class); - System.out.println("Eid" + event.get("eid")); - - if ("ASSESS".equals(event.get("eid"))) { - assertEquals("ASSESS:6ac822896cd8a1736d55806c13ada64c", event.get("mid")); - assertEquals("kafka", stream.getSystem()); - assertEquals("telemetry.assess.redact", stream.getStream()); - } else if ("RESPONSE".equals(event.get("eid"))) { - assertEquals("RESPONSE:17d97e7cba61bd7c996d3ae71b9706ef", event.get("mid")); - assertEquals("kafka", stream.getSystem()); - assertEquals("telemetry.assess.redact", stream.getStream()); - } else { - assertEquals("kafka", stream.getSystem()); - assertEquals("telemetry.raw", stream.getStream()); - } - return true; - } - }; - } - - @Test - public void synctsShouldBeStamped() throws Exception { - String spec = EventFixture.getEventAsString("event"); - stub(envelope.getMessage()).toReturn(spec); - - task.process(envelope, collector, coordinator); - - OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope; - SystemStream stream = envelope.getSystemStream(); - - assertEquals("kafka", stream.getSystem()); - assertEquals("telemetry.raw", stream.getStream()); - - Map output = new Gson().fromJson((String) envelope.getMessage(), mapType); - assertTrue(output.containsKey("syncts")); - assertTrue(output.containsKey("@timestamp")); - assertEquals(1529500243955L, ((Number) output.get("syncts")).longValue()); - assertEquals("2018-06-20T13:10:43.955Z", output.get("@timestamp")); - - } - - @Test - public void assessEventShouldBeRouted() throws Exception { - String spec = EventFixture.getEventAsString("event"); - stub(envelope.getMessage()).toReturn(spec); - MessageCollector collectorMock = mock(MessageCollector.class); - - task.process(envelope, collectorMock, coordinator); - verify(collectorMock, times(23)).send(argThat(validateEventObject())); - } - - @Test - public void shouldSendDuplicateBatchEventToFailedTopic() throws Exception { - String spec = EventFixture.getEventAsString("event_app"); - stub(envelope.getMessage()).toReturn(spec); - stub(deDupEngineMock.isUniqueEvent(anyString())).toReturn(false); - task.process(envelope, collector, coordinator); - - OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope; - SystemStream stream = envelope.getSystemStream(); - - - assertEquals("kafka", stream.getSystem()); - assertEquals("telemetry.extractor.failed", stream.getStream()); - - Map output = new Gson().fromJson((String) envelope.getMessage(), mapType); - assertTrue(output.containsKey("flags")); - - } - - - @Test - public void eventCountShouldZero() throws Exception { - String spec = EventFixture.getEventAsString("empty_events"); - stub(envelope.getMessage()).toReturn(spec); - task.process(envelope, collector, coordinator); - - OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope; - SystemStream stream = envelope.getSystemStream(); - - assertEquals("kafka", stream.getSystem()); - assertEquals("telemetry.raw", stream.getStream()); - - String output = (String) envelope.getMessage(); - Map event = (Map) new Gson().fromJson(output, Map.class); - List> edata = (List>) ((Map) event.get("edata")).get("params"); - int event_count = ((Number) edata.get(0).get("events_count")).intValue(); - assertEquals(0, event_count); - - Map context = (Map) event.get("context"); - String channel = (String) context.get("channel"); - assertEquals("01250894314817126443", channel); - - Map padata = (Map) context.get("pdata"); - assertEquals("pipeline", padata.get("id")); - } - - @Test - public void specWithoutSyncts() throws Exception { - String spec = EventFixture.getEventAsString("event_without_syncts"); - stub(envelope.getMessage()).toReturn(spec); - task.process(envelope, collector, coordinator); - - OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope; - SystemStream stream = envelope.getSystemStream(); - - String output = (String) envelope.getMessage(); - assertEquals("kafka", stream.getSystem()); - assertEquals("telemetry.raw", stream.getStream()); - assertEquals(true, output.contains("syncts")); - } - - @Test - public void specDoesNotContainJsonData() throws Exception { - String spec = EventFixture.getEventAsString("not_json_event"); - stub(envelope.getMessage()).toReturn(spec); - task.process(envelope, collector, coordinator); - - OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope; - SystemStream stream = envelope.getSystemStream(); - - String output = (String) envelope.getMessage(); - assertEquals("kafka", stream.getSystem()); - assertEquals("telemetry.extractor.failed", stream.getStream()); - List events = (List) ((Map) new Gson().fromJson(output, Map.class)).get("events"); - String content = events.get(0); - assertEquals("testing events", content); - } - - @Test - public void specDoesNotContainEventsKey() throws Exception { - String spec = EventFixture.getEventAsString("without_event_key"); - stub(envelope.getMessage()).toReturn(spec); - - task.process(envelope, collector, coordinator); - - OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope; - SystemStream stream = envelope.getSystemStream(); - - String output = (String) envelope.getMessage(); - assertEquals("kafka", stream.getSystem()); - assertEquals("telemetry.extractor.failed", stream.getStream()); - assertEquals(false, output.contains("events")); - } - - @Test - public void auditEventMetrics() throws Exception { - String spec = EventFixture.getEventAsString("event1"); - stub(envelope.getMessage()).toReturn(spec); - - task.process(envelope, collector, coordinator); - - OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope; - SystemStream stream = envelope.getSystemStream(); - - String output = (String) envelope.getMessage(); - Map event = (Map) new Gson().fromJson(output, Map.class); - assertEquals("kafka", stream.getSystem()); - assertEquals("LOG", (String) event.get("eid")); - Map edata = (Map) event.get("edata"); - assertEquals("telemetry_audit", (String) edata.get("type")); - assertEquals("INFO", (String) edata.get("level")); - - Map param = (Map) ((List) edata.get("params")).get(0); - assertEquals(2, ((Number) param.get("events_count")).intValue()); - assertEquals("SUCCESS", (String) param.get("sync_status")); - } -} diff --git a/data-pipeline/telemetry-extractor/src/test/resources/empty_events.json b/data-pipeline/telemetry-extractor/src/test/resources/empty_events.json deleted file mode 100644 index 0cd7a82dc6..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/resources/empty_events.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":[],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} \ No newline at end of file diff --git a/data-pipeline/telemetry-extractor/src/test/resources/event.json b/data-pipeline/telemetry-extractor/src/test/resources/event.json deleted file mode 100644 index 996187ff97..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/resources/event.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":[{"eid":"RESPONSE","ets":1586759864200,"ver":"3.0","mid":"RESPONSE:17d97e7cba61bd7c996d3ae71b9706ef","actor":{"id":"c5b5732b-b8d6-44ad-bf52-d8f7fc513ac3","type":"User"},"context":{"channel":"0126825293972439041","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"zSizgfO7mzdZ--MhlGCzszgdNhKW6MCp","did":"a915339fbca0afddbf537a1792f01791","cdata":[{"id":"28d1f55313c9c14df8ea86ca1dc73a92","type":"ContentSession"},{"id":"fed7932935f2223870beb120bf58aeed","type":"PlaySession"}],"rollup":{"l1":"0126825293972439041"}},"object":{"id":"do_2129946748092334081914","type":"Content","ver":"2","rollup":{}},"tags":["0126825293972439041"],"edata":{"target":{"id":"do_21299463432309145612386","ver":"1.0","type":"AssessmentItem"},"type":"CHOOSE","values":[{"option0":"

1

\n"}]}},{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"ASSESS","ets":1529499987116,"ver":"3.0","mid":"ASSESS:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} \ No newline at end of file diff --git a/data-pipeline/telemetry-extractor/src/test/resources/event1.json b/data-pipeline/telemetry-extractor/src/test/resources/event1.json deleted file mode 100644 index 73e606e079..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/resources/event1.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":[{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} \ No newline at end of file diff --git a/data-pipeline/telemetry-extractor/src/test/resources/event_app.json b/data-pipeline/telemetry-extractor/src/test/resources/event_app.json deleted file mode 100644 index e194e7fe18..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/resources/event_app.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} \ No newline at end of file diff --git a/data-pipeline/telemetry-extractor/src/test/resources/event_without_syncts.json b/data-pipeline/telemetry-extractor/src/test/resources/event_without_syncts.json deleted file mode 100644 index 059720429c..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/resources/event_without_syncts.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0"} \ No newline at end of file diff --git a/data-pipeline/telemetry-extractor/src/test/resources/not_json_event.json b/data-pipeline/telemetry-extractor/src/test/resources/not_json_event.json deleted file mode 100644 index b040d7751d..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/resources/not_json_event.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":["testing events"],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} \ No newline at end of file diff --git a/data-pipeline/telemetry-extractor/src/test/resources/without_event_key.json b/data-pipeline/telemetry-extractor/src/test/resources/without_event_key.json deleted file mode 100644 index 1060389692..0000000000 --- a/data-pipeline/telemetry-extractor/src/test/resources/without_event_key.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"data":[],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955} \ No newline at end of file diff --git a/data-pipeline/telemetry-location-updater/pom.xml b/data-pipeline/telemetry-location-updater/pom.xml deleted file mode 100644 index fe2bd49ebd..0000000000 --- a/data-pipeline/telemetry-location-updater/pom.xml +++ /dev/null @@ -1,206 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - - org.ekstep.ecosystem.jobs - telemetry-location-updater - 0.2.3 - jar - TelemetryLocationUpdater - - Updates State District/City and device information for all telemetry events - - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - - com.google.guava - guava - 18.0 - - - redis.clients - jedis - 2.9.0 - - - com.datastax.cassandra - cassandra-driver-core - 3.1.0 - - - com.squareup.okhttp3 - okhttp - 3.2.0 - - - com.fiftyonred - mock-jedis - 0.4.0 - jar - test - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - diff --git a/data-pipeline/telemetry-location-updater/src/main/assembly/src.xml b/data-pipeline/telemetry-location-updater/src/main/assembly/src.xml deleted file mode 100644 index 76a938a35b..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/telemetry-location-updater.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-kv-rocksdb_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:telemetry-location-updater - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/telemetry-location-updater/src/main/config/telemetry-location-updater.properties b/data-pipeline/telemetry-location-updater/src/main/config/telemetry-location-updater.properties deleted file mode 100644 index 6d4aac36f4..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/config/telemetry-location-updater.properties +++ /dev/null @@ -1,84 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.TelemetryLocationUpdater - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__telemetry_location_updater_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.TelemetryLocationUpdaterTask -task.inputs=kafka.__env__.telemetry.sink -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -output.success.topic.name=__env__.telemetry.with_location -output.failed.topic.name=__env__.telemetry.failed -output.malformed.topic.name=__env__.telemetry.malformed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,error-message-count,cache-hit-count,cache-miss-count,cache-empty-values-count,processed-message-count,unprocessed-message-count,user-cache-hit-count,user-declared-hit-count,ip-location-hit-count - -# redis -# redis.host=localhost -redis.host=__metadata_redis_host__ -# redis.port=6379 -redis.port=__metadata_redis_port__ -redis.connection.max=2 -location.db.redis.key.expiry.seconds=__location_db_redis_key_expiry_seconds__ -cache.unresolved.location.key.expiry.seconds=__cache_unresolved_location_key_expiry_seconds__ -redis.connection.idle.max=2 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.database.userStore.id=4 -redis.database.deviceLocationStore.id=2 \ No newline at end of file diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java deleted file mode 100644 index 788322232d..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; - -public class DeviceProfile { - private String countryCode; - private String country; - private String stateCode; - private String state; - private String city; - private String districtCustom; - private String stateCodeCustom; - private String stateCustomName; - private String userDeclaredState; - private String userDeclaredDistrict; - private Map devicespec; - private Long firstaccess; - private Gson gson = new Gson(); - private Type type = new TypeToken>() {}.getType(); - - public DeviceProfile() { - this.countryCode = ""; - this.country = ""; - this.stateCode = ""; - this.state = ""; - this.city = ""; - this.districtCustom = ""; - this.stateCodeCustom = ""; - this.stateCustomName = ""; - this.userDeclaredState = ""; - this.userDeclaredDistrict = ""; - this.devicespec = new HashMap<>(); - this.firstaccess = 0L; - } - - public Map toMap() { - Map values = new HashMap<>(); - values.put("country_code", DeviceProfile.getValueOrDefault(this.countryCode, "")); - values.put("country", DeviceProfile.getValueOrDefault(this.country, "")); - values.put("state_code", DeviceProfile.getValueOrDefault(this.stateCode, "")); - values.put("state", DeviceProfile.getValueOrDefault(this.state, "")); - values.put("city", DeviceProfile.getValueOrDefault(this.city, "")); - values.put("district_custom", DeviceProfile.getValueOrDefault(this.districtCustom, "")); - values.put("state_custom", DeviceProfile.getValueOrDefault(this.stateCustomName, "")); - values.put("state_code_custom", DeviceProfile.getValueOrDefault(this.stateCodeCustom, "")); - values.put("user_declared_state", DeviceProfile.getValueOrDefault(this.userDeclaredState, "")); - values.put("user_declared_district", DeviceProfile.getValueOrDefault(this.userDeclaredDistrict, "")); - values.put("devicespec", gson.toJson(DeviceProfile.getValueOrDefault(this.devicespec, new HashMap<>()))); - values.put("firstaccess", DeviceProfile.getValueOrDefault(String.valueOf(this.firstaccess), "")); - return values; - } - - public DeviceProfile fromMap(Map map) { - this.countryCode = map.getOrDefault("country_code", ""); - this.country = map.getOrDefault("country", ""); - this.stateCode = map.getOrDefault("state_code", ""); - this.state = map.getOrDefault("state", ""); - this.city = map.getOrDefault("city", ""); - this.districtCustom = map.getOrDefault("district_custom", ""); - this.stateCustomName = map.getOrDefault("state_custom", ""); - this.stateCodeCustom = map.getOrDefault("state_code_custom", ""); - this.userDeclaredState = map.getOrDefault("user_declared_state", ""); - this.userDeclaredDistrict = map.getOrDefault("user_declared_district", ""); - this.devicespec = gson.fromJson(map.getOrDefault("devicespec", "{}"), type); - this.firstaccess = Long.valueOf(map.getOrDefault("firstaccess", "0")); - return this; - } - - public DeviceProfile(String countryCode, String country, String stateCode, String state, String city) { - this.countryCode = countryCode; - this.country = country; - this.stateCode = stateCode; - this.state = state; - this.city = city; - } - - public DeviceProfile(String countryCode, String country, String stateCode, String state, - String city, String districtCustom, String stateCustomName, - String stateCodeCustom, Map device_spec, Long first_access, - String userDeclaredDistrict, String userDeclaredState) { - this.countryCode = countryCode; - this.country = country; - this.stateCode = stateCode; - this.state = state; - this.city = city; - this.districtCustom = districtCustom; - this.stateCustomName = stateCustomName; - this.stateCodeCustom = stateCodeCustom; - this.userDeclaredState = userDeclaredState; - this.userDeclaredDistrict = userDeclaredDistrict; - this.devicespec = device_spec; - this.firstaccess = first_access; - } - - public String getCity() { - return this.city; - } - - public String getState() { - return this.state; - } - - public String getCountryCode() { - return countryCode; - } - - public String getCountry() { - return country; - } - - public String getStateCode() { - return stateCode; - } - - public String getDistrictCustom() { - return districtCustom; - } - - public String getstateCustomName() { - return stateCustomName; - } - - public String getstateCodeCustom() { - return stateCodeCustom; - } - - public String getUserDeclaredState() { return userDeclaredState; } - - public String getUserDeclaredDistrict() { return userDeclaredDistrict; } - - public Map getDevicespec() { - return devicespec; - } - - public Long getFirstaccess() { - return firstaccess; - } - - public Boolean isLocationResolved() { - return this.state != null && !this.state.isEmpty(); - } - - public Boolean isDeviceProfileResolved() { - return this.isLocationResolved() || (!this.devicespec.isEmpty() || this.firstaccess > 0); - } - - public static T getValueOrDefault(T value, T defaultValue) { - return value == null ? defaultValue : value; - } -} diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index 0ea74f15ac..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.google.gson.reflect.TypeToken; -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.reader.NullableValue; -import org.ekstep.ep.samza.task.TelemetryLocationUpdaterConfig; - -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; - -public class Event extends Events { - - private Gson gson = new Gson(); - private Type type = new TypeToken>() { - }.getType(); - - public Event(Map map) { - super(map); - } - - public void markFailure(String error, TelemetryLocationUpdaterConfig config) { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.tr_processed", false); - - telemetry.addFieldIfAbsent("metadata", new HashMap()); - telemetry.add("metadata.tr_error", error); - telemetry.add("metadata.src", config.jobName()); - } - - public void addDeviceProfile(DeviceProfile deviceProfile) { - Map ldata = new HashMap<>(); - - ldata.put("countrycode", deviceProfile.getCountryCode()); - ldata.put("country", deviceProfile.getCountry()); - ldata.put("statecode", deviceProfile.getStateCode()); - ldata.put("state", deviceProfile.getState()); - ldata.put("city", deviceProfile.getCity()); - ldata.put("statecustomcode", deviceProfile.getstateCodeCustom()); - ldata.put("statecustomname", deviceProfile.getstateCustomName()); - ldata.put("districtcustom", deviceProfile.getDistrictCustom()); - ldata.put("devicespec", deviceProfile.getDevicespec()); - ldata.put("firstaccess", deviceProfile.getFirstaccess()); - String iso3166statecode = addISOStateCodeToDeviceProfile(deviceProfile); - if (!iso3166statecode.isEmpty()) { - ldata.put("iso3166statecode", iso3166statecode); - } - - Map userDeclared = new HashMap<>(); - userDeclared.put("state", deviceProfile.getUserDeclaredState()); - userDeclared.put("district", deviceProfile.getUserDeclaredDistrict()); - ldata.put("userdeclared", userDeclared); - telemetry.add(path.deviceData(), ldata); - } - - public void addDerivedLocation(Map derivedData) { - telemetry.add(path.derivedLocationData(), derivedData); - } - - public String addISOStateCodeToDeviceProfile(DeviceProfile deviceProfile) { - // add new statecode field - String statecode = deviceProfile.getStateCode(); - if (statecode != null && !statecode.isEmpty()) { - return "IN-" + statecode; - } else return ""; - } - - public void removeEdataLoc() { - Gson gson = new Gson(); - JsonObject json = gson.toJsonTree(getMap()).getAsJsonObject().get("edata").getAsJsonObject(); - json.remove("loc"); - telemetry.add(path.edata(), json); - } - - public void setFlag(String key, Object value) { - NullableValue> telemetryFlag = telemetry.read(path.flags()); - Map flags = telemetryFlag.isNull() ? new HashMap<>() : telemetryFlag.value(); - flags.put(key, value); - telemetry.add(path.flags(), flags); - } -} diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/service/TelemetryLocationUpdaterService.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/service/TelemetryLocationUpdaterService.java deleted file mode 100644 index b894ccc2ee..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/service/TelemetryLocationUpdaterService.java +++ /dev/null @@ -1,176 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.JsonSyntaxException; -import com.google.gson.reflect.TypeToken; -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.DeviceProfile; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.events.domain.Path; -import org.ekstep.ep.samza.task.TelemetryLocationUpdaterConfig; -import org.ekstep.ep.samza.task.TelemetryLocationUpdaterSink; -import org.ekstep.ep.samza.task.TelemetryLocationUpdaterSource; -import org.ekstep.ep.samza.util.DeviceProfileCache; - -import org.ekstep.ep.samza.util.RedisConnect; -import redis.clients.jedis.Jedis; -import com.google.gson.Gson; -import redis.clients.jedis.exceptions.JedisException; - -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; - -public class TelemetryLocationUpdaterService { - - private static Logger LOGGER = new Logger(TelemetryLocationUpdaterService.class); - private DeviceProfileCache deviceProfileCache; - private int userStoreDb; - private RedisConnect redisConnect; - private Jedis userDataStoreConnection; - private JobMetrics metrics; - private Gson gson = new Gson(); - private Path path = new Path(); - private Type mapType = new TypeToken>() { }.getType(); - - public TelemetryLocationUpdaterService(DeviceProfileCache deviceLocationCache, JobMetrics metrics, RedisConnect redisConnect, Config config) { - this.deviceProfileCache = deviceLocationCache; - this.metrics = metrics; - this.userStoreDb = config.getInt("redis.database.userStore.id", 4); - this.redisConnect = redisConnect; - this.userDataStoreConnection = this.redisConnect.getConnection(userStoreDb); - } - - public void process(TelemetryLocationUpdaterSource source, TelemetryLocationUpdaterSink sink) { - try { - Event event = source.getEvent(); - // Temporary fix for skipping pipeline metrics - if (null != event.eid()) { - String did = event.did(); - DeviceProfile deviceProfile = null; - if (did != null && !did.isEmpty()) { - // get device profile from cache - deviceProfile = deviceProfileCache.getDeviceProfileForDeviceId(did); - - // check for user profile location - Map derivedLocation = getLocationFromUserCache(event); - - // get user declared location if user profile location is empty - if (derivedLocation.isEmpty()) derivedLocation = getUserDeclaredLocation(deviceProfile); - - // get ip resolved location if user declared location is empty - if (derivedLocation.isEmpty()) derivedLocation = getIpResolvedLocation(deviceProfile); - - // Add derived location to telemetry - if (derivedLocation.isEmpty()) { - metrics.incCacheMissCounter(); - event.setFlag(TelemetryLocationUpdaterConfig.getDerivedLocationJobFlag(), false); - } else { - // update derived location metrics - updateDerivedLocationMetrics(derivedLocation); - event.addDerivedLocation(derivedLocation); - event.setFlag(TelemetryLocationUpdaterConfig.getDerivedLocationJobFlag(), true); - } - - // Add device profile details to the event - updateEvent(event, deviceProfile); - metrics.incProcessedMessageCount(); - sink.toSuccessTopic(event); - } else { - updateEvent(event, deviceProfile); - metrics.incUnprocessedMessageCount(); - sink.toSuccessTopic(event); - } - } - } catch (JsonSyntaxException ex) { - LOGGER.error(null, "INVALID EVENT: " + source.getMessage(), ex); - sink.toMalformedTopic(source.getMessage()); - } - } - - private Map getLocationFromUserCache(Event event) { - String uid = event.actorId(); - Map locationData; - try { - locationData = getLocationForUser(uid); - return locationData; - } - catch (JedisException ex) { - LOGGER.error(null, "Reconnecting with Redis store due to exception: ", ex); - this.userDataStoreConnection.close(); - this.userDataStoreConnection = redisConnect.getConnection(userStoreDb); - return getLocationForUser(uid); - } - } - - private Map getLocationForUser(String uid) { - Map userCacheData; - Map locationData = new HashMap<>(); - if (uid != null) { - String data = userDataStoreConnection.get(uid); - if (data != null && !data.isEmpty()) { - userCacheData = gson.fromJson(data, mapType); - if (!userCacheData.isEmpty() && userCacheData.containsKey("state") && !userCacheData.get("state").toString().isEmpty()) { - locationData.put(path.stateKey(), userCacheData.get("state").toString()); - locationData.put(path.districtKey(), userCacheData.getOrDefault("district", "").toString()); - locationData.put(path.locDerivedFromKey(), "user-profile"); - } - } - } - return locationData; - } - - private Map getUserDeclaredLocation(DeviceProfile deviceProfile) { - Map locationData = new HashMap<>(); - if (null != deviceProfile) { - Map data = deviceProfile.toMap(); - if (!data.isEmpty() && data.containsKey("user_declared_state") && !data.get("user_declared_state").isEmpty()) { - locationData.put(path.stateKey(), data.get("user_declared_state").toString()); - locationData.put(path.districtKey(), data.getOrDefault("user_declared_district", "").toString()); - locationData.put(path.locDerivedFromKey(), "user-declared"); - } - } - return locationData; - } - - private Map getIpResolvedLocation(DeviceProfile deviceProfile) { - Map locationData = new HashMap<>(); - if (null != deviceProfile) { - Map data = deviceProfile.toMap(); - if(!data.isEmpty() && data.containsKey("state")) { - locationData.put(path.stateKey(), data.get("state").toString()); - locationData.put(path.districtKey(), data.getOrDefault("district_custom", "").toString()); - locationData.put(path.locDerivedFromKey(), "ip-resolved"); - } - } - return locationData; - } - - private void updateDerivedLocationMetrics(Map derivedLocation) { - String derivedFrom = derivedLocation.get("from"); - if("user-profile".equalsIgnoreCase(derivedFrom)) metrics.incUserCacheHitCount(); - if("user-declared".equalsIgnoreCase(derivedFrom)) metrics.incUserDeclaredHitCount(); - if("ip-resolved".equalsIgnoreCase(derivedFrom)) metrics.incIpLocationHitCount(); - } - - public void updateEvent(Event event, DeviceProfile deviceProfile) { - event.removeEdataLoc(); - if (null != deviceProfile) { - event.addDeviceProfile(deviceProfile); - if (deviceProfile.isDeviceProfileResolved()) { - event.setFlag(TelemetryLocationUpdaterConfig.getDeviceProfileJobFlag(), true); - } else { - event.setFlag(TelemetryLocationUpdaterConfig.getDeviceProfileJobFlag(), false); - } - - if (deviceProfile.isLocationResolved()) { - event.setFlag(TelemetryLocationUpdaterConfig.getDeviceLocationJobFlag(), true); - } else { - event.setFlag(TelemetryLocationUpdaterConfig.getDeviceLocationJobFlag(), false); - } - } else { - event.setFlag(TelemetryLocationUpdaterConfig.getDeviceProfileJobFlag(), false); - } - } -} diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterConfig.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterConfig.java deleted file mode 100644 index 904b63b5d0..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterConfig.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; - -public class TelemetryLocationUpdaterConfig { - - private final String JOB_NAME = "TelemetryLocationUpdater"; - private static final String deviceProfileJobFlag = "device_profile_retrieved"; - private static final String deviceLocationJobFlag = "device_location_retrieved"; - private static final String derivedLocationJobFlag = "derived_location_retrieved"; - - private String successTopic; - private String failedTopic; - private String malformedTopic; - - public TelemetryLocationUpdaterConfig(Config config) { - - successTopic = config.get("output.success.topic.name", "telemetry.with_loation"); - failedTopic = config.get("output.failed.topic.name", "telemetry.failed"); - malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed"); - } - - public String successTopic() { - return successTopic; - } - - public String failedTopic() { - return failedTopic; - } - - public String malformedTopic() { - return malformedTopic; - } - - public String jobName() { - return JOB_NAME; - } - - public static String getDeviceProfileJobFlag() { return deviceProfileJobFlag; } - - public static String getDeviceLocationJobFlag() { - return deviceLocationJobFlag; - } - - public static String getDerivedLocationJobFlag() { - return derivedLocationJobFlag; - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSink.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSink.java deleted file mode 100644 index 6d3ef99466..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSink.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - - -public class TelemetryLocationUpdaterSink extends BaseSink { - - private TelemetryLocationUpdaterConfig config; - - public TelemetryLocationUpdaterSink(MessageCollector collector, JobMetrics metrics, TelemetryLocationUpdaterConfig config) { - super(collector, metrics); - this.config = config; - } - - public void toSuccessTopic(Event event) { - toTopic(config.successTopic(), event.did(), event.getJson()); - metrics.incSuccessCounter(); - } - - public void toErrorTopic(Event event, String errorMessage) { - event.markFailure(errorMessage, config); - toTopic(config.failedTopic(), event.did(), event.getJson()); - metrics.incErrorCounter(); - } - - public void toMalformedTopic(String message) { - toTopic(config.malformedTopic(), null, message); - metrics.incErrorCounter(); - } - -} diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSource.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSource.java deleted file mode 100644 index 60edb1411b..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSource.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.gson.Gson; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; - -import java.util.Map; - -public class TelemetryLocationUpdaterSource { - static Logger LOGGER = new Logger(TelemetryLocationUpdaterSource.class); - - private IncomingMessageEnvelope envelope; - - public TelemetryLocationUpdaterSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent(){ - return new Event(getMap()); - } - - @SuppressWarnings("unchecked") - private Map getMap() { - String message = (String) envelope.getMessage(); - return (Map) new Gson().fromJson(message, Map.class); - } - - public String getMessage() { - return envelope.toString(); - } - -} diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTask.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTask.java deleted file mode 100644 index 5e5042cdcd..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTask.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.*; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.TelemetryLocationUpdaterService; -import org.ekstep.ep.samza.util.DeviceProfileCache; -import org.ekstep.ep.samza.util.RedisConnect; - -public class TelemetryLocationUpdaterTask extends BaseSamzaTask { - - private TelemetryLocationUpdaterConfig config; - private JobMetrics metrics; - private TelemetryLocationUpdaterService service; - private DeviceProfileCache deviceProfileCache; - private RedisConnect redisConnect; - - public TelemetryLocationUpdaterTask(Config config, TaskContext context, DeviceProfileCache deviceProfileCache, RedisConnect redisConnect) { - init(config, context, deviceProfileCache, redisConnect); - } - - public TelemetryLocationUpdaterTask() { - } - - @Override - public void init(Config config, TaskContext context) { - init(config, context, deviceProfileCache, redisConnect); - } - - public void init(Config config, TaskContext context, DeviceProfileCache deviceProfileCache, RedisConnect redisConnect) { - - this.config = new TelemetryLocationUpdaterConfig(config); - this.metrics = new JobMetrics(context, this.config.jobName()); - this.redisConnect = redisConnect == null ? new RedisConnect(config) : redisConnect; - this.deviceProfileCache = deviceProfileCache == null ? new DeviceProfileCache(config, metrics, this.redisConnect) : deviceProfileCache; - this.service = new TelemetryLocationUpdaterService(this.deviceProfileCache, metrics, this.redisConnect, config); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, - TaskCoordinator taskCoordinator) throws Exception { - - TelemetryLocationUpdaterSink sink = new TelemetryLocationUpdaterSink(collector, metrics, config); - TelemetryLocationUpdaterSource source = new TelemetryLocationUpdaterSource(envelope); - service.process(source, sink); - } - -} diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/util/DeviceProfileCache.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/util/DeviceProfileCache.java deleted file mode 100644 index 04b415bbf2..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/util/DeviceProfileCache.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.ekstep.ep.samza.util; - -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.DeviceProfile; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.exceptions.JedisException; -import java.util.Map; - - -public class DeviceProfileCache { - - private static Logger LOGGER = new Logger(DeviceProfileCache.class); - private RedisConnect redisConnect; - private JobMetrics metrics; - private Jedis redisConnection; - private int databaseIndex; - - public DeviceProfileCache(Config config, JobMetrics metrics, RedisConnect redisConnect) { - this.databaseIndex = config.getInt("redis.database.deviceLocationStore.id", 2); - this.redisConnect = redisConnect; - this.redisConnection = this.redisConnect.getConnection(databaseIndex); - this.metrics = metrics; - } - - public DeviceProfile getDeviceProfileForDeviceId(String did) { - DeviceProfile deviceProfile = null; - if (did != null && !did.isEmpty()) { - // Get location from redis. Safe retry once for redis connection loss or error - try { - deviceProfile = getDeviceProfileFromCache(did); - } catch (JedisException ex) { - LOGGER.error(null, "Reconnecting with Redis store due to exception: ", ex); - this.redisConnection.close(); - this.redisConnection = redisConnect.getConnection(databaseIndex); - deviceProfile = getDeviceProfileFromCache(did); - } - if (null != deviceProfile && deviceProfile.isLocationResolved()) { - metrics.incCacheHitCounter(); - } else { - metrics.incEmptyCacheValueCounter(); - } - } - return deviceProfile; - } - - public DeviceProfile getDeviceProfileFromCache(String deviceId) { - Map data = redisConnection.hgetAll(deviceId); - if (data.size() > 0) { - return new DeviceProfile().fromMap(data); - } else { - return null; - } - } -} diff --git a/data-pipeline/telemetry-location-updater/src/main/resources/log4j.xml b/data-pipeline/telemetry-location-updater/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/telemetry-location-updater/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/UserLocationCacheTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/UserLocationCacheTest.java deleted file mode 100644 index c338f3397e..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/UserLocationCacheTest.java +++ /dev/null @@ -1,22 +0,0 @@ -//package org.ekstep.ep.samza; -// -//import org.apache.samza.config.Config; -//import org.ekstep.ep.samza.util.*; -//import org.junit.Before; -//import static org.mockito.Mockito.*; -// -//public class UserLocationCacheTest { -// -// private UserLocationCache userLocationCache; -// private RedisConnect redisConnectMock; -// private CassandraConnect cassandraConnectMock; -// -// @SuppressWarnings("unchecked") -// @Before -// public void setUp() { -// Config config = mock(Config.class); -// this.redisConnectMock = mock(RedisConnect.class); -// this.cassandraConnectMock = mock(CassandraConnect.class); -// this.userLocationCache = new UserLocationCache(config, redisConnectMock, cassandraConnectMock); -// } -//} diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index 78c4e91709..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,469 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import java.util.Map; - -public class EventFixture { - - public static final String LOG_EVENT = "{\n" + - " \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" + - " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"data\": \"\",\n" + - " \"err\": \"10\",\n" + - " \"eventId\": \"GE_SIGNUP\",\n" + - " \"id\": \"2131165210\",\n" + - " \"type\": \"GENIE\"\n" + - " }\n" + - " },\n" + - " \"eid\": \"LOG\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genie.android\",\n" + - " \"ver\": \"2.2.15\"\n" + - " },\n" + - " \"pdata\": {\n" + - " \"id\": \"genie\",\n" + - " \"ver\": \"2.0\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"ets\": 1454064092546,\n" + - " \"uid\": \"\",\n" + - " \"ver\": \"2.2\",\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"correlationid\",\n" + - " \"type\": \"correlationtype\"\n" + - " }\n" + - " ]\n" + - "}"; - public static final String ERROR_EVENT = "{\n" + - " \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" + - " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + - " \"eid\": \"ERROR\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genie.android\",\n" + - " \"ver\": \"2.2.15\"\n" + - " },\n" + - " \"pdata\": {\n" + - " \"id\": \"genie\",\n" + - " \"ver\": \"2.0\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"ets\": 1454064092546,\n" + - " \"uid\": \"\",\n" + - " \"ver\": \"2.2\",\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"correlationid\",\n" + - " \"type\": \"correlationtype\"\n" + - " }\n" + - " ]\n" + - "}"; - public static final String UNPARSABLE_START_EVENT = "{\n" + - " \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n" + - " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"mode\": \"WIFI\",\n" + - " \"ver\": \"12\",\n" + - " \"size\": 12.67,\n" + - " \"err\": \"\",\n" + - " \"referrer\": [\n" + - " {\n" + - " \"action\": \"INSTALL\",\n" + - " \"utmsource\": \"Ekstep\",\n" + - " \"utmmedium\": \"Portal\",\n" + - " \"utmterm\": \"December 2016\",\n" + - " \"utmcontent\": \"Ramayana\",\n" + - " \"utmcampaign\": \"Epics of India\"\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " \"eid\": \"START\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genie.android\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"ets\": 1454064092546,\n" + - " \"uid\": \"\",\n" + - " \"ver\": \"2.0\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"correlationid\",\n" + - " \"type\": \"correlationtype\"\n" + - " "; - public static final String START_EVENT = "{\n" + - " \"cdata\": [],\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"did\": \"0427fedf56eea1c8a127d876fd1907ffb245684f\",\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"extype\": \"\",\n" + - " \"id\": \"\",\n" + - " \"pos\": [],\n" + - " \"stageid\": \"Genie-TelemetrySync\",\n" + - " \"subtype\": \"ManualSync-Success\",\n" + - " \"tid\": \"\",\n" + - " \"type\": \"OTHER\",\n" + - " \"uri\": \"\",\n" + - " \"values\": [\n" + - " {\n" + - " \"SizeOfFileInKB\": \"0.81\"\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " \"eid\": \"GE_INTERACT\",\n" + - " \"etags\": {\n" + - " \"app\": []\n" + - " },\n" + - " \"ets\": 1506328824238,\n" + - " \"gdata\": {\n" + - " \"id\": \"org.ekstep.genieservices.qa\",\n" + - " \"ver\": \"6.5.localdev-debug\"\n" + - " },\n" + - " \"mid\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\",\n" + - " \"pdata\": {\n" + - " \"id\": \"genie\",\n" + - " \"ver\": \"6.5.localdev-debug\"\n" + - " },\n" + - " \"sid\": \"e9662bb5-cf08-42d6-ad11-700b23566961\",\n" + - " \"ts\": \"2017-09-25T08:40:24.238+0000\",\n" + - " \"uid\": \"03762014-f67b-466b-bf20-467f46542563\",\n" + - " \"ver\": \"2.2\",\n" + - " \"@version\": \"1\",\n" + - " \"@timestamp\": \"2017-09-25T08:35:44.037Z\",\n" + - " \"metadata\": {\n" + - " \"checksum\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\"\n" + - " },\n" + - " \"uuid\": \"7f2c9f88-cbf7-4527-9f8d-667d4dde0d1c1\",\n" + - " \"key\": \"03762014-f67b-466b-bf20-467f46542563\",\n" + - " \"type\": \"events\",\n" + - " \"flags\": {\n" + - " \"dd_processed\": true\n" + - " }\n" + - "}"; - - public static final String INTERACT_EVENT = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"loc\":\"xyz\",\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1541574545180,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"object\": {\n" + - " \"id\": \"do_31249561779090227216256\",\n" + - " \"type\": \"Content\",\n" + - " \"version\": \"\"\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" + - "}"; - - public static final String INTERACT_EVENT_WITHOUT_DID = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"loc\":\"xyz\",\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1541574545180,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"object\": {\n" + - " \"id\": \"do_31249561779090227216256\",\n" + - " \"type\": \"Content\",\n" + - " \"version\": \"\"\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" + - "}"; - - public static final String INTERACT_EVENT_WITH_ACTOR_AS_SYSTEM = "{\n" + - " \"actor\": {\n" + - " \"type\": \"System\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"INTERACT\",\n" + - " \"edata\": {\n" + - " \"loc\":\"xyz\",\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1541574545180,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"object\": {\n" + - " \"id\": \"do_31249561779090227216256\",\n" + - " \"type\": \"Content\",\n" + - " \"version\": \"\"\n" + - " },\n" + - " \"tags\": [\n" + - " \n" + - " ],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" + - "}"; - - public static final String ANY_STRING = "Hey Samza, Whats Up?"; - public static final String EMPTY_JSON = "{}"; - - public static final String CHANNEL_RESPONSE_BODY = "{\n" + - " \"id\": \"api.org.search\",\n" + - " \"ver\": \"v1\",\n" + - " \"ts\": \"2019-01-08 06:40:12:316+0000\",\n" + - " \"params\": {\n" + - " \"resmsgid\": null,\n" + - " \"msgid\": \"0f93ff40-582f-4599-a718-7bd898cd8b5d\",\n" + - " \"err\": null,\n" + - " \"status\": \"success\",\n" + - " \"errmsg\": null\n" + - " },\n" + - " \"responseCode\": \"OK\",\n" + - " \"result\": {\n" + - " \"response\": {\n" + - " \"count\": 1,\n" + - " \"content\": [\n" + - " {\n" + - " \"dateTime\": null,\n" + - " \"preferredLanguage\": \"English\",\n" + - " \"approvedBy\": null,\n" + - " \"channel\": \"ROOT_ORG\",\n" + - " \"description\": \"Andhra State Boardsssssss\",\n" + - " \"updatedDate\": \"2018-11-28 10:00:08:675+0000\",\n" + - " \"addressId\": null,\n" + - " \"provider\": null,\n" + - " \"locationId\": null,\n" + - " \"orgCode\": \"sunbird\",\n" + - " \"theme\": null,\n" + - " \"id\": \"ORG_001\",\n" + - " \"communityId\": null,\n" + - " \"isApproved\": null,\n" + - " \"email\": \"support_dev@sunbird.org\",\n" + - " \"slug\": \"sunbird\",\n" + - " \"identifier\": \"ORG_001\",\n" + - " \"thumbnail\": null,\n" + - " \"orgName\": \"Sunbird\",\n" + - " \"updatedBy\": \"1d7b85b0-3502-4536-a846-d3a51fd0aeea\",\n" + - " \"locationIds\": [\n" + - " \"969dd3c1-4e98-4c17-a994-559f2dc70e18\"\n" + - " ],\n" + - " \"externalId\": null,\n" + - " \"isRootOrg\": true,\n" + - " \"rootOrgId\": \"ORG_001\",\n" + - " \"approvedDate\": null,\n" + - " \"imgUrl\": null,\n" + - " \"homeUrl\": null,\n" + - " \"orgTypeId\": null,\n" + - " \"isDefault\": true,\n" + - " \"contactDetail\": \"[{\\\"phone\\\":\\\"213124234234\\\",\\\"email\\\":\\\"test@test.com\\\"},{\\\"phone\\\":\\\"+91213124234234\\\",\\\"email\\\":\\\"test1@test.com\\\"}]\",\n" + - " \"createdDate\": null,\n" + - " \"createdBy\": null,\n" + - " \"parentOrgId\": null,\n" + - " \"hashTagId\": \"b00bc992ef25f1a9a8d63291e20efc8d\",\n" + - " \"noOfMembers\": 5,\n" + - " \"status\": 1\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - "}"; - - public static final String LOCATION_SEARCH_RESPONSE_BODY = "{\n" + - " \"id\": \"api.location.search\",\n" + - " \"ver\": \"v1\",\n" + - " \"ts\": \"2019-01-08 06:10:57:676+0000\",\n" + - " \"params\": {\n" + - " \"resmsgid\": null,\n" + - " \"msgid\": \"bddca208-34b5-4054-9a05-24eecfc12c99\",\n" + - " \"err\": null,\n" + - " \"status\": \"success\",\n" + - " \"errmsg\": null\n" + - " },\n" + - " \"responseCode\": \"OK\",\n" + - " \"result\": {\n" + - " \"response\": [\n" + - " {\n" + - " \"code\": \"29\",\n" + - " \"name\": \"Karnataka\",\n" + - " \"id\": \"969dd3c1-4e98-4c17-a994-559f2dc70e18\",\n" + - " \"type\": \"state\"\n" + - " }\n" + - " ]\n" + - " }\n" + - "}"; - - public static final String LOCATION_SEARCH_UNSUCCESSFUL_RESPONSE = "{\n"+ - " \"id\": \"api.location.search\",\n"+ - " \"ver\": \"v1\",\n"+ - " \"ts\": \"2019-01-08 07:22:25:891+0000\",\n"+ - " \"params\": {\n"+ - " \"resmsgid\": null,\n"+ - " \"msgid\": \"15b23224-d7b5-4645-bbbe-3fd1172b4112\",\n"+ - " \"err\": null,\n"+ - " \"status\": \"success\",\n"+ - " \"errmsg\": null\n"+ - " },\n"+ - " \"responseCode\": \"OK\",\n"+ - " \"result\": {\n"+ - " \"response\": []\n"+ - " }\n"+ - "}"; - - public static final String DEVICE_SUMMARY = "{\"eid\":\"ME_DEVICE_SUMMARY\",\"ets\":1573607175904,\"syncts\":1573557231120,\"ver\":\"1.0\",\"mid\":\"2D03E6A051F77F4EEE1F48E4AD7FE575\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"DeviceSummary\"},\"granularity\":\"DAY\",\"date_range\":{\"from\":1573196386089,\"to\":1573196390938},\"channel\":\"b00bc992ef25f1a9a8d63291e20efc8d\"},\"dimensions\":{\"did\":\"099988ce86c4dbb9a4057ff611d38427\",\"channel\":\"b00bc992ef25f1a9a8d63291e20efc8d\"},\"edata\":{\"eks\":{\"firstAccess\":1573196380482,\"dial_stats\":{\"total_count\":0,\"success_count\":0,\"failure_count\":0},\"content_downloads\":0,\"contents_played\":0,\"total_ts\":4.85,\"total_launches\":1,\"unique_contents_played\":0}},\"flags\":{\"derived_dd_processed\":true},\"type\":\"events\",\"ts\":\"2019-11-13T01:06:15.904+0000\"}"; - - public static final String CHANNEL_SEARCH_EMPTY_LOCATIONIDS_RESPONSE = "{\n" + - " \"id\": \"api.org.search\",\n" + - " \"ver\": \"v1\",\n" + - " \"ts\": \"2019-01-08 08:57:47:192+0000\",\n" + - " \"params\": {\n" + - " \"resmsgid\": null,\n" + - " \"msgid\": \"7b1cf31f-0b9f-4d88-b405-4d9c61be8081\",\n" + - " \"err\": null,\n" + - " \"status\": \"success\",\n" + - " \"errmsg\": null\n" + - " },\n" + - " \"responseCode\": \"OK\",\n" + - " \"result\": {\n" + - " \"response\": {\n" + - " \"count\": 1,\n" + - " \"content\": [\n" + - " {\n" + - " \"dateTime\": null,\n" + - " \"preferredLanguage\": \"English\",\n" + - " \"approvedBy\": null,\n" + - " \"channel\": \"sunbird-staging\",\n" + - " \"description\": \"default user will be associated with this\",\n" + - " \"updatedDate\": null,\n" + - " \"addressId\": null,\n" + - " \"orgType\": null,\n" + - " \"provider\": null,\n" + - " \"locationId\": null,\n" + - " \"orgCode\": \"defaultRootOrg\",\n" + - " \"theme\": null,\n" + - " \"id\": \"0125134851644620800\",\n" + - " \"communityId\": null,\n" + - " \"isApproved\": null,\n" + - " \"slug\": \"sunbird-staging\",\n" + - " \"identifier\": \"0125134851644620800\",\n" + - " \"thumbnail\": null,\n" + - " \"orgName\": \"defaultRootOrg\",\n" + - " \"updatedBy\": null,\n" + - " \"locationIds\": [],\n" + - " \"externalId\": null,\n" + - " \"isRootOrg\": true,\n" + - " \"rootOrgId\": \"0125134851644620800\",\n" + - " \"approvedDate\": null,\n" + - " \"imgUrl\": null,\n" + - " \"homeUrl\": null,\n" + - " \"orgTypeId\": null,\n" + - " \"isDefault\": null,\n" + - " \"contactDetail\": [],\n" + - " \"createdDate\": \"2018-05-28 16:23:38:330+0000\",\n" + - " \"createdBy\": \"8217108a-6931-491c-9009-1ae95cb0477f\",\n" + - " \"parentOrgId\": null,\n" + - " \"hashTagId\": \"0125134851644620800\",\n" + - " \"noOfMembers\": null,\n" + - " \"status\": 1\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - "}"; - - public static Map getMap(String message) { - return (Map) new Gson().fromJson(message, Map.class); - } - -} diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/DeviceProfileTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/DeviceProfileTest.java deleted file mode 100644 index 616206d3bb..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/DeviceProfileTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.ekstep.ep.samza.system; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.ekstep.ep.samza.domain.DeviceProfile; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class DeviceProfileTest { - - /** - * Method should return the city, country, countryCode - */ - @Test - public void shouldReturnValues() { - - DeviceProfile df = new DeviceProfile("IN", "India", "KA", "Karnatka", "Bengaluru"); - DeviceProfile df2 = new DeviceProfile("IN", "India", "KA", "Karnatka", "Bengaluru"); - Assert.assertEquals("Bengaluru", df.getCity()); - Assert.assertEquals("India", df.getCountry()); - Assert.assertEquals("IN", df.getCountryCode()); - Assert.assertEquals("KA", df.getStateCode()); - } - - - @Test - public void deviceProfileValuesShouldBeEmpty() { - DeviceProfile df = new DeviceProfile(); - Assert.assertEquals("", df.getCity()); - Assert.assertEquals("", df.getCountry()); - Assert.assertEquals("", df.getCountryCode()); - Assert.assertEquals("", df.getStateCode()); - Assert.assertEquals("", df.getDistrictCustom()); - Assert.assertEquals("", df.getstateCodeCustom()); - Assert.assertEquals("", df.getUserDeclaredState()); - Assert.assertEquals("", df.getstateCustomName()); - - } - - @Test - public void deviceProfileValuesShouldAddTheValuesToCache() { - ObjectMapper mapper = new ObjectMapper(); - String json = "{\"country_code\":\"IN\",\"country\":\"India\",\"state_code\":\"KA\",\"state\":\"Karnataka\",\"city\":\"Bengaluru\",\"user_declared_state\":\"Karnataka\",\"user_declared_district\":\"Tumkur\",\"district_custom\":\"BA\"}"; - try { - - // convert JSON string to Map - Map values = mapper.readValue(json, Map.class); - DeviceProfile df = new DeviceProfile(); - df.fromMap(values); - - Assert.assertEquals("Bengaluru", df.getCity()); - Assert.assertEquals("India", df.getCountry()); - Assert.assertEquals("IN", df.getCountryCode()); - Assert.assertEquals("KA", df.getStateCode()); - Assert.assertEquals("BA", df.getDistrictCustom()); - Assert.assertEquals("Karnataka", df.getUserDeclaredState()); - Assert.assertEquals("Tumkur", df.getUserDeclaredDistrict()); - - } catch (IOException e) { - e.printStackTrace(); - } - - - } - -} diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/EventTest.java deleted file mode 100644 index c550fcfff5..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/EventTest.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.ekstep.ep.samza.system; - -import java.util.Map; - -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.task.TelemetryLocationUpdaterConfig; -import org.junit.Assert; -import org.junit.Test; - -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; -import com.google.gson.reflect.TypeToken; -import org.mockito.Mockito; - -public class EventTest { - private TelemetryLocationUpdaterConfig configMock; - @Test - public void shouldReturnMid() { - - Event event = new Event(EventFixture.getMap(EventFixture.ERROR_EVENT)); - Assert.assertEquals("43288930-e54a-230b-b56e-876gnm8712ok", event.mid()); - } - - @Test - public void shouldReturnEid() { - - Event event = new Event(EventFixture.getMap(EventFixture.LOG_EVENT)); - Assert.assertEquals("LOG", event.eid()); - } - - @Test(expected = JsonSyntaxException.class) - public void shouldThrowException() { - new Gson().fromJson(EventFixture.UNPARSABLE_START_EVENT, - new TypeToken>() { - }.getType()); - } - - @Test - public void shouldMarkFailure() { - configMock = Mockito.mock(TelemetryLocationUpdaterConfig.class); - String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}"; - Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class)); - event.markFailure("Invalid Event", configMock); - Map flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class); - Assert.assertNotNull(flagData); - Assert.assertEquals(flagData.get("tr_processed"), false); - Object metaData = event.getMap().get("metadata"); - Assert.assertNotNull(metaData); - } -} diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTaskTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTaskTest.java deleted file mode 100644 index 22a08fd0e4..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTaskTest.java +++ /dev/null @@ -1,385 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import org.apache.commons.lang.StringUtils; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.domain.DeviceProfile; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.util.DeviceProfileCache; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import redis.clients.jedis.Jedis; -import com.fiftyonred.mock_jedis.MockJedis; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; -import static org.junit.Assert.*; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -public class TelemetryLocationUpdaterTaskTest { - - private static final String SUCCESS_TOPIC = "telemetry.with_location"; - private static final String FAILED_TOPIC = "telemetry.failed"; - private static final String MALFORMED_TOPIC = "telemetry.malformed"; - - private MessageCollector collectorMock; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private DeviceProfileCache deviceProfileCacheMock; - private TelemetryLocationUpdaterTask telemetryLocationUpdaterTask; - private Jedis jedisMock = new MockJedis("test"); - - @SuppressWarnings("unchecked") - @Before - public void setUp() { - collectorMock = mock(MessageCollector.class); - TaskContext contextMock = mock(TaskContext.class); - MetricsRegistry metricsRegistry = mock(MetricsRegistry.class); - Counter counter = mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - Config configMock = mock(Config.class); - RedisConnect redisConnectMock = mock(RedisConnect.class); - - deviceProfileCacheMock = mock(DeviceProfileCache.class); - - stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC); - stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC); - stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC); - stub(configMock.getInt("redis.database.userStore.id", 4)).toReturn(4); - - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - stub(envelopeMock.getOffset()).toReturn("2"); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1))); - stub(redisConnectMock.getConnection(4)).toReturn(jedisMock); - - telemetryLocationUpdaterTask = new TelemetryLocationUpdaterTask(configMock, contextMock, deviceProfileCacheMock, redisConnectMock); - } - - @Test - public void shouldSendEventsToSuccessTopicIfDidIsNull() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITHOUT_DID); - stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(null); - telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals("3.0", outputEvent.get("ver")); - assertFalse(outputEvent.containsKey("devicedata")); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(false, flags.get("device_profile_retrieved")); - return true; - } - })); - - } - - @Test - public void shouldSendEventsToSuccessTopicWithUserProfileLocationAsDerived() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT); - // Map uaspec = new HashMap<>(); - Map device_spec = new HashMap<>(); - Long first_access = 1559484698000L; - device_spec.put("os", "Android 6.0"); - device_spec.put("make", "Motorola XT1706"); - /* - uaspec.put("agent", "Mozilla"); - uaspec.put("ver", "5.0"); - uaspec.put("system", "iPad"); - uaspec.put("platform", "AppleWebKit/531.21.10"); - uaspec.put("raw", "Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)"); - */ - DeviceProfile deviceProfile = new DeviceProfile - ("IN", "India", "KA", "Karnataka", - "Bangalore", "Banglore-Custom", "Karnatak-Custom", - "KA-Custom", device_spec, first_access,"Bangalore", - "Karnataka"); - stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile); - - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}"); - telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals("3.0", outputEvent.get("ver")); - Map devicedata = new Gson().fromJson(new Gson().toJson(outputEvent.get("devicedata")), mapType); - - assertEquals("Karnataka", devicedata.get("state")); - assertEquals("Banglore-Custom", devicedata.get("districtcustom")); - assertEquals("IN", devicedata.get("countrycode")); - assertEquals("India", devicedata.get("country")); - assertEquals("KA", devicedata.get("statecode")); - assertEquals("Bangalore", devicedata.get("city")); - assertEquals("KA-Custom", devicedata.get("statecustomcode")); - assertEquals("Karnatak-Custom", devicedata.get("statecustomname")); - assertEquals("IN-KA", devicedata.get("iso3166statecode")); -// assertEquals(Long.valueOf("1559484698000"), Long.valueOf(devicedata.get("firstaccess").toString())); - - Map deviceSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("devicespec")), mapType); - assertEquals("Android 6.0", deviceSpec.get("os")); - assertEquals("Motorola XT1706", deviceSpec.get("make")); - - /* - Map uaSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("uaspec")), mapType); - assertEquals("Mozilla", uaSpec.get("agent")); - assertEquals("5.0", uaSpec.get("ver")); - assertEquals("iPad", uaSpec.get("system")); - assertEquals("AppleWebKit/531.21.10", uaSpec.get("platform")); - assertEquals("Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)", uaSpec.get("raw")); - */ - - Map userDeclared = new Gson().fromJson(new Gson().toJson(devicedata.get("userdeclared")), mapType); - assertEquals("Karnataka", userDeclared.get("state")); - assertEquals("Bangalore", userDeclared.get("district")); - - Map derivedLocationData = new Gson().fromJson(new Gson().toJson(outputEvent.get("derivedlocationdata")), mapType); - assertEquals("Karnataka", derivedLocationData.get("state")); - assertEquals("Bengaluru", derivedLocationData.get("district")); - assertEquals("user-profile", derivedLocationData.get("from")); - - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(true, flags.get("device_profile_retrieved")); - assertEquals(true, flags.get("derived_location_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithUserDeclaredLocationAsDerived() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT); - DeviceProfile deviceProfile = new DeviceProfile - ("IN", "India", "KA", "Karnataka", - "Bangalore", "Banglore-Custom", "Karnatak-Custom", - "KA-Custom", new HashMap<>(), 0L,"Bangalore", - "Karnataka"); - stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile); - - telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals("3.0", outputEvent.get("ver")); - Map devicedata = new Gson().fromJson(new Gson().toJson(outputEvent.get("devicedata")), mapType); - - Map deviceSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("devicespec")), mapType); - assertTrue(deviceSpec.isEmpty()); - - // Map uaSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("uaspec")), mapType); - // assertEquals(true, uaSpec.isEmpty()); - - Map userDeclared = new Gson().fromJson(new Gson().toJson(devicedata.get("userdeclared")), mapType); - assertEquals("Karnataka", userDeclared.get("state")); - assertEquals("Bangalore", userDeclared.get("district")); - - Map derivedLocationData = new Gson().fromJson(new Gson().toJson(outputEvent.get("derivedlocationdata")), mapType); - assertEquals("Karnataka", derivedLocationData.get("state")); - assertEquals("Bangalore", derivedLocationData.get("district")); - assertEquals("user-declared", derivedLocationData.get("from")); - - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(true, flags.get("device_profile_retrieved")); - assertEquals(true, flags.get("derived_location_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithIpLocationAsDerived() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT); - DeviceProfile deviceProfile = new DeviceProfile - ("IN", "India", "KA", "Karnataka", - "Bangalore", "Banglore-Custom", "Karnatak-Custom", - "KA-Custom", new HashMap<>(), 0L,"", - ""); - stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile); - - telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertEquals("3.0", outputEvent.get("ver")); - Map devicedata = new Gson().fromJson(new Gson().toJson(outputEvent.get("devicedata")), mapType); - - Map deviceSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("devicespec")), mapType); - assertTrue(deviceSpec.isEmpty()); - - // Map uaSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("uaspec")), mapType); - // assertEquals(true, uaSpec.isEmpty()); - - Map userDeclared = new Gson().fromJson(new Gson().toJson(devicedata.get("userdeclared")), mapType); - assertEquals("", userDeclared.get("state")); - assertEquals("", userDeclared.get("district")); - - Map derivedLocationData = new Gson().fromJson(new Gson().toJson(outputEvent.get("derivedlocationdata")), mapType); - assertEquals("Karnataka", derivedLocationData.get("state")); - assertEquals("Banglore-Custom", derivedLocationData.get("district")); - assertEquals("ip-resolved", derivedLocationData.get("from")); - - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(true, flags.get("device_profile_retrieved")); - assertEquals(true, flags.get("derived_location_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithOutDeviceProfile() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT); - DeviceProfile deviceProfile = null; - stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile); - - telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(false, flags.get("device_profile_retrieved")); - assertEquals(false, flags.get("derived_location_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendEventsToSuccessTopicWithOutDeviceProfileAndWithUserProfileLocationAsDerived() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT); - DeviceProfile deviceProfile = null; - stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile); - - jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}"); - telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - - Map derivedLocationData = new Gson().fromJson(new Gson().toJson(outputEvent.get("derivedlocationdata")), mapType); - assertEquals("Karnataka", derivedLocationData.get("state")); - assertEquals("Bengaluru", derivedLocationData.get("district")); - assertEquals("user-profile", derivedLocationData.get("from")); - - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(false, flags.get("device_profile_retrieved")); - assertEquals(true, flags.get("derived_location_retrieved")); - return true; - } - })); - } - - @Test - public void deviceProfileFromMapShouldNotFailForEmptyValues() { - Map deviceProfileMap = new HashMap<>(); - deviceProfileMap.put("user_declared_state", "Tamil Nadu"); - deviceProfileMap.put("user_declared_district", "Tiruchirappalli"); - - DeviceProfile deviceProfile = new DeviceProfile().fromMap(deviceProfileMap); - assertEquals("Tamil Nadu", deviceProfile.getUserDeclaredState()); - assertEquals("Tiruchirappalli", deviceProfile.getUserDeclaredDistrict()); - - assertEquals(0, deviceProfile.getDevicespec().size()); - assertTrue(StringUtils.isEmpty(deviceProfile.getState())); - assertTrue(StringUtils.isEmpty(deviceProfile.getCity())); - assertTrue(StringUtils.isEmpty(deviceProfile.getDistrictCustom())); - assertTrue(StringUtils.isEmpty(deviceProfile.getstateCodeCustom())); - assertTrue(StringUtils.isEmpty(deviceProfile.getstateCustomName())); - assertEquals(0L, deviceProfile.getFirstaccess().longValue()); - - assertFalse(deviceProfile.isLocationResolved()); - assertFalse(deviceProfile.isDeviceProfileResolved()); - } - - @Test - public void shouldNotFailIfUserIdIsMissing() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_SUMMARY); - DeviceProfile deviceProfile = null; - stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("099988ce86c4dbb9a4057ff611d38427")).toReturn(deviceProfile); - - telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType); - assertEquals(false, flags.get("device_profile_retrieved")); - assertEquals(false, flags.get("derived_location_retrieved")); - return true; - } - })); - } - - @Test - public void shouldSendEventToFailedTopicIfEventIsNotParseable() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSABLE_START_EVENT); - telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - @Test - public void shouldSendEventToMalformedTopicIfEventIsAnyRandomString() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING); - telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - return true; - } - }; - } -} diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/util/DeviceProfileCacheTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/util/DeviceProfileCacheTest.java deleted file mode 100644 index 87db5b0d83..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/util/DeviceProfileCacheTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.ekstep.ep.samza.util; - -import org.ekstep.ep.samza.domain.DeviceProfile; -import org.junit.Assert; -import org.junit.Test; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.stub; - -public class DeviceProfileCacheTest { - - - private DeviceProfileCache cache; - - @Test - public void shouldReturnNullWhenDeviceIdIsNotPresent() { - cache = mock(DeviceProfileCache.class); - stub(cache.getDeviceProfileFromCache("test")).toReturn(null); - Assert.assertEquals(cache.getDeviceProfileFromCache("564783-5439"), null); - } - - @Test - public void shouldReturnProfileInfo() { - DeviceProfile profile = new DeviceProfile("IN", "India", "KA", "Karnataka", - "Bengaluru"); - cache = mock(DeviceProfileCache.class); - stub(cache.getDeviceProfileFromCache("37654387-543534")).toReturn(profile); - DeviceProfile res = cache.getDeviceProfileFromCache("37654387-543534"); - Assert.assertEquals(res.getCountryCode(), "IN"); - Assert.assertEquals(res.getCountry(), "India"); - Assert.assertEquals(res.getStateCode(), "KA"); - Assert.assertEquals(res.getState(), "Karnataka"); - Assert.assertEquals(res.getCity(), "Bengaluru"); - - } - - -} diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/cdata.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/cdata.json deleted file mode 100755 index 4c7261b025..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/cdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/cdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/cdata/id", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ "type", "id"] - } -} diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/common_empty_sid_uid.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/common_empty_sid_uid.json deleted file mode 100755 index 9ac4103bd6..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/common_empty_sid_uid.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "pdata": { - "$ref": "file:src/test/resources/2.2/pdata.json" - }, - "gdata": { - "$ref": "file:src/test/resources/2.2/gdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "etags": { - "$ref": "file:src/test/resources/2.2/etags.json" - }, - "cdata": { - "$ref": "file:src/test/resources/2.2/cdata.json" - }, - "channel": { - "id": "http://api.ekstep.org/telemetry/2.2/channel", - "type": "string", - "minLength": 1 - } - } -} diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/etags.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/etags.json deleted file mode 100755 index e00c02e041..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/etags.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "object", - "properties": { - "app": { - "id": "http://api.ekstep.org/telemetry/etags/app", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/app/items", - "type": "string" - } - }, - "partner": { - "id": "http://api.ekstep.org/telemetry/etags/partner", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/partner/items", - "type": "string" - } - }, - "dims": { - "id": "http://api.ekstep.org/telemetry/etags/dims", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/dims/items", - "type": "string" - } - } - }, - "additionalProperties": false -} diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/gdata.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/gdata.json deleted file mode 100755 index 1b603a1b7b..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/gdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/ge_error.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/ge_error.json deleted file mode 100755 index 14350be425..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/ge_error.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "file:src/test/resources/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id", - "err", - "eventId", - "data" - ], - "properties": { - "err": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "SYSTEM", - "GENIE", - "GAME", - "GENIESERVICES" - ] - }, - "id": { - "type": "string" - }, - "eventId": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/pdata.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/pdata.json deleted file mode 100755 index cd509bc58b..0000000000 --- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/pdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/pdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/pdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/data-pipeline/telemetry-redacter/pom.xml b/data-pipeline/telemetry-redacter/pom.xml deleted file mode 100644 index 69a1ac02f7..0000000000 --- a/data-pipeline/telemetry-redacter/pom.xml +++ /dev/null @@ -1,190 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - - org.ekstep.ecosystem.jobs - telemetry-redacter - 0.0.1 - jar - TelemetryRedacter - - Telemetry redacter to redact user input values - - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - - com.google.guava - guava - 18.0 - - - it.ozimov - embedded-redis - 0.7.1 - test - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - diff --git a/data-pipeline/telemetry-redacter/src/main/assembly/src.xml b/data-pipeline/telemetry-redacter/src/main/assembly/src.xml deleted file mode 100644 index d3ea2ee0e2..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/telemetry-redacter.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-kv-rocksdb_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:telemetry-redacter - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/telemetry-redacter/src/main/config/telemetry-redacter.properties b/data-pipeline/telemetry-redacter/src/main/config/telemetry-redacter.properties deleted file mode 100644 index 81fe7fda24..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/config/telemetry-redacter.properties +++ /dev/null @@ -1,85 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.TelemetryRedacter - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__telemetry_redacter_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.TelemetryRedacterTask -task.inputs=kafka.__env__.telemetry.assess.redact -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - - - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -redacted.route.topic=__env__.telemetry.raw -nonredacted.route.topic=__env__.telemetry.assess.raw -failed.topic.name=__env__.telemetry.failed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,assess-route-success-count,failed-message-count,skipped-message-count,cache-miss-count,cache-hit-count - -# redis -# redis.host=localhost -redis.host=__metadata_redis_host__ -# redis.port=6379 -redis.port=__metadata_redis_port__ -redis.connection.max=2 -redis.connection.idle.max=2 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 - -# redis database index -redis.contentDB.index=5 \ No newline at end of file diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index 7361424e13..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.reader.NullableValue; - -public class Event extends Events { - - public Event(Map map) { - super(map); - } - - public void clearUserInput() { - if ("ASSESS".equals(eid())) { - telemetry.add("edata.resvalues", new ArrayList<>()); - } else { - telemetry.add("edata.values", new ArrayList<>()); - } - } - - public String questionId() { - if ("ASSESS".equals(eid())) { - NullableValue itemId = telemetry.read("edata.item.id"); - return itemId.isNull() ? null : itemId.value(); - } else { - NullableValue itemId = telemetry.read("edata.target.id"); - return itemId.isNull() ? null : itemId.value(); - } - } - - public List readResValues() { - if ("ASSESS".equals(eid())) { - return telemetry.>read("edata.resvalues").value(); - } else { - return telemetry.>read("edata.values").value(); - } - } - -} diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/service/TelemetryRedacterService.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/service/TelemetryRedacterService.java deleted file mode 100644 index 2ffa144691..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/service/TelemetryRedacterService.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.ekstep.ep.samza.service; - -import static java.text.MessageFormat.format; - -import java.util.Map; - -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.TelemetryRedacterSink; -import org.ekstep.ep.samza.task.TelemetryRedacterSource; -import org.ekstep.ep.samza.util.QuestionDataCache; - -public class TelemetryRedacterService { - - private static Logger LOGGER = new Logger(TelemetryRedacterService.class); - private QuestionDataCache cache = null; - private JobMetrics metrics = null; - - public TelemetryRedacterService(QuestionDataCache cache, JobMetrics metrics) { - this.cache = cache; - this.metrics = metrics; - } - - public void process(TelemetryRedacterSource source, TelemetryRedacterSink sink) { - Event event = null; - try { - boolean redact = false; - event = source.getEvent(); - String questionId = event.questionId(); - if (null == questionId) { - metrics.incSkippedCounter(); - } else { - Map questionData = cache.getData(event.questionId()); - if (null == questionData) { - metrics.incCacheMissCounter(); - } else { - metrics.incCacheHitCounter(); - if (questionData.containsKey("questionType") - && "Registration".equalsIgnoreCase((String) questionData.get("questionType"))) { - redact = true; - } - } - } - if (redact) { - sink.toNonRedactedRoute(event); - event.clearUserInput(); - } - sink.toRedactedRoute(event); - } catch (Exception e) { - LOGGER.error(null, - format("EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:", event), - e); - sink.toErrorTopic(source.getMessage()); - } - } -} diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterConfig.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterConfig.java deleted file mode 100644 index 48bc873baa..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; - -public class TelemetryRedacterConfig { - - private final String JOB_NAME = "TelemetryRedacter"; - - private String redactedRouteTopic; - private String nonRedactedRouteTopic; - private String failedTopic; - - public TelemetryRedacterConfig(Config config) { - redactedRouteTopic = config.get("redacted.route.topic", "telemetry.raw"); - failedTopic = config.get("failed.topic.name", "telemetry.failed"); - nonRedactedRouteTopic = config.get("nonredacted.route.topic", "telemetry.assess.raw"); - } - - public String getRedactedRouteTopic() { - return redactedRouteTopic; - } - - public String getNonRedactedRouteTopic() { - return nonRedactedRouteTopic; - } - - public String jobName() { - return JOB_NAME; - } - - public String failedTopic() { - return failedTopic; - } - -} \ No newline at end of file diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSink.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSink.java deleted file mode 100644 index dbdce6ddbb..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSink.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - -public class TelemetryRedacterSink extends BaseSink { - - private TelemetryRedacterConfig config; - - public TelemetryRedacterSink(MessageCollector collector, JobMetrics metrics, TelemetryRedacterConfig config) { - super(collector, metrics); - this.config = config; - } - - public void toRedactedRoute(Event event) { - toTopic(config.getRedactedRouteTopic(), event.did(), event.getJson()); - metrics.incSuccessCounter(); - } - - public void toNonRedactedRoute(Event event) { - toTopic(config.getNonRedactedRouteTopic(), event.did(), event.getJson()); - metrics.incAssessRouteSuccessCounter(); - } - - public void toErrorTopic(String message) { - toTopic(config.failedTopic(), null, message); - metrics.incFailedCounter(); - } - -} diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSource.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSource.java deleted file mode 100644 index d2912c191e..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSource.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ekstep.ep.samza.task; - -import java.util.Map; - -import org.apache.samza.system.IncomingMessageEnvelope; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; - -import com.google.gson.Gson; - -public class TelemetryRedacterSource { - - static Logger LOGGER = new Logger(TelemetryRedacterSource.class); - - private IncomingMessageEnvelope envelope; - - public TelemetryRedacterSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - return new Event(getMap()); - } - - @SuppressWarnings("unchecked") - private Map getMap() { - String message = (String) envelope.getMessage(); - return (Map) new Gson().fromJson(message, Map.class); - } - - public String getMessage() { - return envelope.toString(); - } - - -} diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterTask.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterTask.java deleted file mode 100644 index 2dfa05711c..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterTask.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.service.TelemetryRedacterService; -import org.ekstep.ep.samza.util.QuestionDataCache; -import org.ekstep.ep.samza.util.RedisConnect; - -public class TelemetryRedacterTask extends BaseSamzaTask { - - static Logger LOGGER = new Logger(TelemetryRedacterTask.class); - private TelemetryRedacterConfig config; - private JobMetrics metrics; - private TelemetryRedacterService service; - private RedisConnect redisConnect; - - public TelemetryRedacterTask(Config config, TaskContext context) { - init(config, context); - } - - public TelemetryRedacterTask() { - - } - - @Override - public void init(Config config, TaskContext context) { - - this.redisConnect = new RedisConnect(config); - this.config = new TelemetryRedacterConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - service = new TelemetryRedacterService(new QuestionDataCache(config, redisConnect), metrics); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) - throws Exception { - - TelemetryRedacterSink sink = new TelemetryRedacterSink(collector, metrics, config); - TelemetryRedacterSource source = new TelemetryRedacterSource(envelope); - service.process(source, sink); - } - -} diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/util/QuestionDataCache.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/util/QuestionDataCache.java deleted file mode 100644 index 21d56e7327..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/util/QuestionDataCache.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.ekstep.ep.samza.util; - -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; - -import org.apache.samza.config.Config; -import org.ekstep.ep.samza.core.Logger; - -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; - -import redis.clients.jedis.Jedis; -import redis.clients.jedis.exceptions.JedisException; - -public class QuestionDataCache { - - private static Logger LOGGER = new Logger(QuestionDataCache.class); - - private RedisConnect redisPool; - private Jedis redisConnection; - @SuppressWarnings("serial") - private Type mapType = new TypeToken>() { - }.getType(); - private Gson gson = new Gson(); - private int databaseIndex; - - public QuestionDataCache(Config config, RedisConnect redisConnect) { - - this.databaseIndex = config.getInt("redis.contentDB.index", 5); - this.redisPool = redisConnect; - this.redisConnection = this.redisPool.getConnection(databaseIndex); - } - - private Map getDataFromCache(String key) { - Map cacheData = new HashMap<>(); - String data = redisConnection.get(key); - if (data != null && !data.isEmpty()) { - cacheData = gson.fromJson(data, mapType); - } - return cacheData; - } - - public Map getData(String key) { - Map cacheDataMap; - try { - cacheDataMap = getDataFromCache(key); - } catch (JedisException ex) { - LOGGER.error("", "Exception when retrieving data from redis cache ", ex); - this.redisConnection.close(); - this.redisConnection.close(); - this.redisConnection = redisPool.getConnection(databaseIndex); - cacheDataMap = getDataFromCache(key); - } - return cacheDataMap; - } -} diff --git a/data-pipeline/telemetry-redacter/src/main/resources/log4j.xml b/data-pipeline/telemetry-redacter/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/telemetry-redacter/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index 16618ba5f3..0000000000 --- a/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,208 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; - -import java.util.Map; - -public class EventFixture { - - public static final String UNPARSABLE_GE_GENIE_UPDATE_EVENT = "{\n" - + " \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n" - + " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + " \"edata\": {\n" + " \"eks\": {\n" - + " \"mode\": \"WIFI\",\n" + " \"ver\": \"12\",\n" + " \"size\": 12.67,\n" - + " \"err\": \"\",\n" + " \"referrer\": [\n" + " {\n" + " \"action\": \"INSTALL\",\n" - + " \"utmsource\": \"Ekstep\",\n" + " \"utmmedium\": \"Portal\",\n" - + " \"utmterm\": \"December 2016\",\n" + " \"utmcontent\": \"Ramayana\",\n" - + " \"utmcampaign\": \"Epics of India\"\n" + " }\n" + " ]\n" + " }\n" + " },\n" - + " \"eid\": \"GE_GENIE_UPDATE\",\n" + " \"gdata\": {\n" + " \"id\": \"genie.android\",\n" - + " \"ver\": \"1.0\"\n" + " },\n" + " \"sid\": \"\",\n" + " \"ets\": 1454064092546,\n" - + " \"uid\": \"\",\n" + " \"ver\": \"2.0\",\n" + " \"cdata\": [\n" + " {\n" - + " \"id\": \"correlationid\",\n" + " \"type\": \"correlationtype\"\n" + " "; - - public static final String RESPONSE_EVENT = "{\"eid\":\"RESPONSE\",\"ets\":1586759864200,\"ver\":\"3.0\",\"mid\":\"RESPONSE:17d97e7cba61bd7c996d3ae71b9706ef\",\"actor\":{\"id\":\"c5b5732b-b8d6-44ad-bf52-d8f7fc513ac3\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.8.7\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"zSizgfO7mzdZ--MhlGCzszgdNhKW6MCp\",\"did\":\"a915339fbca0afddbf537a1792f01791\",\"cdata\":[{\"id\":\"28d1f55313c9c14df8ea86ca1dc73a92\",\"type\":\"ContentSession\"},{\"id\":\"fed7932935f2223870beb120bf58aeed\",\"type\":\"PlaySession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_2129946748092334081914\",\"type\":\"Content\",\"ver\":\"2\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"target\":{\"id\":\"do_21299463432309145612386\",\"ver\":\"1.0\",\"type\":\"AssessmentItem\"},\"type\":\"CHOOSE\",\"values\":[{\"option0\":\"text\"}]}}"; - - public static final String ASSESS_EVENT = "{" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1568891738245,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:135815023ec32a430632ba5d7f84fe18\",\n" + - " \"actor\": {\n" + - " \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124784842112040965\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" + - " \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128415652377067521127\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"012846671379595264119\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124784842112040965\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_212686723743318016173\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128415652377067521127\",\n" + - " \"l2\": \"do_2128415660716359681128\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124784842112040965\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"id\": \"801ae93c-8807-4be5-8853-dd49362d8776\",\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"What is the Full form of WHO..?\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 4\n" + - " }\n" + - " }"; - - public static final String ASSESS_EVENT_WITHOUT_QID = "{" + - " \"eid\": \"ASSESS\",\n" + - " \"ets\": 1568891738245,\n" + - " \"ver\": \"3.1\",\n" + - " \"mid\": \"ASSESS:135815023ec32a430632ba5d7f84fe18\",\n" + - " \"actor\": {\n" + - " \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"0124784842112040965\",\n" + - " \"pdata\": {\n" + - " \"id\": \"staging.diksha.portal\",\n" + - " \"ver\": \"2.4.0\",\n" + - " \"pid\": \"sunbird-portal.contentplayer\"\n" + - " },\n" + - " \"env\": \"contentplayer\",\n" + - " \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" + - " \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"do_2128415652377067521127\",\n" + - " \"type\": \"course\"\n" + - " },\n" + - " {\n" + - " \"type\": \"batch\",\n" + - " \"id\": \"012846671379595264119\"\n" + - " },\n" + - " {\n" + - " \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" + - " \"type\": \"ContentSession\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0124784842112040965\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_212686723743318016173\",\n" + - " \"type\": \"Content\",\n" + - " \"ver\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_2128415652377067521127\",\n" + - " \"l2\": \"do_2128415660716359681128\"\n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"0124784842112040965\"\n" + - " ],\n" + - " \"edata\": {\n" + - " \"item\": {\n" + - " \"maxscore\": 1,\n" + - " \"type\": \"mcq\",\n" + - " \"exlength\": 0,\n" + - " \"params\": [\n" + - " {\n" + - " \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" + - " },\n" + - " {\n" + - " \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" + - " }\n" + - " ],\n" + - " \"uri\": \"\",\n" + - " \"title\": \"What is the Full form of WHO..?\\n\",\n" + - " \"mmc\": [],\n" + - " \"mc\": [],\n" + - " \"desc\": \"\"\n" + - " },\n" + - " \"index\": 1,\n" + - " \"pass\": \"No\",\n" + - " \"score\": 0,\n" + - " \"resvalues\": [\n" + - " {\n" + - " \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" + - " }\n" + - " ],\n" + - " \"duration\": 4\n" + - " }\n" + - " }"; - - - public static Map getMap(String message) { - return (Map) new Gson().fromJson(message, Map.class); - } - -} diff --git a/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/task/TelemetryRedacterTaskTest.java b/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/task/TelemetryRedacterTaskTest.java deleted file mode 100644 index ca8a37a370..0000000000 --- a/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/task/TelemetryRedacterTaskTest.java +++ /dev/null @@ -1,196 +0,0 @@ -package org.ekstep.ep.samza.task; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.stub; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; - -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; - -import com.google.gson.Gson; - -import redis.clients.jedis.Jedis; -import redis.embedded.RedisServer; - -public class TelemetryRedacterTaskTest { - - private MessageCollector collectorMock; - private TaskContext contextMock; - private MetricsRegistry metricsRegistry; - private Counter counter; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private Config configMock; - private TelemetryRedacterTask telemetryRedacterTask; - private RedisServer redisServer; - private RedisConnect redisConnect; - - @Before - public void setUp() throws IOException { - - redisServer = new RedisServer(6379); - redisServer.start(); - collectorMock = mock(MessageCollector.class); - contextMock = Mockito.mock(TaskContext.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = Mockito.mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - configMock = Mockito.mock(Config.class); - when(configMock.get("redis.host", "localhost")).thenReturn("localhost"); - when(configMock.getInt("redis.port", 6379)).thenReturn(6379); - when(configMock.getInt("redis.connection.max", 2)).thenReturn(2); - when(configMock.getInt("redis.connection.idle.max", 2)).thenReturn(2); - when(configMock.getInt("redis.connection.idle.min", 1)).thenReturn(1); - when(configMock.getInt("redis.connection.minEvictableIdleTimeSeconds", 120)).thenReturn(120); - when(configMock.getInt("redis.connection.timeBetweenEvictionRunsSeconds", 300)).thenReturn(300); - when(configMock.getInt("redis.contentDB.index", 5)).thenReturn(5); - redisConnect = new RedisConnect(configMock); - - stub(configMock.get("redacted.route.topic", "telemetry.raw")).toReturn("telemetry.raw"); - stub(configMock.get("nonredacted.route.topic", "telemetry.assess.raw")).toReturn("telemetry.assess.raw"); - stub(configMock.get("failed.topic.name", "telemetry.failed")).toReturn("telemetry.failed"); - - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - - telemetryRedacterTask = new TelemetryRedacterTask(configMock, contextMock); - } - - @After - public void tearDown() { - redisServer.stop(); - } - - @Test - public void shouldSendEventToErroTopicIfEventIsNotParseable() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSABLE_GE_GENIE_UPDATE_EVENT); - telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic("telemetry.failed"))); - } - - @Test - public void shouldRedactEventAndSendToRawTopic() throws Exception { - - Jedis jedis = redisConnect.getConnection(5); - jedis.set("801ae93c-8807-4be5-8853-dd49362d8776", "{\"questionType\":\"Registration\"}"); - jedis.close(); - stub(envelopeMock.getMessage()).toReturn(EventFixture.ASSESS_EVENT); - telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock, times(2)).send(argThat(validateRedactedTopic(true))); - } - - @Test - public void shouldRedactEventAndSendToRawTopic2() throws Exception { - - Jedis jedis = redisConnect.getConnection(5); - jedis.set("do_21299463432309145612386", "{\"questionType\":\"Registration\"}"); - jedis.close(); - stub(envelopeMock.getMessage()).toReturn(EventFixture.RESPONSE_EVENT); - telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock, times(2)).send(argThat(validateRedactedTopic(true))); - } - - @Test - public void shouldNotRedactEventAndSendToRawTopic1() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.ASSESS_EVENT_WITHOUT_QID); - telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock, times(1)).send(argThat(validateOutputTopic("telemetry.raw"))); - } - - @Test - public void shouldNotRedactEventAndSendToRawTopic2() throws Exception { - - Jedis jedis = redisConnect.getConnection(5); - jedis.del("801ae93c-8807-4be5-8853-dd49362d8776"); - jedis.close(); - - stub(envelopeMock.getMessage()).toReturn(EventFixture.ASSESS_EVENT); - telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock, times(1)).send(argThat(validateOutputTopic("telemetry.raw"))); - } - - @Test - public void shouldNotRedactEventAndSendToRawTopic3() throws Exception { - - Jedis jedis = redisConnect.getConnection(5); - jedis.set("801ae93c-8807-4be5-8853-dd49362d8776", "{\"questionType\":\"Survey\"}"); - jedis.close(); - stub(envelopeMock.getMessage()).toReturn(EventFixture.ASSESS_EVENT); - telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic("telemetry.raw"))); - } - - public ArgumentMatcher validateRedactedTopic(final boolean redacted) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertTrue(Arrays.asList("telemetry.raw", "telemetry.assess.raw").contains(systemStream.getStream())); - - if ("telemetry.raw".equals(systemStream.getStream())) { - String message = (String) outgoingMessageEnvelope.getMessage(); - Map eventMap = new Gson().fromJson(message, Map.class); - Event event = new Event(eventMap); - if (redacted) { - assertEquals(0, event.readResValues().size()); - } - } - if ("telemetry.assess.raw".equals(systemStream.getStream())) { - System.out.println("Redacted and message sent to raw stream"); - String message = (String) outgoingMessageEnvelope.getMessage(); - Map eventMap = new Gson().fromJson(message, Map.class); - Event event = new Event(eventMap); - assertEquals(1, event.readResValues().size()); - } - - return true; - } - }; - } - - public ArgumentMatcher validateOutputTopic(final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - - return true; - } - }; - } - -} diff --git a/data-pipeline/telemetry-router/pom.xml b/data-pipeline/telemetry-router/pom.xml deleted file mode 100644 index 0a2bcf01dc..0000000000 --- a/data-pipeline/telemetry-router/pom.xml +++ /dev/null @@ -1,184 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - - org.ekstep.ecosystem.jobs - telemetry-router - 0.1.6 - jar - TelemetryRouter - - Telemetry router based on event priority - - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - - com.google.guava - guava - 18.0 - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - diff --git a/data-pipeline/telemetry-router/src/main/assembly/src.xml b/data-pipeline/telemetry-router/src/main/assembly/src.xml deleted file mode 100644 index 3dd4a548b0..0000000000 --- a/data-pipeline/telemetry-router/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/telemetry-router.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-kv-rocksdb_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:telemetry-router - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/telemetry-router/src/main/config/telemetry-router.properties b/data-pipeline/telemetry-router/src/main/config/telemetry-router.properties deleted file mode 100644 index 7775b12bc2..0000000000 --- a/data-pipeline/telemetry-router/src/main/config/telemetry-router.properties +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.TelemetryRouter - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__telemetry_router_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.TelemetryRouterTask -task.inputs=kafka.__env__.telemetry.unique,kafka.__env__.telemetry.derived.unique -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - - - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -systems.kafka.streams.__env__.telemetry.derived.unique.samza.priority=1 - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -router.events.primary.route.topic=__env__.telemetry.sink -router.events.secondary.route.events=__tr_secondary_route_events__ -router.events.secondary.route.topic=__env__.telemetry.log -router.events.audit.route.topic=__env__.telemetry.audit -router.events.share.route.topic=__env__.telemetry.share -output.failed.topic.name=__env__.telemetry.failed -output.malformed.topic.name=__env__.telemetry.malformed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -router.events.log.topic.name=__env__.events.log -router.events.error.topic.name=__env__.events.error -pipeline.metrics.list=error-message-count,primary-route-success-count,secondary-route-success-count,audit-route-success-count,log-route-success-count,error-route-success-count diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index c6a879ac00..0000000000 --- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.reader.NullableValue; -import org.ekstep.ep.samza.task.TelemetryRouterConfig; - -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -public class Event extends Events { - - - public Event(Map map) { - super(map); - } - - public void markFailure(String error, TelemetryRouterConfig config) { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.tr_processed", false); - - telemetry.addFieldIfAbsent("metadata", new HashMap()); - telemetry.add("metadata.tr_error", error); - telemetry.add("metadata.src", config.jobName()); - } - - public void setTimestamp() { - Double ets = safelyParse(path.ets()); - SimpleDateFormat simple = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - - if (ets != null) { - String updatedTs = simple.format(new Date(ets.longValue())); - telemetry.add(path.ts(), updatedTs); - } else { - telemetry.add(path.ts(), simple.format(new Date())); - } - } - - private Double safelyParse(String etsField) { - try { - NullableValue time = telemetry.read(etsField); - return time.value(); - } catch (ClassCastException e) { - NullableValue timeInLong = telemetry.read(etsField); - return Double.valueOf(timeInLong.value()); - } - } - -} diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/service/TelemetryRouterService.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/service/TelemetryRouterService.java deleted file mode 100644 index 6d021465a3..0000000000 --- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/service/TelemetryRouterService.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.google.gson.JsonSyntaxException; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.TelemetryRouterConfig; -import org.ekstep.ep.samza.task.TelemetryRouterSink; -import org.ekstep.ep.samza.task.TelemetryRouterSource; - -import java.util.List; - -import static java.text.MessageFormat.format; - -public class TelemetryRouterService { - - private static Logger LOGGER = new Logger(TelemetryRouterService.class); - private final TelemetryRouterConfig config; - - public TelemetryRouterService(TelemetryRouterConfig config) { - this.config = config; - } - - public void process(TelemetryRouterSource source, TelemetryRouterSink sink) { - Event event = null; - try { - event = source.getEvent(); - String eid = event.eid(); - List secondaryRouteEvents = this.config.getSecondaryRouteEvents(); - if (secondaryRouteEvents.contains(eid)) { - if("LOG".equalsIgnoreCase(eid)) { sink.toLogRoute(event); } - else if("ERROR".equalsIgnoreCase(eid)) { sink.toErrorRoute(event); } - else { sink.toSecondaryRoute(event); } - } else { - if ("SHARE".equalsIgnoreCase(eid)) { - sink.toShareEventRouter(event); - } else { - sink.toPrimaryRoute(event); - } - } - if ("AUDIT".equalsIgnoreCase(eid)) { - sink.toAuditRoute(event); - } - } catch (JsonSyntaxException e) { - LOGGER.error(null, "INVALID EVENT: " + source.getMessage()); - sink.toMalformedTopic(source.getMessage()); - } catch (Exception e) { - LOGGER.error(null, - format("EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:", - event), - e); - sink.toErrorTopic(event, e.getMessage()); - } - } -} diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterConfig.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterConfig.java deleted file mode 100644 index 209061f957..0000000000 --- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterConfig.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.ekstep.ep.samza.task; - - -import org.apache.samza.config.Config; - -import java.util.Arrays; -import java.util.List; - -public class TelemetryRouterConfig { - - private final String JOB_NAME = "TelemetryRouter"; - - private String failedTopic; - private String primaryRouteTopic; - private String secondaryRouteTopic; - private String secondaryRouteEvents; - private String malformedTopic; - private String auditRouteTopic; - private String shareEventRouterTopic; - private String logRouteTopic; - private String errorRouteTopic; - - public TelemetryRouterConfig(Config config) { - failedTopic = config.get("output.failed.topic.name", "telemetry.failed"); - primaryRouteTopic = config.get("router.events.primary.route.topic", "telemetry.sink"); - secondaryRouteEvents = config.get("router.events.secondary.route.events", "LOG,ERROR"); - secondaryRouteTopic = config.get("router.events.secondary.route.topic", "telemetry.log"); - auditRouteTopic = config.get("router.events.audit.route.topic", "telemetry.audit"); - shareEventRouterTopic = config.get("router.events.share.route.topic", "telemetry.share"); - malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed"); - logRouteTopic = config.get("router.events.log.topic.name","events.log"); - errorRouteTopic = config.get("router.events.error.topic.name","events.error"); - - } - - public String getPrimaryRouteTopic() { - return primaryRouteTopic; - } - - - public String getSecondaryRouteTopic() { - return secondaryRouteTopic; - } - - public String getLogRouteTopic() { - return logRouteTopic; - } - - public String getErrorRouteTopic() { - return errorRouteTopic; - } - - - public List getSecondaryRouteEvents() { - String[] events = this.secondaryRouteEvents.split(","); - return Arrays.asList(events); - } - - public String getAuditRouteTopic() { - return auditRouteTopic; - } - - public String getShareEventRouterTopic() { - return shareEventRouterTopic; - } - - - public String failedTopic() { - return failedTopic; - } - - public String malformedTopic() { - return malformedTopic; - } - - public String jobName() { - return JOB_NAME; - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSink.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSink.java deleted file mode 100644 index 7934d1412a..0000000000 --- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSink.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - -public class TelemetryRouterSink extends BaseSink { - - private TelemetryRouterConfig config; - - public TelemetryRouterSink(MessageCollector collector, JobMetrics metrics, TelemetryRouterConfig config) { - super(collector, metrics); - this.config = config; - } - - public void toPrimaryRoute(Event event) { - event.setTimestamp(); - toTopic(config.getPrimaryRouteTopic(), event.did(), event.getJson()); - metrics.incPrimaryRouteSuccessCounter(); - } - - public void toErrorTopic(Event event, String errorMessage) { - event.markFailure(errorMessage, config); - toTopic(config.failedTopic(), event.mid(), event.getJson()); - metrics.incErrorCounter(); - } - - public void toMalformedTopic(String message) { - toTopic(config.malformedTopic(), null, message); - metrics.incErrorCounter(); - } - - public void toSecondaryRoute(Event event) { - toTopic(config.getSecondaryRouteTopic(), event.did(), event.getJson()); - metrics.incSecondaryRouteSuccessCounter(); - } - - public void toLogRoute(Event event) { - toTopic(config.getLogRouteTopic(), event.did(), event.getJson()); - metrics.incLogRouteSuccessCounter(); - } - - public void toErrorRoute(Event event) { - toTopic(config.getErrorRouteTopic(), event.did(), event.getJson()); - metrics.incErrorRouteSuccessCounter(); - } - - public void toAuditRoute(Event event) { - toTopic(config.getAuditRouteTopic(), event.mid(), event.getJson()); - metrics.incAuditRouteSuccessCounter(); - } - - public void toShareEventRouter(Event event) { - toTopic(config.getShareEventRouterTopic(), event.mid(), event.getJson()); - metrics.incShareEventRouteSuccessCounter(); - } - -} diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSource.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSource.java deleted file mode 100644 index 48f0a98eb1..0000000000 --- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSource.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ekstep.ep.samza.task; - -import java.util.Map; - -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; - -import com.google.gson.Gson; - -public class TelemetryRouterSource { - static Logger LOGGER = new Logger(TelemetryRouterSource.class); - - private IncomingMessageEnvelope envelope; - - public TelemetryRouterSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - return new Event(getMap()); - } - - @SuppressWarnings("unchecked") - private Map getMap() { - String message = (String) envelope.getMessage(); - return (Map) new Gson().fromJson(message, Map.class); - } - - public String getMessage() { - return envelope.toString(); - } - - -} diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterTask.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterTask.java deleted file mode 100644 index da28eee5c1..0000000000 --- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterTask.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.service.TelemetryRouterService; - -public class TelemetryRouterTask extends BaseSamzaTask { - - static Logger LOGGER = new Logger(TelemetryRouterTask.class); - private TelemetryRouterConfig config; - private JobMetrics metrics; - private TelemetryRouterService service; - - public TelemetryRouterTask(Config config, TaskContext context) { - init(config, context); - } - - public TelemetryRouterTask() { - - } - - @Override - public void init(Config config, TaskContext context) { - - this.config = new TelemetryRouterConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - service = new TelemetryRouterService(this.config); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) - throws Exception { - - TelemetryRouterSink sink = new TelemetryRouterSink(collector, metrics, config); - TelemetryRouterSource source = new TelemetryRouterSource(envelope); - service.process(source, sink); - } - -} diff --git a/data-pipeline/telemetry-router/src/main/resources/log4j.xml b/data-pipeline/telemetry-router/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/telemetry-router/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index ce9a78a2cd..0000000000 --- a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,340 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; - -import java.util.Map; - -public class EventFixture { - - public static final String LOG_EVENT = "{\n" + - " \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" + - " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"data\": \"\",\n" + - " \"err\": \"10\",\n" + - " \"eventId\": \"GE_SIGNUP\",\n" + - " \"id\": \"2131165210\",\n" + - " \"type\": \"GENIE\"\n" + - " }\n" + - " },\n" + - " \"eid\": \"LOG\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genie.android\",\n" + - " \"ver\": \"2.2.15\"\n" + - " },\n" + - " \"pdata\": {\n" + - " \"id\": \"genie\",\n" + - " \"ver\": \"2.0\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"ets\": 1454064092546,\n" + - " \"uid\": \"\",\n" + - " \"ver\": \"2.2\",\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"correlationid\",\n" + - " \"type\": \"correlationtype\"\n" + - " }\n" + - " ]\n" + - "}"; - public static final String ERROR_EVENT = "{\n" + - " \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" + - " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + - " \"eid\": \"ERROR\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genie.android\",\n" + - " \"ver\": \"2.2.15\"\n" + - " },\n" + - " \"pdata\": {\n" + - " \"id\": \"genie\",\n" + - " \"ver\": \"2.0\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"ets\": 1454064092546,\n" + - " \"uid\": \"\",\n" + - " \"ver\": \"2.2\",\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"correlationid\",\n" + - " \"type\": \"correlationtype\"\n" + - " }\n" + - " ]\n" + - "}"; - public static final String UNPARSABLE_START_EVENT = "{\n" + - " \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n" + - " \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"mode\": \"WIFI\",\n" + - " \"ver\": \"12\",\n" + - " \"size\": 12.67,\n" + - " \"err\": \"\",\n" + - " \"referrer\": [\n" + - " {\n" + - " \"action\": \"INSTALL\",\n" + - " \"utmsource\": \"Ekstep\",\n" + - " \"utmmedium\": \"Portal\",\n" + - " \"utmterm\": \"December 2016\",\n" + - " \"utmcontent\": \"Ramayana\",\n" + - " \"utmcampaign\": \"Epics of India\"\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " \"eid\": \"START\",\n" + - " \"gdata\": {\n" + - " \"id\": \"genie.android\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"sid\": \"\",\n" + - " \"ets\": 1454064092546,\n" + - " \"uid\": \"\",\n" + - " \"ver\": \"2.0\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"correlationid\",\n" + - " \"type\": \"correlationtype\"\n" + - " "; - public static final String START_EVENT = "{\n" + - " \"cdata\": [],\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"did\": \"0427fedf56eea1c8a127d876fd1907ffb245684f\",\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"extype\": \"\",\n" + - " \"id\": \"\",\n" + - " \"pos\": [],\n" + - " \"stageid\": \"Genie-TelemetrySync\",\n" + - " \"subtype\": \"ManualSync-Success\",\n" + - " \"tid\": \"\",\n" + - " \"type\": \"OTHER\",\n" + - " \"uri\": \"\",\n" + - " \"values\": [\n" + - " {\n" + - " \"SizeOfFileInKB\": \"0.81\"\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " \"eid\": \"GE_INTERACT\",\n" + - " \"etags\": {\n" + - " \"app\": []\n" + - " },\n" + - " \"ets\": 1506328824238,\n" + - " \"gdata\": {\n" + - " \"id\": \"org.ekstep.genieservices.qa\",\n" + - " \"ver\": \"6.5.localdev-debug\"\n" + - " },\n" + - " \"mid\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\",\n" + - " \"pdata\": {\n" + - " \"id\": \"genie\",\n" + - " \"ver\": \"6.5.localdev-debug\"\n" + - " },\n" + - " \"sid\": \"e9662bb5-cf08-42d6-ad11-700b23566961\",\n" + - " \"ts\": \"2017-09-25T08:40:24.238+0000\",\n" + - " \"uid\": \"03762014-f67b-466b-bf20-467f46542563\",\n" + - " \"ver\": \"2.2\",\n" + - " \"@version\": \"1\",\n" + - " \"@timestamp\": \"2017-09-25T08:35:44.037Z\",\n" + - " \"metadata\": {\n" + - " \"checksum\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\"\n" + - " },\n" + - " \"uuid\": \"7f2c9f88-cbf7-4527-9f8d-667d4dde0d1c1\",\n" + - " \"key\": \"03762014-f67b-466b-bf20-467f46542563\",\n" + - " \"type\": \"events\",\n" + - " \"flags\": {\n" + - " \"dd_processed\": true\n" + - " }\n" + - "}"; - - public static final String AUDIT_EVENT ="{\n" + - " \"actor\": {\n" + - " \"type\": \"Consumer\",\n" + - " \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" + - " },\n" + - " \"eid\": \"AUDIT\",\n" + - " \"edata\": {\n" + - " \"state\": \"Create\",\n" + - " \"props\": [\n" + - " \"firstName\",\n" + - " \"email\",\n" + - " \"emailVerified\",\n" + - " \"id\",\n" + - " \"userId\",\n" + - " \"createdBy\",\n" + - " \"rootOrgId\",\n" + - " \"channel\",\n" + - " \"userType\",\n" + - " \"roles\",\n" + - " \"phoneVerified\",\n" + - " \"isDeleted\",\n" + - " \"createdDate\",\n" + - " \"status\",\n" + - " \"userName\",\n" + - " \"loginId\",\n" + - " \"externalIds\"\n" + - " ]\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1561739260405,\n" + - " \"context\": {\n" + - " \"channel\": \"0126684405014528002\",\n" + - " \"pdata\": {\n" + - " \"pid\": \"learner-service\",\n" + - " \"ver\": \"2.0.0\",\n" + - " \"id\": \"prod.diksha.learning.service\"\n" + - " },\n" + - " \"env\": \"User\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"cf2bdab7-0778-4b8e-bfa5-fa7df36a5d19\"\n" + - " },\n" + - " {\n" + - " \"type\": \"SignupType\",\n" + - " \"id\": \"google\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Source\",\n" + - " \"id\": \"android\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Request\",\n" + - " \"id\": \"a5f58660-99c1-11e9-8ef2-cdfa267be9f0\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0126684405014528002\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"1561739260405.1ab659ef-25b5-4a26-bd94-ff2daef86b62\",\n" + - " \"object\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"cf2bdab7-0778-4b8e-bfa5-fa7df36a5d19\"\n" + - " },\n" + - " \"syncts\": 1561739269966,\n" + - " \"@timestamp\": \"2019-06-28T16:27:49.966Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String START_EVENT_WITHOUT_ETS = "{\n" + - " \"cdata\": [],\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"did\": \"0427fedf56eea1c8a127d876fd1907ffb245684f\",\n" + - " \"edata\": {\n" + - " \"eks\": {\n" + - " \"extype\": \"\",\n" + - " \"id\": \"\",\n" + - " \"pos\": [],\n" + - " \"stageid\": \"Genie-TelemetrySync\",\n" + - " \"subtype\": \"ManualSync-Success\",\n" + - " \"tid\": \"\",\n" + - " \"type\": \"OTHER\",\n" + - " \"uri\": \"\",\n" + - " \"values\": [\n" + - " {\n" + - " \"SizeOfFileInKB\": \"0.81\"\n" + - " }\n" + - " ]\n" + - " }\n" + - " },\n" + - " \"eid\": \"GE_INTERACT\",\n" + - " \"etags\": {\n" + - " \"app\": []\n" + - " },\n" + - " \"gdata\": {\n" + - " \"id\": \"org.ekstep.genieservices.qa\",\n" + - " \"ver\": \"6.5.localdev-debug\"\n" + - " },\n" + - " \"mid\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\",\n" + - " \"pdata\": {\n" + - " \"id\": \"genie\",\n" + - " \"ver\": \"6.5.localdev-debug\"\n" + - " },\n" + - " \"sid\": \"e9662bb5-cf08-42d6-ad11-700b23566961\",\n" + - " \"ts\": \"2017-09-25T08:40:24.238+0000\",\n" + - " \"uid\": \"03762014-f67b-466b-bf20-467f46542563\",\n" + - " \"ver\": \"2.2\",\n" + - " \"@version\": \"1\",\n" + - " \"@timestamp\": \"08:35:44.037\",\n" + - " \"metadata\": {\n" + - " \"checksum\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\"\n" + - " },\n" + - " \"uuid\": \"7f2c9f88-cbf7-4527-9f8d-667d4dde0d1c1\",\n" + - " \"key\": \"03762014-f67b-466b-bf20-467f46542563\",\n" + - " \"type\": \"events\",\n" + - " \"flags\": {\n" + - " \"dd_processed\": true\n" + - " }\n" + - "}"; - - public static final String INVALID_EVENT ="{\n" + - " \"actor\": {\n" + - " \"type\": \"Consumer\",\n" + - " \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"state\": \"\",\n" + - " \"props\": [\n" + - " ]\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1561739260405,\n" + - " \"context\": {\n" + - " \"channel\": \"0126684405014528002\",\n" + - " \"pdata\": {\n" + - " \"pid\": \"learner-service\",\n" + - " \"ver\": \"2.0.0\",\n" + - " \"id\": \"prod.diksha.learning.service\"\n" + - " },\n" + - " \"env\": \"User\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"cf2bdab7-0778-4b8e-bfa5-fa7df36a5d19\"\n" + - " },\n" + - " {\n" + - " \"type\": \"SignupType\",\n" + - " \"id\": \"google\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Source\",\n" + - " \"id\": \"android\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Request\",\n" + - " \"id\": \"a5f58660-99c1-11e9-8ef2-cdfa267be9f0\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0126684405014528002\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"1561739260405.1ab659ef-25b5-4a26-bd94-ff2daef86b62\",\n" + - " \"object\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"cf2bdab7-0778-4b8e-bfa5-fa7df36a5d19\"\n" + - " },\n" + - " \"syncts\": 1561739269966,\n" + - " \"@timestamp\": \"2019-06-28T16:27:49.966Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String ANY_STRING = "Hey Samza, Whats Up?"; - public static final String EMPTY_JSON = "{}"; - public static final String SHARE_EVENT = "{\"ver\":\"3.0\",\"eid\":\"SHARE\",\"mid\":\"02ba33e5-15fe-4ec5-b360-3d03429fae84\",\"ets\":1577278681178,\"actor\":{\"type\":\"User\",\"id\":\"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"},\"context\":{\"cdata\":[{\"id\":\"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\"type\":\"API\"},{\"id\":\"SearchResult\",\"type\":\"Section\"}],\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"pdata\":{\"id\":\"prod.diksha.app\",\"pid\":\"sunbird.app\",\"ver\":\"2.3.162\"},\"env\":\"app\",\"sid\":\"82e41d87-e33f-4269-aeae-d56394985599\",\"did\":\"1b17c32bad61eb9e33df281eecc727590d739b2b\"},\"edata\":{\"dir\":\"In\",\"type\":\"File\",\"items\":[{\"origin\":{\"id\":\"1b17c32bad61eb9e33df281eecc727590d739b2b\",\"type\":\"Device\"},\"id\":\"do_312785709424099328114191\",\"type\":\"CONTENT\",\"ver\":\"1\",\"params\":[{\"transfers\":0,\"size\":21084308}]},{\"origin\":{\"id\":\"1b17c32bad61eb9e33df281eecc727590d739b2b\",\"type\":\"Device\"},\"id\":\"do_31277435209002188818711\",\"type\":\"CONTENT\",\"ver\":\"18\",\"params\":[{\"transfers\":12,\"size\":\"123\"}]},{\"origin\":{\"id\":\"1b17c32bad61eb9e33df281eecc727590d739b2b\",\"type\":\"Device\"},\"id\":\"do_31278794857559654411554\",\"type\":\"TextBook\",\"ver\":\"1\"}]},\"object\":{\"id\":\"do_312528116260749312248818\",\"type\":\"TextBook\",\"version\":\"10\",\"rollup\":{}},\"syncts\":1577278682630,\"@timestamp\":\"2019-12-25T12:58:02.630Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\"}"; - - public static Map getMap(String message) { - return (Map) new Gson().fromJson(message, Map.class); - } - -} diff --git a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java deleted file mode 100644 index ef488947cd..0000000000 --- a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ekstep.ep.samza.system; - -import java.util.Map; - -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.junit.Assert; -import org.junit.Test; - -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; -import com.google.gson.reflect.TypeToken; - -public class EventTest { - - @Test - public void shouldReturnMid() { - - Event event = new Event(EventFixture.getMap(EventFixture.ERROR_EVENT)); - Assert.assertEquals("43288930-e54a-230b-b56e-876gnm8712ok", event.mid()); - } - - @Test - public void shouldReturnEid() { - - Event event = new Event(EventFixture.getMap(EventFixture.LOG_EVENT)); - Assert.assertEquals("LOG", event.eid()); - } - - @Test(expected = JsonSyntaxException.class) - public void shouldThrowException() { - new Gson().fromJson(EventFixture.UNPARSABLE_START_EVENT, - new TypeToken>() { - }.getType()); - } -} diff --git a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/task/TelemetryRouterTaskTest.java b/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/task/TelemetryRouterTaskTest.java deleted file mode 100644 index d502416c7d..0000000000 --- a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/task/TelemetryRouterTaskTest.java +++ /dev/null @@ -1,260 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; - -import java.lang.reflect.Type; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -public class TelemetryRouterTaskTest { - - private static final String PRIMARY_TOPIC = "telemetry.sink"; - private static final String FAILED_TOPIC = "telemetry.failed"; - private static final String SECONDARY_TOPIC = "telemetry.log"; - private static final String MALFORMED_TOPIC = "telemetry.malformed"; - private static final String AUDIT_TOPIC = "telemetry.audit"; - private static final String SHARE_EVENT_TOPIC = "telemetry.share"; - private static final String LOG_TOPIC = "events.log"; - private static final String ERROR_TOPIC = "events.error"; - - private MessageCollector collectorMock; - private TaskContext contextMock; - private MetricsRegistry metricsRegistry; - private Counter counter; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private Config configMock; - private TelemetryRouterTask telemetryRouterTask; - - @Before - public void setUp() { - collectorMock = mock(MessageCollector.class); - contextMock = Mockito.mock(TaskContext.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - counter = Mockito.mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - configMock = Mockito.mock(Config.class); - - stub(configMock.get("router.events.primary.route.topic", PRIMARY_TOPIC)).toReturn(PRIMARY_TOPIC); - stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC); - stub(configMock.get("router.events.secondary.route.topic", SECONDARY_TOPIC)).toReturn(SECONDARY_TOPIC); - stub(configMock.get("router.events.audit.route.topic", AUDIT_TOPIC)).toReturn(AUDIT_TOPIC); - stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC); - stub(configMock.get("router.events.share.route.topic", SHARE_EVENT_TOPIC)).toReturn(SHARE_EVENT_TOPIC); - stub(configMock.get("router.events.log.topic.name", LOG_TOPIC)).toReturn(LOG_TOPIC); - stub(configMock.get("router.events.error.topic.name", ERROR_TOPIC)).toReturn(ERROR_TOPIC); - - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - stub(envelopeMock.getOffset()).toReturn("2"); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1))); - -// telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - } - - @Test - public void shouldSendLOGEventToLogEventsRoute() throws Exception { - - stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG"); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - - stub(envelopeMock.getMessage()).toReturn(EventFixture.LOG_EVENT); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), LOG_TOPIC))); - } - - @Test - public void shouldSendERROREventToPrimaryRoute() throws Exception { - - stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG"); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - - stub(envelopeMock.getMessage()).toReturn(EventFixture.ERROR_EVENT); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), PRIMARY_TOPIC))); - - } - - @Test - public void shouldSendERROREventToErrorEventsRoute() throws Exception { - - stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG,ERROR"); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - - stub(envelopeMock.getMessage()).toReturn(EventFixture.ERROR_EVENT); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), ERROR_TOPIC))); - - } - - @Test - public void shouldSendSTARTEventToPrimaryRoute() throws Exception { - - stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG"); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - - stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), PRIMARY_TOPIC))); - - } - - @Test - public void shouldSendAUDITEventToAuditRoute() throws Exception { - - stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG"); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - - verify(collectorMock, times(2)).send(argThat(new ArgumentMatcher() { - int invocation = 0; - - @Override - public boolean matches(Object o) { - invocation = invocation + 1; - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - if (invocation == 1) { - assertEquals("kafka", systemStream.getSystem()); - assertEquals(PRIMARY_TOPIC, systemStream.getStream()); - } - if (invocation == 2) { - assertEquals("kafka", systemStream.getSystem()); - assertEquals(AUDIT_TOPIC, systemStream.getStream()); - } - return true; - } - })); - } - - @Test - public void shouldSendSHAREEventToShareEventRoute() throws Exception { - - stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG"); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - stub(envelopeMock.getMessage()).toReturn(EventFixture.SHARE_EVENT); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock, times(1)).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(SHARE_EVENT_TOPIC, systemStream.getStream()); - return true; - } - })); - } - - @Test - public void shouldStampTSForPrimaryEvent() throws Exception { - stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG"); - stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - assertNotNull(outputEvent.get("ts")); - return true; - } - })); - } - - @Test - public void shouldSendEventToFailedTopicIfEventIsNotParseable() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSABLE_START_EVENT); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - @Test - public void shouldSendEventToMalformedTopicIfEventIsAnyRandomString() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - @Test - public void shouldAddTodayDateIfETSIsNotPresent() throws Exception { - stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG"); - stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT_WITHOUT_ETS); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock); - String simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm").format(new Date()); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - String updatedTS = outputEvent.get("ts").toString(); - assertNotNull(updatedTS); - assertEquals(updatedTS.substring(0,updatedTS.indexOf(".")-3), simpleDateFormat); - return true; - } - })); - } - - @Test - public void shouldSendEventsToErrorTopic() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INVALID_EVENT); - telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock); - telemetryRouterTask.process(envelopeMock,collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - } - - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - - return true; - } - }; - } -} diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/cdata.json b/data-pipeline/telemetry-router/src/test/resources/2.2/cdata.json deleted file mode 100755 index 4c7261b025..0000000000 --- a/data-pipeline/telemetry-router/src/test/resources/2.2/cdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/cdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/cdata/id", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ "type", "id"] - } -} diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/common_empty_sid_uid.json b/data-pipeline/telemetry-router/src/test/resources/2.2/common_empty_sid_uid.json deleted file mode 100755 index 9ac4103bd6..0000000000 --- a/data-pipeline/telemetry-router/src/test/resources/2.2/common_empty_sid_uid.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "pdata": { - "$ref": "file:src/test/resources/2.2/pdata.json" - }, - "gdata": { - "$ref": "file:src/test/resources/2.2/gdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "etags": { - "$ref": "file:src/test/resources/2.2/etags.json" - }, - "cdata": { - "$ref": "file:src/test/resources/2.2/cdata.json" - }, - "channel": { - "id": "http://api.ekstep.org/telemetry/2.2/channel", - "type": "string", - "minLength": 1 - } - } -} diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/etags.json b/data-pipeline/telemetry-router/src/test/resources/2.2/etags.json deleted file mode 100755 index e00c02e041..0000000000 --- a/data-pipeline/telemetry-router/src/test/resources/2.2/etags.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "object", - "properties": { - "app": { - "id": "http://api.ekstep.org/telemetry/etags/app", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/app/items", - "type": "string" - } - }, - "partner": { - "id": "http://api.ekstep.org/telemetry/etags/partner", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/partner/items", - "type": "string" - } - }, - "dims": { - "id": "http://api.ekstep.org/telemetry/etags/dims", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/dims/items", - "type": "string" - } - } - }, - "additionalProperties": false -} diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/gdata.json b/data-pipeline/telemetry-router/src/test/resources/2.2/gdata.json deleted file mode 100755 index 1b603a1b7b..0000000000 --- a/data-pipeline/telemetry-router/src/test/resources/2.2/gdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/ge_error.json b/data-pipeline/telemetry-router/src/test/resources/2.2/ge_error.json deleted file mode 100755 index 14350be425..0000000000 --- a/data-pipeline/telemetry-router/src/test/resources/2.2/ge_error.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "file:src/test/resources/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id", - "err", - "eventId", - "data" - ], - "properties": { - "err": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "SYSTEM", - "GENIE", - "GAME", - "GENIESERVICES" - ] - }, - "id": { - "type": "string" - }, - "eventId": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/pdata.json b/data-pipeline/telemetry-router/src/test/resources/2.2/pdata.json deleted file mode 100755 index cd509bc58b..0000000000 --- a/data-pipeline/telemetry-router/src/test/resources/2.2/pdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/pdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/pdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/data-pipeline/telemetry-validator/pom.xml b/data-pipeline/telemetry-validator/pom.xml deleted file mode 100644 index e3212db210..0000000000 --- a/data-pipeline/telemetry-validator/pom.xml +++ /dev/null @@ -1,188 +0,0 @@ - - - - 4.0.0 - - 3.0.1 - - - - org.ekstep.ecosystem - jobs - 0.0.1 - - - - org.ekstep.ecosystem.jobs - telemetry-validator - 0.2.2 - jar - TelemetryValidator - - Validating Event Against Schema - - - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.schwering - irclib - 1.10 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - com.cedarsoftware - json-io - 4.0.0 - - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - com.google.code.gson - gson - 2.4 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - com.github.java-json-tools - json-schema-validator - 2.2.8 - - - joda-time - joda-time - - - - - com.google.guava - guava - 18.0 - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - diff --git a/data-pipeline/telemetry-validator/src/main/assembly/src.xml b/data-pipeline/telemetry-validator/src/main/assembly/src.xml deleted file mode 100644 index 0d5552e0a4..0000000000 --- a/data-pipeline/telemetry-validator/src/main/assembly/src.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/telemetry-validator.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:telemetry-validator - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - - true - - - diff --git a/data-pipeline/telemetry-validator/src/main/config/telemetry-validator.properties b/data-pipeline/telemetry-validator/src/main/config/telemetry-validator.properties deleted file mode 100644 index 910780c7c9..0000000000 --- a/data-pipeline/telemetry-validator/src/main/config/telemetry-validator.properties +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.TelemetryValidator - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__telemetry_validator_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.TelemetryValidatorTask -task.inputs=kafka.__env__.telemetry.raw -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -output.success.topic.name=__env__.telemetry.valid -output.failed.topic.name=__env__.telemetry.failed -output.malformed.topic.name=__env__.telemetry.malformed -output.metrics.topic.name=__env__.pipeline_metrics -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,failed-message-count,error-message-count -default.channel=__default_channel__ - -telemetry.schema.path=__telemetry_schema_path__ -telemetry.schema.version=3.0 \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index b9c85cf6ae..0000000000 --- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.apache.commons.lang.StringUtils; -import org.ekstep.ep.samza.events.domain.Events; -import org.ekstep.ep.samza.reader.NullableValue; -import org.ekstep.ep.samza.task.TelemetryValidatorConfig; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.Map; - -public class Event extends Events { - - private DateTimeFormatter df = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZoneUTC(); - - public Event(Map map) { - super(map); - } - - public String schemaName() { - String eid = eid(); - if (eid != null) { - return MessageFormat.format("{0}.json", eid.toLowerCase()); - } else { - return "envelope.json"; - } - } - - public void markSuccess() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.tv_processed", true); - telemetry.add("type", "events"); - } - - public void markFailure(String error, TelemetryValidatorConfig config) { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.tv_processed", false); - - telemetry.addFieldIfAbsent("metadata", new HashMap()); - if (null != error) { - telemetry.add("metadata.tv_error", error); - telemetry.add("metadata.src", config.jobName()); - } - - } - - public void markSkipped() { - telemetry.addFieldIfAbsent("flags", new HashMap()); - telemetry.add("flags.tv_skipped", true); - } - - public void updateActorId(String actorId) { - telemetry.add("actor.id", actorId); - } - - public void correctDialCodeKey() { - NullableValue dialcodes = telemetry.read("edata.filters.dialCodes"); - if (dialcodes != null && dialcodes.value() != null) { - telemetry.add("edata.filters.dialcodes", dialcodes.value()); - telemetry.add("edata.filters.dialCodes", null); - } - } - - public void correctDialCodeValue() { - String dialcode = telemetry.read("object.id").value(); - telemetry.add("object.id", dialcode.toUpperCase()); - } - - public void updateDefaults(TelemetryValidatorConfig config) { - String channelString = telemetry.read("context.channel").value(); - String channel = StringUtils.deleteWhitespace(channelString); - if (channel == null || channel.isEmpty()) { - telemetry.addFieldIfAbsent("context", new HashMap()); - telemetry.add("context.channel", config.defaultChannel()); - } - String atTimestamp = telemetry.getAtTimestamp(); - String strSyncts = telemetry.getSyncts(); - if (null == atTimestamp && null == strSyncts) { - long syncts = System.currentTimeMillis(); - telemetry.addFieldIfAbsent("syncts", syncts); - telemetry.addFieldIfAbsent("@timestamp", df.print(syncts)); - } else { - if (atTimestamp != null) { - telemetry.addFieldIfAbsent("syncts", df.parseMillis(atTimestamp)); - } else if (strSyncts != null) { - telemetry.addFieldIfAbsent("@timestamp", strSyncts); - } - } - - } - - public String actorId() { - NullableValue checksum = telemetry.read("actor.id"); - return checksum.value(); - } -} diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/service/TelemetryValidatorService.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/service/TelemetryValidatorService.java deleted file mode 100644 index f3969b0b92..0000000000 --- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/service/TelemetryValidatorService.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.google.gson.JsonSyntaxException; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.TelemetryValidatorConfig; -import org.ekstep.ep.samza.task.TelemetryValidatorSink; -import org.ekstep.ep.samza.task.TelemetryValidatorSource; -import org.ekstep.ep.samza.util.TelemetrySchemaValidator; - -import static java.text.MessageFormat.format; - -public class TelemetryValidatorService { - private static Logger LOGGER = new Logger(TelemetryValidatorService.class); - private final TelemetryValidatorConfig config; - private TelemetrySchemaValidator telemetrySchemaValidator; - private String eidNullErrorMsg = "VALIDATION FAILED as eid is missing"; - private String schemaNotFoundErrorMsg = "SCHEMA NOT FOUND"; - - public TelemetryValidatorService(TelemetryValidatorConfig config, TelemetrySchemaValidator telemetrySchemaValidator) { - this.config = config; - this.telemetrySchemaValidator = telemetrySchemaValidator; - } - - public void process(TelemetryValidatorSource source, TelemetryValidatorSink sink) { - Event event = null; - try { - event = source.getEvent(); - if(event.eid() == null) { - LOGGER.error(null, eidNullErrorMsg); - sink.toFailedTopic(event, eidNullErrorMsg); - return; - } - - if (!telemetrySchemaValidator.schemaFileExists(event)) { - LOGGER.debug(null, schemaNotFoundErrorMsg); - sink.toFailedTopic(event, schemaNotFoundErrorMsg); - return; - } - - ProcessingReport report = telemetrySchemaValidator.validate(event); - - if (report.isSuccess()) { - LOGGER.debug("VALIDATION SUCCESS", event.mid()); - event.markSuccess(); - event.updateDefaults(config); - sink.toSuccessTopic(dataCorrection(event)); - } else { - LOGGER.error(null, "VALIDATION FAILED: " + report.toString()); - String fieldName = getInvalidFieldName(report.toString()); - sink.toFailedTopic(event, String.format("validation failed. fieldname: %s", fieldName)); - } - } catch (JsonSyntaxException e) { - LOGGER.error(null, "INVALID EVENT: " + source.getMessage()); - sink.toMalformedEventsTopic(source.getMessage()); - } catch (Exception e) { - LOGGER.error(null, format( - "EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:", - event), e); - sink.toErrorTopic(event, e.getMessage()); - } - } - - public String getInvalidFieldName(String errorInfo) { - String[] message = errorInfo.split("reports:"); - String noFieldNameMsg = "unable to get the field name"; - if (message.length > 1) { - String[] fields = message[1].split(","); - if (fields.length > 2) { - String[] pointer = fields[3].split("\"pointer\":"); - return pointer[1].substring(0, pointer[1].length() - 1); - } - } - return noFieldNameMsg; - } - - public Event dataCorrection(Event event) { - // Remove prefix from federated userIds - String eventActorId = event.actorId(); - if (eventActorId != null && !eventActorId.isEmpty() && eventActorId.startsWith("f:")) { - event.updateActorId(eventActorId.substring(eventActorId.lastIndexOf(":") + 1)); - } - - if (event.eid() != null && event.eid().equalsIgnoreCase("SEARCH")) { - event.correctDialCodeKey(); - } - - if (event.objectFieldsPresent() && (event.objectType().equalsIgnoreCase("DialCode") || event.objectType().equalsIgnoreCase("qr"))) { - event.correctDialCodeValue(); - } - return event; - } - -} diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorConfig.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorConfig.java deleted file mode 100644 index 8ac679cd08..0000000000 --- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorConfig.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.ekstep.ep.samza.task; - - -import org.apache.samza.config.Config; - -public class TelemetryValidatorConfig { - - private final String JOB_NAME = "TelemetryValidator"; - private String schemaPath; - private String successTopic; - private String failedTopic; - private String malformedTopic; - private String defaultChannel; - private String schema_version; - - public TelemetryValidatorConfig(Config config) { - successTopic = config.get("output.success.topic.name", "telemetry.valid"); - failedTopic = config.get("output.failed.topic.name", "telemetry.failed"); - malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed"); - defaultChannel = config.get("default.channel", "org.sunbird"); - schemaPath = config.get("telemetry.schema.path", "/etc/samza-jobs/schemas"); - schema_version = config.get("telemetry.schema.version", "3.0"); - - } - - public String successTopic() { - return successTopic; - } - - public String failedTopic() { - return failedTopic; - } - - public String malformedTopic() { - return malformedTopic; - } - - public String defaultChannel() { - return defaultChannel; - } - - public String schemaPath() { - return schemaPath; - } - - public String schemaVersion() { - return schema_version; - } - - public String jobName() { - return JOB_NAME; - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSink.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSink.java deleted file mode 100644 index 870c24b26e..0000000000 --- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSink.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.domain.Event; - -public class TelemetryValidatorSink extends BaseSink { - - private TelemetryValidatorConfig config; - - public TelemetryValidatorSink(MessageCollector collector, JobMetrics metrics, - TelemetryValidatorConfig config) { - - super(collector, metrics); - this.config = config; - } - - public void toSuccessTopic(Event event) { - toTopic(config.successTopic(), event.mid(), event.getJson()); - metrics.incSuccessCounter(); - } - - public void toFailedTopic(Event event, String failedMessage) { - event.markFailure(failedMessage, config); - toTopic(config.failedTopic(), event.mid(), event.getJson()); - metrics.incFailedCounter(); - } - - public void toErrorTopic(Event event, String errorMessage) { - event.markFailure(errorMessage, config); - toTopic(config.failedTopic(), event.mid(), event.getJson()); - metrics.incErrorCounter(); - } - - public void toMalformedEventsTopic(String message) { - toTopic(config.malformedTopic(), null, message); - metrics.incFailedCounter(); - } - -} diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSource.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSource.java deleted file mode 100644 index 277b73bf4d..0000000000 --- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSource.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ekstep.ep.samza.task; - -import java.util.Map; - -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; - -import com.google.gson.Gson; - -public class TelemetryValidatorSource { - static Logger LOGGER = new Logger(TelemetryValidatorSource.class); - - private IncomingMessageEnvelope envelope; - - public TelemetryValidatorSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - return new Event(getMap()); - } - - @SuppressWarnings("unchecked") - private Map getMap() { - String message = (String) envelope.getMessage(); - return (Map) new Gson().fromJson(message, Map.class); - } - - public String getMessage() { - return envelope.toString(); - } - - -} diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorTask.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorTask.java deleted file mode 100644 index de41d8d09a..0000000000 --- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorTask.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.TelemetryValidatorService; -import org.ekstep.ep.samza.util.TelemetrySchemaValidator; - -public class TelemetryValidatorTask extends BaseSamzaTask { - - private TelemetryValidatorConfig config; - private JobMetrics metrics; - private TelemetryValidatorService service; - - public TelemetryValidatorTask(Config config, TaskContext context, TelemetrySchemaValidator telemetrySchemaValidator) throws Exception { - init(config, context, telemetrySchemaValidator); - } - - public TelemetryValidatorTask() { - - } - - - @Override - public void init(Config config, TaskContext context) throws Exception { - init(config, context, null); - } - - - public void init(Config config, TaskContext context, TelemetrySchemaValidator schemaValidator) throws Exception { - this.config = new TelemetryValidatorConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - TelemetrySchemaValidator telemetrySchemaValidator = schemaValidator == null ? new TelemetrySchemaValidator(this.config) : schemaValidator; - service = new TelemetryValidatorService(this.config, telemetrySchemaValidator); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, - TaskCoordinator taskCoordinator) throws Exception { - TelemetryValidatorSource source = new TelemetryValidatorSource(envelope); - TelemetryValidatorSink sink = new TelemetryValidatorSink(collector, metrics, config); - - service.process(source, sink); - } - -} diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/util/TelemetrySchemaValidator.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/util/TelemetrySchemaValidator.java deleted file mode 100644 index 3ef33ca3d5..0000000000 --- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/util/TelemetrySchemaValidator.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.ekstep.ep.samza.util; - -import com.fasterxml.jackson.databind.JsonNode; -import com.github.fge.jackson.JsonLoader; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchema; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.TelemetryValidatorConfig; - -import java.io.File; -import java.io.IOException; -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.Map; - -public class TelemetrySchemaValidator { - - private Map schemaJsonMap = new HashMap<>(); - - public TelemetrySchemaValidator(TelemetryValidatorConfig config) throws IOException, ProcessingException { - - String schemaVersion = config.schemaVersion(); - File schemaDirectory = new File(MessageFormat.format("{0}/{1}/", config.schemaPath(), schemaVersion)); - File[] schemaFiles = schemaDirectory.listFiles(); - for (File schemafile : schemaFiles) { - String schemaKey = String.format("%s", schemafile.getName()); - schemaJsonMap.put(schemaKey, JsonSchemaFactory - .byDefault().getJsonSchema(JsonLoader.fromFile(schemafile))); - - } - - } - - public boolean schemaFileExists(Event event) { - return schemaJsonMap.containsKey(String.format("%s", event.schemaName())); - } - - public ProcessingReport validate(Event event) throws IOException, ProcessingException { - JsonNode eventJson = JsonLoader.fromString(event.getJson()); - String schemaKey = String.format("%s", event.schemaName()); - ProcessingReport report = schemaJsonMap.get(schemaKey).validate(eventJson); - return report; - } - -} diff --git a/data-pipeline/telemetry-validator/src/main/resources/log4j.xml b/data-pipeline/telemetry-validator/src/main/resources/log4j.xml deleted file mode 100644 index e416dd51b3..0000000000 --- a/data-pipeline/telemetry-validator/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java deleted file mode 100644 index 98544883d6..0000000000 --- a/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java +++ /dev/null @@ -1,597 +0,0 @@ -package org.ekstep.ep.samza.fixtures; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -import java.util.Map; - -public class EventFixture { - - public static final String VALID_IMPRESSION_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"f:5a8a3f2b-3409-42e0-9001-f913bc0fde31:874ed8a5-782e-4f6c-8f36-e0288455901e\"},\"eid\":\"IMPRESSION\",\"edata\":{\"type\":\"detail\",\"pageid\":\"content-detail\",\"uri\":\"content-detail\"},\"ver\":\"3.0\",\"ets\":1.592591395144E12,\"context\":{\"pdata\":{\"pid\":\"sunbird.app\",\"ver\":\"2.10.307\",\"id\":\"prod.diksha.app\"},\"channel\":\"0126684405014528002\",\"env\":\"home\",\"did\":\"77f505f41501d86a0ccaec9cf3b9ec99bec5af06\",\"cdata\":[],\"sid\":\"57b8bc17-1705-4cdd-885b-99db23f4b36b\",\"rollup\":{\"l1\":\"0126684405014528002\"}},\"mid\":\"f5114fee-ab6b-42da-afbe-13bd2bdb0476\",\"object\":{\"id\":\"do_3130160314974781441206\",\"type\":\"ExplanationResource\",\"version\":\"1\",\"rollup\":{\"l1\":\"do_3130160314974781441206\"}},\"syncts\":1592591400233,\"@timestamp\":\"2020-06-19T18:30:00.233Z\"}"; - - public static final String UNPARSEABLE_EVENT = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"5a75030f-79be-4a08-a6b2-eb27769a23f5\"\n" + - " },\n" + - " \"eid\": \"IMPRESSION\",\n" + - " \"edata\": {\n" + - " \"type\": \"detail\",\n" + - " \"pageid\": \"content-detail\",\n" + - " \"uri\": \"content-detail\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1.592591395144E12,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"ver\": \"2.10.307\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0126684405014528002\",\n" + - " \"env\": \"home\",\n" + - " \"did\": \"77f505f41501d86a0ccaec9cf3b9ec99bec5af06\",\n" + - " \"cdata\": [\n" + - " \n" + - " ],\n" + - " \"sid\": \"57b8bc17-1705-4cdd-885b-99db23f4b36b\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"0126684405014528002\"\n" + - " }\n" + - " \"mid\": \"f5114fee-ab6b-42da-afbe-13bd2bdb0476\",\n" + - " \"object\": {\n" + - " \"id\": \"do_3130160314974781441206\",\n" + - " \"type\": \"ExplanationResource\",\n" + - " \"version\": \"1\",\n" + - " \"rollup\": {\n" + - " \"l1\": \"do_3130160314974781441206\"\n" + - " }\n" + - " },\n" + - " \"syncts\": 1592591400233,\n" + - " \"@timestamp\": \"2020-06-19T18:30:00.233Z\"\n" + - "}"; - - public static final String SEARCH_EVENT_WITH_INCORRECT_DIALCODES_KEY = "{\n" + - " \"eid\": \"SEARCH\",\n" + - " \"ets\": 1.554241415598E12,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"LP.1554241415598.bff9398b-a827-41ec-b1c9-e569546174b1\",\n" + - " \"actor\": {\n" + - " \"id\": \"org.ekstep.learning.platform\",\n" + - " \"type\": \"System\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"in.ekstep\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.ntp.learning.platform\",\n" + - " \"pid\": \"search-service\",\n" + - " \"ver\": \"1.0\"\n" + - " },\n" + - " \"env\": \"search\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"size\": 499413.0,\n" + - " \"query\": \"\",\n" + - " \"filters\": {\n" + - " \"dialCodes\": [\n" + - " \"ATAZRI\",\n" + - " \"CGSX7D\"\n" + - " ]\n" + - " },\n" + - " \"sort\": {\n" + - " \n" + - " },\n" + - " \"type\": \"all\",\n" + - " \"topn\": [\n" + - " {\n" + - " \"identifier\": \"do_31268582767737241615189\"\n" + - " },\n" + - " {\n" + - " \"identifier\": \"do_31269107959395942417491\"\n" + - " },\n" + - " {\n" + - " \"identifier\": \"do_31269108472948326417493\"\n" + - " },\n" + - " {\n" + - " \"identifier\": \"do_31269113788995174417318\"\n" + - " },\n" + - " {\n" + - " \"identifier\": \"do_31270597860728832015700\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"syncts\": 1.55424141664E12,\n" + - " \"@timestamp\": \"2019-04-02T21:43:36.640Z\"\n" + - "}"; - - public static final String EVENT_WITH_EID_MISSING = "{\n" + - " \"ver\":\"3.0\",\n" + - " \"syncts\":0.0,\n" + - " \"ets\":0.0,\n" + - " \"tags\":[\n" + - "\n" + - " ],\n" + - " \"context\":{\n" + - " \"channel\":\"01250894314817126443\"\n" + - " },\n" + - " \"ts\":\"1970-01-01T00:00:00.000+0000\",\n" + - " \"metadata\":{\n" + - " \"checksum\":\"10b890f9783ddcae406e7d6514d073eed4327941\",\n" + - " \"odn_status\":\"failed\",\n" + - " \"odn_error\":null\n" + - " }\n" + - "}"; - - public static final String INVALID_EVENT = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" + - " },\n" + - " \"eid\": \"LOG\",\n" + - " \"edata\": {\n" + - " \"id\": \"ContentDetail\",\n" + - " \"pageid\": \"ContentDetail\",\n" + - " \"type\": \"TOUCH\",\n" + - " \"subtype\": \"ContentDownload-Initiate\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": \"1541574545180\",\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"ver\": \"2.1.8\",\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"0123221617357783046602\",\n" + - " \"env\": \"sdk\",\n" + - " \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"qr\",\n" + - " \"id\": \"K4KCXE\"\n" + - " },\n" + - " {\n" + - " \"type\": \"API\",\n" + - " \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" + - " },\n" + - " \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" + - " \"object\": {\n" + - " \"id\": \"do_31249561779090227216256\",\n" + - " \"type\": \"Content\",\n" + - " \"version\": \"\"\n" + - " },\n" + - " \"tags\": [],\n" + - " \"syncts\": 1539846605341,\n" + - " \"@timestamp\": \"2018-10-18T07:10:05.341Z\",\n" + - " \"derivedlocationdata\": {\n" + - " \"district\": \"Bengaluru\",\n" + - " \"state\": 1234,\n" + - " \"from\": \"user-profile\"\n" + - " },\n" + - " \"devicedata\": {\n" + - " \"statecustomcode\": \"KA-Custom\",\n" + - " \"country\": \"India\",\n" + - " \"iso3166statecode\": \"IN-KA\",\n" + - " \"city\": \"Bangalore\",\n" + - " \"countrycode\": \"IN\",\n" + - " \"statecode\": \"KA\",\n" + - " \"devicespec\": {\n" + - " \"os\": \"Android 6.0\",\n" + - " \"make\": \"Motorola XT1706\"\n" + - " },\n" + - " \"districtcustom\": \"Banglore-Custom\",\n" + - " \"firstaccess\": 1559484698000,\n" + - " \"uaspec\": {\n" + - " \"agent\": \"Mozilla\",\n" + - " \"ver\": \"5.0\",\n" + - " \"system\": \"iPad\",\n" + - " \"raw\": \"Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)\",\n" + - " \"platform\": \"AppleWebKit/531.21.10\"\n" + - " },\n" + - " \"state\": \"Karnataka\",\n" + - " \"statecustomname\": \"Karnatak-Custom\",\n" + - " \"userdeclared\": {\n" + - " \"district\": \"Bangalore\",\n" + - " \"state\": \"Karnataka\"\n" + - " }\n" + - " }\n" + - "}"; - - public static final String VALID_LOG_EVENT = "{\n" + - " \"eid\": \"LOG\",\n" + - " \"ets\": 1570817279146,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"LOG:2a210813fac274656f194f9807ad8abf\",\n" + - " \"actor\": {\n" + - " \"id\": \"1\",\n" + - " \"type\": \"service\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.content-service\",\n" + - " \"ver\": \"1.0\",\n" + - " \"pid\": \"sunbird-content-service\"\n" + - " },\n" + - " \"env\": \"content\",\n" + - " \"sid\": \"\",\n" + - " \"did\": \"1e465bed2138b8f2764d946492293e2b2f628789\",\n" + - " \"cdata\": [],\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312776645813747712111638\",\n" + - " \"type\": \"content\",\n" + - " \"ver\": \"\",\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"tags\": [],\n" + - " \"edata\": {\n" + - " \"type\": \"api_access\",\n" + - " \"level\": \"INFO\",\n" + - " \"message\": \"successful\",\n" + - " \"params\": [\n" + - " {\n" + - " \"rid\": \"0ece8640-ec52-11e9-a3d6-697ee5684e40\"\n" + - " },\n" + - " {\n" + - " \"title\": \"Content read api\"\n" + - " },\n" + - " {\n" + - " \"category\": \"contentread\"\n" + - " },\n" + - " {\n" + - " \"url\": \"content/read\"\n" + - " },\n" + - " {\n" + - " \"method\": \"GET\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"syncts\": 1570817431399,\n" + - " \"@timestamp\": \"2019-10-11T18:10:31.399Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String VALID_LOG_WITH_MISSING_CHANNEL="{\n" + - " \"eid\": \"LOG\",\n" + - " \"ets\": 1570817279146,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"LOG:2a210813fac274656f194f9807ad8abf\",\n" + - " \"actor\": {\n" + - " \"id\": \"1\",\n" + - " \"type\": \"service\"\n" + - " },\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.content-service\",\n" + - " \"ver\": \"1.0\",\n" + - " \"pid\": \"sunbird-content-service\"\n" + - " },\n" + - " \"env\": \"content\",\n" + - " \"sid\": \"\",\n" + - " \"did\": \"1e465bed2138b8f2764d946492293e2b2f628789\",\n" + - " \"cdata\": [],\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312776645813747712111638\",\n" + - " \"type\": \"content\",\n" + - " \"ver\": \"\",\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"tags\": [],\n" + - " \"edata\": {\n" + - " \"type\": \"api_access\",\n" + - " \"level\": \"INFO\",\n" + - " \"message\": \"successful\",\n" + - " \"params\": [\n" + - " {\n" + - " \"rid\": \"0ece8640-ec52-11e9-a3d6-697ee5684e40\"\n" + - " },\n" + - " {\n" + - " \"title\": \"Content read api\"\n" + - " },\n" + - " {\n" + - " \"category\": \"contentread\"\n" + - " },\n" + - " {\n" + - " \"url\": \"content/read\"\n" + - " },\n" + - " {\n" + - " \"method\": \"GET\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"syncts\": 1570817431399,\n" + - " \"@timestamp\": \"2019-10-11T18:10:31.399Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String VALID_LOG_WITH_MISSING_SYNCTS="{\n" + - " \"eid\": \"LOG\",\n" + - " \"ets\": 1570817279146,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"LOG:2a210813fac274656f194f9807ad8abf\",\n" + - " \"actor\": {\n" + - " \"id\": \"1\",\n" + - " \"type\": \"service\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"pdata\": {\n" + - " \"id\": \"prod.diksha.content-service\",\n" + - " \"ver\": \"1.0\",\n" + - " \"pid\": \"sunbird-content-service\"\n" + - " },\n" + - " \"env\": \"content\",\n" + - " \"sid\": \"\",\n" + - " \"did\": \"1e465bed2138b8f2764d946492293e2b2f628789\",\n" + - " \"cdata\": [],\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"do_312776645813747712111638\",\n" + - " \"type\": \"content\",\n" + - " \"ver\": \"\",\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"tags\": [],\n" + - " \"edata\": {\n" + - " \"type\": \"api_access\",\n" + - " \"level\": \"INFO\",\n" + - " \"message\": \"successful\",\n" + - " \"params\": [\n" + - " {\n" + - " \"rid\": \"0ece8640-ec52-11e9-a3d6-697ee5684e40\"\n" + - " },\n" + - " {\n" + - " \"title\": \"Content read api\"\n" + - " },\n" + - " {\n" + - " \"category\": \"contentread\"\n" + - " },\n" + - " {\n" + - " \"url\": \"content/read\"\n" + - " },\n" + - " {\n" + - " \"method\": \"GET\"\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String IMPRESSION_EVENT_WITH_DIALCODE_OBJECT_IN_LOWERCASE = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"anonymous\"\n" + - " },\n" + - " \"eid\": \"IMPRESSION\",\n" + - " \"edata\": {\n" + - " \"visits\": [\n" + - " \n" + - " ],\n" + - " \"type\": \"view\",\n" + - " \"pageid\": \"dialcode\",\n" + - " \"subtype\": \"pageexit\",\n" + - " \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1570817279146,\n" + - " \"context\": {\n" + - " \"uid\": \"anonymous\",\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.7.1\",\n" + - " \"pid\": \"sunbird-portal\",\n" + - " \"id\": \"dev.sunbird.portal\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"public\",\n" + - " \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" + - " \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" + - " \"cdata\": [\n" + - " \n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" + - " \"object\": {\n" + - " \"ver\": \"1.0\",\n" + - " \"id\": \"977d3i\",\n" + - " \"type\": \"dialcode\",\n" + - " \"rollup\": {\n" + - " \n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " ],\n" + - " \"syncts\": 1550501698819,\n" + - " \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true,\n" + - " \"ldata_retrieved\": false\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" + - " \"ldata\": {\n" + - " \"country_code\": \"\",\n" + - " \"country\": \"\",\n" + - " \"city\": \"\",\n" + - " \"state\": \"\",\n" + - " \"state_code\": \"\"\n" + - " } \n" + - "}"; - - public static final String IMPRESSION_EVENT_WITH_DIALCODE_OBJECT_IN_UPPERCASE = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"anonymous\"\n" + - " },\n" + - " \"eid\": \"IMPRESSION\",\n" + - " \"edata\": {\n" + - " \"visits\": [\n" + - " \n" + - " ],\n" + - " \"type\": \"view\",\n" + - " \"pageid\": \"dialcode\",\n" + - " \"subtype\": \"pageexit\",\n" + - " \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1570817279146,\n" + - " \"context\": {\n" + - " \"uid\": \"anonymous\",\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.7.1\",\n" + - " \"pid\": \"sunbird-portal\",\n" + - " \"id\": \"dev.sunbird.portal\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"public\",\n" + - " \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" + - " \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" + - " \"cdata\": [\n" + - " \n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" + - " \"object\": {\n" + - " \"ver\": \"1.0\",\n" + - " \"id\": \"977D3I\",\n" + - " \"type\": \"qr\",\n" + - " \"rollup\": {\n" + - " \n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " ],\n" + - " \"syncts\": 1550501698819,\n" + - " \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true,\n" + - " \"ldata_retrieved\": false\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" + - " \"ldata\": {\n" + - " \"country_code\": \"\",\n" + - " \"country\": \"\",\n" + - " \"city\": \"\",\n" + - " \"state\": \"\",\n" + - " \"state_code\": \"\"\n" + - " } \n" + - "}"; - - public static final String EVENT_WITHOUT_EID = "{\n" + - " \"@timestamp\": \"2020-06-14T18:31:18.487Z\",\n" + - " \"@version\": \"1\",\n" + - " \"context\": {\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " },\n" + - " \"message\": \"478097.fdedf464-72ca-45ae-8100-9a92fac0c46b\\\",\\\"actor\\\":{\\\"id\\\":\\\"org.ekstep.learning.platform\\\",\\\"type\\\":\\\"System\\\"},\\\"context\\\":{\\\"channel\\\":\\\"in.ekstep\\\",\\\"pdata\\\":{\\\"id\\\":\\\"prod.ntp.learning.platform\\\",\\\"pid\\\":\\\"search-service\\\",\\\"ver\\\":\\\"1.0\\\"},\\\"env\\\":\\\"search\\\"},\\\"edata\\\":{\\\"size\\\":0,\\\"query\\\":\\\"\\\",\\\"filters\\\":{\\\"status\\\":[\\\"Live\\\",\\\"Draft\\\",\\\"Review\\\"],\\\"contentType\\\":[\\\"Resource\\\",\\\"Collection\\\"],\\\"createdFor\\\":\\\"01241408242723225614\\\",\\\"createdOn\\\":{\\\">=\\\":\\\"2020-06-08T00:00:00.000+0000\\\",\\\"<\\\":\\\"2020-06-15T00:00:00.000+0000\\\"}},\\\"sort\\\":{},\\\"type\\\":\\\"all\\\",\\\"topn\\\":[]},\\\"syncts\\\":1592159478097}\",\n" + - " \"tags\": [\n" + - " \"_jsonparsefailure\"\n" + - " ]\n" + - "}"; - - public static final String EVENT_WITH_NO_VALIDATION_SCHEMA = "{\n" + - " \"actor\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"anonymous\"\n" + - " },\n" + - " \"eid\": \"NO_SCHEMA_DEFINED\",\n" + - " \"edata\": {\n" + - " \"visits\": [\n" + - " \n" + - " ],\n" + - " \"type\": \"view\",\n" + - " \"pageid\": \"dialcode\",\n" + - " \"subtype\": \"pageexit\",\n" + - " \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1570817279146,\n" + - " \"context\": {\n" + - " \"uid\": \"anonymous\",\n" + - " \"pdata\": {\n" + - " \"ver\": \"1.7.1\",\n" + - " \"pid\": \"sunbird-portal\",\n" + - " \"id\": \"dev.sunbird.portal\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"public\",\n" + - " \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" + - " \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" + - " \"cdata\": [\n" + - " \n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" + - " \"object\": {\n" + - " \"ver\": \"1.0\",\n" + - " \"id\": \"977d3i\",\n" + - " \"type\": \"dialcode\",\n" + - " \"rollup\": {\n" + - " \n" + - " }\n" + - " },\n" + - " \"tags\": [\n" + - " \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" + - " ],\n" + - " \"syncts\": 1550501698819,\n" + - " \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true,\n" + - " \"ldata_retrieved\": false\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" + - " \"ldata\": {\n" + - " \"country_code\": \"\",\n" + - " \"country\": \"\",\n" + - " \"city\": \"\",\n" + - " \"state\": \"\",\n" + - " \"state_code\": \"\"\n" + - " } \n" + - "}"; - - public static final String EVENT_WITH_DIFFERENT_SCHEMA_VERSION = "{\"actor\":{\"type\":\"User\",\"id\":\"4ba2bb28-16c6-483d-9d94-58c6364e5e3c\"},\"eid\":\"ASSESS\",\"edata\":{\"resvalues\":[{\"3\":{\"text\":\"24n\"}}],\"duration\":21.0,\"score\":0.0,\"item\":{\"mmc\":[],\"mc\":[],\"exlength\":0.0,\"id\":\"do_31296568773394432011706\",\"maxscore\":1.0,\"type\":\"mcq\",\"params\":[{\"1\":{\"text\":\"22n\"}},{\"2\":{\"text\":\"23n\"}},{\"answer\":{\"correct\":[\"4\"]}}],\"title\":\"Test title\",\"uri\":\"\",\"desc\":\"\"},\"pass\":\"No\",\"index\":1.0},\"ver\":\"3.1\",\"ets\":1.592982195439E12,\"context\":{\"pdata\":{\"ver\":\"3.0.0\",\"pid\":\"sunbird-portal.contentplayer\",\"id\":\"prod.diksha.portal\"},\"channel\":\"0126684405014528002\",\"env\":\"contentplayer\",\"did\":\"6798fa5a2d8335c43ba64d5b96a944b9\",\"sid\":\"lRc_DKH5C4034jqv8q1R4O3jxBACvd7e\",\"cdata\":[{\"type\":\"course\",\"id\":\"do_31296569862596198411604\"},{\"type\":\"batch\",\"id\":\"0129657011429376000\"},{\"type\":\"Source\",\"id\":[\"0126087586736619522\",\"0125196274181898243\"]}],\"rollup\":{\"l1\":\"0126684405014528002\"}},\"mid\":\"ASSESS:4ca1ce884b8eeb679abe8ec58c0ac763\",\"object\":{\"ver\":\"1\",\"id\":\"do_3129656882081382401299\",\"type\":\"Content\",\"rollup\":{\"l1\":\"do_3129655631697100801195\",\"l2\":\"do_31296569862596198411604\",\"l3\":\"do_3129656882081382401299\"}},\"tags\":[\"0126684405014528002\"],\"syncts\":1.592982225289E12,\"@timestamp\":\"2020-06-24T07:03:45.289Z\",\"flags\":{\"tv_skipped\":true,\"dd_processed\":true},\"type\":\"events\"}"; - - public static final String ANY_STRING = "Hey Samza, Whats Up?"; - public static final String EMPTY_JSON = "{}"; - -} diff --git a/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryValidatorTaskTest.java b/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryValidatorTaskTest.java deleted file mode 100644 index 907499ef1b..0000000000 --- a/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryValidatorTaskTest.java +++ /dev/null @@ -1,362 +0,0 @@ -package org.ekstep.ep.samza.task; - - -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.OutgoingMessageEnvelope; -import org.apache.samza.system.SystemStream; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.fixtures.EventFixture; -import org.ekstep.ep.samza.util.TelemetrySchemaValidator; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentMatcher; - -import java.lang.reflect.Type; -import java.util.Map; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - -public class TelemetryValidatorTaskTest { - - private static final String SUCCESS_TOPIC = "telemetry.valid"; - private static final String FAILED_TOPIC = "telemetry.failed"; - private static final String MALFORMED_TOPIC = "telemetry.malformed"; - private static final String SCHEMA_PATH = "src/test/resources"; - private static final String SCHEMA_VERSION = "3.0"; - private MessageCollector collectorMock; - private TaskCoordinator coordinatorMock; - private IncomingMessageEnvelope envelopeMock; - private TelemetryValidatorTask telemetryValidatorTask; - - @Before - public void setUp() throws Exception { - collectorMock = mock(MessageCollector.class); - TaskContext contextMock = mock(TaskContext.class); - MetricsRegistry metricsRegistry = mock(MetricsRegistry.class); - - Counter counter = mock(Counter.class); - coordinatorMock = mock(TaskCoordinator.class); - envelopeMock = mock(IncomingMessageEnvelope.class); - Config configMock = mock(Config.class); - - stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC); - stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC); - stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC); - stub(configMock.get("telemetry.schema.path", "/etc/samza-jobs/schemas")).toReturn(SCHEMA_PATH); - stub(configMock.get("telemetry.schema.version", "3.0")).toReturn(SCHEMA_VERSION); - TelemetrySchemaValidator telemetrySchemaValidator = new TelemetrySchemaValidator(new TelemetryValidatorConfig(configMock)); - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - stub(envelopeMock.getOffset()).toReturn("2"); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1))); - - - telemetryValidatorTask = new TelemetryValidatorTask(configMock, contextMock, telemetrySchemaValidator); - } - - @Test - public void shouldSendEventToSuccessTopicIfEventIsValid() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_IMPRESSION_EVENT); - - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - } - - @Test - public void shouldSendEventToFailedTopicIfSchemaIsNotPresent() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITH_NO_VALIDATION_SCHEMA); - - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType); - Map metadata = new Gson().fromJson(new Gson().toJson(outputEvent.get("metadata")), mapType); - assertEquals(false, flags.get("tv_processed")); - assertEquals("SCHEMA NOT FOUND", metadata.get("tv_error")); - assertEquals("TelemetryValidator", metadata.get("src")); - return true; - } - })); - } - - @Test - public void shouldSendEventToMalformedTopicIfEventIsNotParseable() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSEABLE_EVENT); - - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - @Test - public void shouldSendEventToMalformedTopicIfEventIsAnyRandomString() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING); - - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC))); - } - - @Test - public void shouldSendEventToSuccessTopicIfEventIsEmptyJSON() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.EMPTY_JSON); - - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - } - - @Test - public void shouldSendEventToFailedTopicIfEidNotPresent() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITH_EID_MISSING); - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - } - - @Test - public void shouldRemoveFederatedUserIdPrefixIfPresent() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_IMPRESSION_EVENT); - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map actorData = new Gson().fromJson(new Gson().toJson(outputEvent.get("actor")), mapType); - assertEquals("874ed8a5-782e-4f6c-8f36-e0288455901e", actorData.get("id")); - return true; - } - })); - } - - @Test - public void whenInvalidEventisPassed() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.INVALID_EVENT); - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType); - assertEquals(false, flags.get("tv_processed")); - return true; - } - })); - } - - /** - * When valid log event is passed then it should push to success topic - * - * @throws Exception - */ - @Test - public void shouldAddTheValidEventToSuccessTopic() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_LOG_EVENT); - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - } - - /** - * When channel field is not present in the context then it should push the event to failed topic - * @throws Exception - */ - @Test - public void shouldValidateTheChannelField() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_LOG_WITH_MISSING_CHANNEL); - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType); - assertEquals(false, flags.get("tv_processed")); - return true; - } - })); - } - - /** - * When synctc and @timestamp fields are missed, Then job should add these fileds to event. - * @throws Exception - */ - @Test - public void shouldUpdateTheDefaultsToValidEvent() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_LOG_WITH_MISSING_SYNCTS); - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC))); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType); - assertEquals(true, flags.get("tv_processed")); - assertEquals(true, outputEvent.containsKey("@timestamp")); - assertEquals(true, outputEvent.containsKey("syncts")); - return true; - } - })); - } - - @Test - public void shouldConvertDialcodesKeytoLowerCaseIfPresent() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_INCORRECT_DIALCODES_KEY); - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map edata = new Gson().fromJson(new Gson().toJson(outputEvent.get("edata")), mapType); - Map edataFilters = new Gson().fromJson(new Gson().toJson(edata.get("filters")), mapType); - assertTrue(edataFilters.containsKey("dialcodes")); - assertFalse(edataFilters.containsKey("dialCodes")); - return true; - } - })); - } - - @Test - public void shouldConvertDialcodeValueToUpperCaseIfPresentInLowerCase() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_DIALCODE_OBJECT_IN_LOWERCASE); - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map object = new Gson().fromJson(new Gson().toJson(outputEvent.get("object")), mapType); - assertFalse(object.get("id").toString().equals("977d3i")); - assertTrue(object.get("id").toString().equals("977D3I")); - return true; - } - })); - } @Test - public void shouldNotConvertDialcodeValueToUpperCaseIfNotPresentInLowerCase() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_DIALCODE_OBJECT_IN_UPPERCASE); - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map object = new Gson().fromJson(new Gson().toJson(outputEvent.get("object")), mapType); - assertTrue(object.get("id").toString().equals("977D3I")); - return true; - } - })); - } - - @Test - public void shouldSendEventToFailedTopicIfEidMissing() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITHOUT_EID); - - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType); - Map metadata = new Gson().fromJson(new Gson().toJson(outputEvent.get("metadata")), mapType); - assertEquals(false, flags.get("tv_processed")); - assertEquals("VALIDATION FAILED as eid is missing", metadata.get("tv_error")); - assertEquals("TelemetryValidator", metadata.get("src")); - return true; - } - })); - } - - @Test - public void shouldValidateAllEventVersions() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITH_DIFFERENT_SCHEMA_VERSION); - - telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock); - verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC))); - Type mapType = new TypeToken>() { - }.getType(); - verify(collectorMock).send(argThat(new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - String outputMessage = (String) outgoingMessageEnvelope.getMessage(); - Map outputEvent = new Gson().fromJson(outputMessage, mapType); - Map flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType); - Map metadata = new Gson().fromJson(new Gson().toJson(outputEvent.get("metadata")), mapType); - assertEquals(false, flags.get("tv_processed")); - assertEquals("validation failed. fieldname: \"/context/cdata/2/id\"", metadata.get("tv_error")); - assertEquals("TelemetryValidator", metadata.get("src")); - return true; - } - })); - } - - - - public ArgumentMatcher validateOutputTopic(final Object message, final String stream) { - return new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o; - SystemStream systemStream = outgoingMessageEnvelope.getSystemStream(); - assertEquals("kafka", systemStream.getSystem()); - assertEquals(stream, systemStream.getStream()); - return true; - } - }; - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/actor.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/actor.json deleted file mode 100644 index cd87b3fb22..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/actor.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/actor/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/actor/type", - "type": "string" - } - }, - "required": [ "type", "id"] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/assess.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/assess.json deleted file mode 100644 index a570db5bc3..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/assess.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/assess", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "ASSESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "item", - "pass", - "score", - "resvalues", - "duration" - ], - "properties": { - "item": { - "$ref": "resource:/3.0/question.json" - }, - "index": { - "id": "http://api.ekstep.org/telemetry/edata/index", - "type": "number" - }, - "pass": { - "id": "http://api.ekstep.org/telemetry/edata/pass", - "type": "string" - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/score", - "type": "number" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/resvalues", - "type": "array", - "items": { - "type": "object" - } - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/audit.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/audit.json deleted file mode 100644 index 8dd50aaf7b..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/audit.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/audit", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "AUDIT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "properties": { - "props": { - "id": "http://api.ekstep.org/telemetry/edata/props", - "type": "array", - "items": { - "type": "string" - } - }, - "state": { - "id": "http://api.ekstep.org/telemetry/edata/state", - "type": "string" - }, - "prevstate": { - "id": "http://api.ekstep.org/telemetry/edata/prevstate", - "type": "string" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/cdata.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/cdata.json deleted file mode 100644 index fd82b2b534..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/cdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/cdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/cdata/id", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ "type", "id"] - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/common.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/common.json deleted file mode 100644 index 83caf03dfa..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/common.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "3.0" - ] - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "actor": { - "$ref": "resource:/3.0/actor.json" - }, - "context": { - "$ref": "resource:/3.0/context.json" - }, - "object": { - "$ref": "resource:/3.0/object.json" - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/context.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/context.json deleted file mode 100644 index 95ac537a49..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/context.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": "object", - "properties": { - "channel": { - "id": "http://api.ekstep.org/telemetry/context/channel", - "type": "string", - "minLength": 1 - }, - "pdata": { - "$ref": "resource:/3.0/pdata.json" - }, - "env": { - "id": "http://api.ekstep.org/telemetry/context/env", - "type": "string" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/context/sid", - "type": "string" - }, - "did": { - "id": "http://api.ekstep.org/telemetry/context/did", - "type": "string" - }, - "cdata": { - "$ref": "resource:/3.0/cdata.json" - } - }, - "required": [ - "channel", - "env", - "pdata" - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/dspec.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/dspec.json deleted file mode 100644 index cc9ee16701..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/dspec.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "type": "object", - "properties": { - "os": { - "id": "http://api.ekstep.org/telemetry/dspec/os", - "type": "string" - }, - "make": { - "id": "http://api.ekstep.org/telemetry/dspec/make", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/dspec/id", - "type": "string" - }, - "mem": { - "id": "http://api.ekstep.org/telemetry/dspec/mem", - "type": "number", - "minimum": -1 - }, - "idisk": { - "id": "http://api.ekstep.org/telemetry/dspec/idisk", - "type": "number", - "minimum": -1 - }, - "edisk": { - "id": "http://api.ekstep.org/telemetry/dspec/edisk", - "type": "number", - "minimum": -1 - }, - "scrn": { - "id": "http://api.ekstep.org/telemetry/dspec/scrn", - "type": "number", - "minimum": -1 - }, - "camera": { - "id": "http://api.ekstep.org/telemetry/dspec/camera", - "type": "string" - }, - "cpu": { - "id": "http://api.ekstep.org/telemetry/dspec/cpu", - "type": "string" - }, - "sims": { - "id": "http://api.ekstep.org/telemetry/dspec/sims", - "type": "number", - "minimum": -1 - }, - "cap": { - "id": "http://api.ekstep.org/telemetry/dspec/cap", - "type": "array", - "items": { - "type": "string" - } - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/end.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/end.json deleted file mode 100644 index 298797cdd1..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/end.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "mode": { - "id": "http://api.ekstep.org/telemetry/edata/mode", - "type": "string" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "summary": { - "id": "http://api.ekstep.org/telemetry/edata/summary", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/envelope.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/envelope.json deleted file mode 100644 index 8d19e8565c..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/envelope.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "id":"http://api.ekstep.org/telemetry/envelope", - "type":"object", - "required":[ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf":[ - { - "$ref":"resource:/3.0/common.json" - }, - { - "properties":{ - "eid":{ - "id":"http://api.ekstep.org/telemetry/mid", - "type":"string", - "minLength": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/error.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/error.json deleted file mode 100644 index 45e808829f..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/error.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "err", - "errtype", - "stacktrace" - ], - "properties": { - "err": { - "id": "http://api.ekstep.org/telemetry/edata/err", - "type": "string" - }, - "errtype": { - "id": "http://api.ekstep.org/telemetry/edata/errtype", - "type": "string" - }, - "stacktrace": { - "id": "http://api.ekstep.org/telemetry/edata/stacktrace", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "object": { - "$ref": "resource:/3.0/inlineobject.json" - }, - "plugin": { - "$ref": "resource:/3.0/plugin.json" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/event.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/event.json deleted file mode 100644 index eab9d5bbf5..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/event.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/event", - "type": "object", - "oneOf": [{ - "$ref": "resource:/3.0/start.json" - }, - { - "$ref": "resource:/3.0/end.json" - }, - { - "$ref": "resource:/3.0/impression.json" - }, - { - "$ref": "resource:/3.0/interact.json" - }, - { - "$ref": "resource:/3.0/assess.json" - }, - { - "$ref": "resource:/3.0/response.json" - }, - { - "$ref": "resource:/3.0/interrupt.json" - }, - { - "$ref": "resource:/3.0/feedback.json" - }, - { - "$ref": "resource:/3.0/share.json" - }, - { - "$ref": "resource:/3.0/audit.json" - }, - { - "$ref": "resource:/3.0/error.json" - }, - { - "$ref": "resource:/3.0/heartbeat.json" - }, - { - "$ref": "resource:/3.0/log.json" - }, - { - "$ref": "resource:/3.0/search.json" - }, - { - "$ref": "resource:/3.0/exdata.json" - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/exdata.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/exdata.json deleted file mode 100644 index 22495c4672..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/exdata.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/exdata", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "EXDATA" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "data": { - "id": "http://api.ekstep.org/telemetry/edata/data", - "type": "string" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/feedback.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/feedback.json deleted file mode 100644 index fa3385f93e..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/feedback.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/feedback", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "FEEDBACK" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "properties": { - "rating": { - "id": "http://api.ekstep.org/telemetry/edata/rating", - "type": "number" - }, - "comments": { - "id": "http://api.ekstep.org/telemetry/edata/comments", - "type": "string" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/heartbeat.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/heartbeat.json deleted file mode 100644 index e561482850..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/heartbeat.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/heartbeat", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "HEARTBEAT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object" - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/impression.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/impression.json deleted file mode 100644 index 44b37a08d2..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/impression.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/impression", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "IMPRESSION" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "pageid", - "uri" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/uri", - "type": "string" - }, - "visits": { - "id": "http://api.ekstep.org/telemetry/edata/visits", - "type": "array", - "items": { - "type": "object", - "properties": { - "objid": { - "id": "http://api.ekstep.org/telemetry/edata/visits/objid", - "type": "string" - }, - "objtype": { - "id": "http://api.ekstep.org/telemetry/edata/visits/objtype", - "type": "string" - }, - "objver": { - "id": "http://api.ekstep.org/telemetry/edata/visits/objver", - "type": "string" - }, - "section": { - "id": "http://api.ekstep.org/telemetry/edata/visits/section", - "type": "string" - }, - "index": { - "id": "http://api.ekstep.org/telemetry/edata/visits/index", - "type": "number" - } - }, - "additionalProperties": false, - "required": [ - "objid", - "objtype" - ] - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/inlineobject.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/inlineobject.json deleted file mode 100644 index 355b8c7834..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/inlineobject.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "type", - "ver" - ], - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/inlineobject/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/inlineobject/type", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/inlineobject/ver", - "type": "string" - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/inlineobject/subtype", - "type": "string" - }, - "name": { - "id": "http://api.ekstep.org/telemetry/inlineobject/name", - "type": "string" - }, - "code": { - "id": "http://api.ekstep.org/telemetry/inlineobject/code", - "type": "string" - }, - "parent": { - "$ref": "resource:/3.0/parent.json" - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/interact.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/interact.json deleted file mode 100644 index 4ee513c522..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/interact.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/interact", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/id", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "target": { - "$ref": "resource:/3.0/target.json" - }, - "plugin": { - "$ref": "resource:/3.0/plugin.json" - }, - "extra": { - "id": "http://api.ekstep.org/telemetry/edata/extra", - "type": "object" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/interrupt.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/interrupt.json deleted file mode 100644 index 2e6b602b89..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/interrupt.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/interrupt", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/items.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/items.json deleted file mode 100644 index 937cd76555..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/items.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/items", - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/items/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/items/type", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/items/ver", - "type": "string" - }, - "params": { - "id": "http://api.ekstep.org/telemetry/items/params", - "type": "array", - "items": { - "type": "object" - } - }, - "origin": { - "id": "http://api.ekstep.org/telemetry/items/origin", - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/items/origin/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/items/origin/type", - "type": "string" - } - } - }, - "to": { - "id": "http://api.ekstep.org/telemetry/items/to", - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/items/to/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/items/to/type", - "type": "string" - } - } - } - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/log.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/log.json deleted file mode 100644 index 72ebf5af95..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/log.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/log", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "LOG" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "level", - "message" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "level": { - "id": "http://api.ekstep.org/telemetry/edata/level", - "type": "string" - }, - "message": { - "id": "http://api.ekstep.org/telemetry/edata/message", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "params": { - "id": "http://api.ekstep.org/telemetry/edata/params", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/object.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/object.json deleted file mode 100644 index a22bf39f09..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/object.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/object/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/object/type", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/object/ver", - "type": "string" - }, - "rollup": { - "$ref": "resource:/3.0/rollup.json" - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/parent.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/parent.json deleted file mode 100644 index 07d4262af7..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/parent.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/parent/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/parent/type", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/pdata.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/pdata.json deleted file mode 100644 index 18f9ba13b6..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/pdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/pdata/id", - "type": "string" - }, - "pid": { - "id": "http://api.ekstep.org/telemetry/pdata/pid", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/pdata/ver", - "type": "string" - } - }, - "required": ["id"] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/plugin.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/plugin.json deleted file mode 100644 index cfd7d80ba9..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/plugin.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "ver" - ], - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/plugin/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/plugin/ver", - "type": "string" - }, - "category": { - "id": "http://api.ekstep.org/telemetry/plugin/category", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/question.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/question.json deleted file mode 100644 index f9ef60f3a0..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/question.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "type": "object", - "additionalProperties": false, - "required": [ - "id" - ], - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/question/id", - "type": "string" - }, - "maxscore": { - "id": "http://api.ekstep.org/telemetry/question/maxscore", - "type": "number" - }, - "exlength": { - "id": "http://api.ekstep.org/telemetry/question/exlength", - "type": "number" - }, - "params": { - "id": "http://api.ekstep.org/telemetry/question/params", - "type": "array", - "items": { - "type": "object" - } - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/question/uri", - "type": "string" - }, - "desc": { - "id": "http://api.ekstep.org/telemetry/question/desc", - "type": "string" - }, - "title": { - "id": "http://api.ekstep.org/telemetry/question/title", - "type": "string" - }, - "mmc": { - "id": "http://api.ekstep.org/telemetry/question/mmc", - "type": "array", - "items": { - "type": "string" - } - }, - "mc": { - "id": "http://api.ekstep.org/telemetry/question/mc", - "type": "array", - "items": { - "type": "string" - } - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/response.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/response.json deleted file mode 100644 index 4d86626635..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/response.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "RESPONSE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "target", - "type", - "values" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/type", - "type": "string" - }, - "target": { - "$ref": "resource:/3.0/target.json" - }, - "values": { - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/rollup.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/rollup.json deleted file mode 100644 index bfd89209f2..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/rollup.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "type": "object", - "properties": { - "l1": { - "id": "http://api.ekstep.org/telemetry/context/rollup/l1", - "type": "string" - }, - "l2": { - "id": "http://api.ekstep.org/telemetry/context/rollup/l2", - "type": "string" - }, - "l3": { - "id": "http://api.ekstep.org/telemetry/context/rollup/l3", - "type": "string" - }, - "l4": { - "id": "http://api.ekstep.org/telemetry/context/rollup/l4", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/search.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/search.json deleted file mode 100644 index 9bbfc3c3e9..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/search.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/search", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "SEARCH" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "query", - "size", - "topn" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "query": { - "id": "http://api.ekstep.org/telemetry/edata/query", - "type": "string" - }, - "filters": { - "id": "http://api.ekstep.org/telemetry/edata/filters", - "type": "object" - }, - "sort": { - "id": "http://api.ekstep.org/telemetry/edata/sort", - "type": "object" - }, - "correlationid": { - "id": "http://api.ekstep.org/telemetry/edata/correlationid", - "type": "string" - }, - "size": { - "id": "http://api.ekstep.org/telemetry/edata/size", - "type": "number" - }, - "topn": { - "id": "http://api.ekstep.org/telemetry/edata/topn", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/share.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/share.json deleted file mode 100644 index 7a21254ff8..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/share.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/share", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "SHARE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "items" - ], - "properties": { - "dir": { - "id": "http://api.ekstep.org/telemetry/edata/dir", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "items": { - "$ref": "resource:/3.0/items.json" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/start.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/start.json deleted file mode 100644 index 78b8f7694a..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/start.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "resource:/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "dspec": { - "$ref": "resource:/3.0/dspec.json" - }, - "uaspec": { - "$ref": "resource:/3.0/uaspec.json" - }, - "loc": { - "id": "http://api.ekstep.org/telemetry/edata/loc", - "type": "string" - }, - "mode": { - "id": "http://api.ekstep.org/telemetry/edata/mode", - "type": "string" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/target.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/target.json deleted file mode 100644 index 6ababb8322..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/target.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "ver", - "type" - ], - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/target/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/target/ver", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/target/type", - "type": "string" - }, - "parent": { - "$ref": "resource:/3.0/parent.json" - } - } -} \ No newline at end of file diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/uaspec.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/uaspec.json deleted file mode 100644 index cc7d9f53c6..0000000000 --- a/data-pipeline/telemetry-validator/src/test/resources/3.0/uaspec.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "type": "object", - "properties": { - "agent": { - "id": "http://api.ekstep.org/telemetry/uaspec/agent", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/uaspec/ver", - "type": "string" - }, - "system": { - "id": "http://api.ekstep.org/telemetry/uaspec/system", - "type": "string" - }, - "platform": { - "id": "http://api.ekstep.org/telemetry/uaspec/platform", - "type": "string" - }, - "raw": { - "id": "http://api.ekstep.org/telemetry/uaspec/raw", - "type": "string" - } - } -} \ No newline at end of file diff --git a/data-pipeline/user-cache-updater/pom.xml b/data-pipeline/user-cache-updater/pom.xml deleted file mode 100644 index 2c0d1a4f94..0000000000 --- a/data-pipeline/user-cache-updater/pom.xml +++ /dev/null @@ -1,169 +0,0 @@ - - - - jobs - org.ekstep.ecosystem - 0.0.1 - - - 4.0.0 - - org.ekstep.ecosystem.jobs - user-cache-updater - 0.0.2 - UserCacheUpdater - - - org.apache.samza - samza-api - ${samza.version} - - - org.apache.samza - samza-log4j - ${samza.version} - - - org.apache.samza - samza-shell - dist - tgz - ${samza.version} - - - org.apache.kafka - kafka_2.11 - ${kafka.version} - - - org.slf4j - slf4j-api - 1.7.15 - - - org.slf4j - slf4j-log4j12 - 1.7.15 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - org.apache.hadoop - hadoop-hdfs - 2.4.0 - - - junit - junit - 4.12 - test - - - org.mockito - mockito-core - 2.0.31-beta - test - - - org.ekstep.ecosystem - ep-core - 0.0.1 - - - org.ekstep.ecosystem - ep-telemetry-reader - 0.0.3 - - - redis.clients - jedis - 2.9.0 - - - com.google.code.gson - gson - 2.4 - - - com.fiftyonred - mock-jedis - 0.4.0 - jar - test - - - com.google.guava - guava - 18.0 - - - org.jetbrains - annotations - RELEASE - compile - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - maven-surefire-plugin - 2.20 - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - - - - jacoco-initialize - - prepare-agent - - - - post-unit-test - package - - report - - - ${basedir}/target/coverage-reports/jacoco-ut - - - - - - - - - diff --git a/data-pipeline/user-cache-updater/src/main/assembly/src.xml b/data-pipeline/user-cache-updater/src/main/assembly/src.xml deleted file mode 100644 index a41bb4514d..0000000000 --- a/data-pipeline/user-cache-updater/src/main/assembly/src.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - distribution - - tar.gz - - false - - - ${basedir} - - README* - LICENSE* - NOTICE* - - - - - - ${basedir}/src/main/resources/log4j.xml - lib - - - - ${basedir}/src/main/config/user-cache-updater.properties - config - true - - - - - bin - - org.apache.samza:samza-shell:tgz:dist:* - - 0744 - true - - - lib - - org.apache.samza:samza-core_2.11 - org.apache.samza:samza-kafka_2.11 - org.apache.samza:samza-yarn_2.11 - org.apache.samza:samza-log4j - org.ekstep.ecosystem.jobs:user-cache-updater - org.slf4j:slf4j-log4j12 - org.apache.kafka:kafka_2.11 - org.apache.hadoop:hadoop-hdfs - redis.clients:2.9.0 - - true - - - diff --git a/data-pipeline/user-cache-updater/src/main/config/user-cache-updater.properties b/data-pipeline/user-cache-updater/src/main/config/user-cache-updater.properties deleted file mode 100644 index a0d6556cdd..0000000000 --- a/data-pipeline/user-cache-updater/src/main/config/user-cache-updater.properties +++ /dev/null @@ -1,96 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Job -job.factory.class=org.apache.samza.job.yarn.YarnJobFactory -job.name=__env__.UserCacheUpdater - -# YARN -#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz -yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz -#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz -job.container.count=__redis_updater_yarn_container_count__ - -# Task -task.class=org.ekstep.ep.samza.task.UserCacheUpdaterTask -task.inputs=kafka.__env__.telemetry.audit -task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory -task.checkpoint.system=kafka -task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__ -task.commit.ms=60000 -task.window.ms=300000 - -# Systems -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -systems.kafka.samza.key.serde=string -systems.kafka.samza.msg.serde=string -# systems.kafka.consumer.zookeeper.connect=localhost:2181/ -systems.kafka.consumer.zookeeper.connect=__zookeepers__ -systems.kafka.consumer.auto.offset.reset=smallest -systems.kafka.samza.offset.default=oldest -systems.kafka.producer.bootstrap.servers=__kafka_brokers__ -systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics -systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__ -systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__ - - -# Metrics -metrics.reporters=snapshot -metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory -metrics.reporter.snapshot.stream=kafka.__env__.metrics -output.metrics.topic.name=__env__.pipeline_metrics -output.malformed.topic.name=__env__.telemetry.malformed -output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics -pipeline.metrics.list=success-message-count,skipped-message-count,error-message-count - -# Job Coordinator -job.coordinator.system=kafka -job.coordinator.replication.factor=__samza_checkpoint_replication_factor__ - -# Serializers -serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory -serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory -serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory - - -# redis -# redis.host=localhost -redis.host=__metadata_redis_host__ -# redis.port=6379 -redis.port=__metadata_redis_port__ -redis.connection.max=3 -redis.connection.idle.max=3 -redis.connection.idle.min=1 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.database.userStore.id=4 - -# content model fields to convert to List from String -contentModel.fields.listType=gradeLevel,subject,medium,language -user.signin.type.default = __user_sigin_type_default__ -user.login.type.default = __user_login_type_default__ -user.selfsignedin.typeList = google,self -user.validated.typeList = sso -user.self-siginin.key = Self-Signed-In -user.valid.key = Validated - -# sunbird middleware cassandra config -middleware.cassandra.host=__middleware_cassandra_host__ -middleware.cassandra.port=__middleware_cassandra_port__ -middleware.cassandra.keyspace=__middleware_cassandra_keyspace__ -middleware.cassandra.user_table=__middleware_cassandra_user_table__ -middleware.cassandra.location_table=__middleware_cassandra_location_table__ diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java deleted file mode 100644 index aa9ad22d9d..0000000000 --- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.ekstep.ep.samza.domain; - -import org.ekstep.ep.samza.reader.NullableValue; -import org.ekstep.ep.samza.reader.Telemetry; - -import java.util.ArrayList; -import java.util.Map; - -public class Event { - private final Telemetry telemetry; - - public Event(Map map) { - this.telemetry = new Telemetry(map); - } - - public String objectUserId() { - NullableValue objectType = telemetry.read("object.type"); - NullableValue objectUserId = null; - if (null != objectType.value() && objectType.value().equalsIgnoreCase("User")) { - objectUserId = telemetry.read("object.id"); - } - return null!=objectUserId ? objectUserId.value(): null; - } - - public String getUserSignInType() { - NullableValue>> cdata = telemetry.read("context.cdata"); - ArrayList> cdataList = cdata.value(); - if (null != cdataList && !cdataList.isEmpty()) { - for (Map cdataMap : cdataList) { - if (cdataMap.containsKey("type") && cdataMap.get("type").equalsIgnoreCase("SignupType")) - return cdataMap.get("id").toString(); - } - } - return null; - } - - public String getUserLoginType() { - NullableValue>> cdata = telemetry.read("context.cdata"); - ArrayList> cdataList = cdata.value(); - if (null != cdataList && !cdataList.isEmpty()) { - for (Map cdataMap : cdataList) { - if (cdataMap.containsKey("type") && cdataMap.get("type").equalsIgnoreCase("UserRole")) - return cdataMap.get("id").toString(); - } - } - return null; - } - - public ArrayList getUserMetdataUpdatedList() { - NullableValue> edata_props = telemetry.read("edata.props"); - if(null != edata_props) { - ArrayList edata_props_list = edata_props.value(); - if(null != edata_props_list && !edata_props_list.isEmpty()) { - return edata_props_list; - } - } - return null; - } - - public String getUserStateValue() { - NullableValue edata_state = telemetry.read("edata.state"); - if( null != edata_state.value() && edata_state.value().equalsIgnoreCase("Create") || edata_state.value().equalsIgnoreCase("Created")) { - return "Create"; - } - else if( null != edata_state.value() && edata_state.value().equalsIgnoreCase("Update") || edata_state.value().equalsIgnoreCase("Updated")) { - return "Update"; - } - return null; - } - -} \ No newline at end of file diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/service/UserCacheUpdaterService.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/service/UserCacheUpdaterService.java deleted file mode 100644 index 0a3ea9ca9e..0000000000 --- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/service/UserCacheUpdaterService.java +++ /dev/null @@ -1,150 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.datastax.driver.core.ColumnDefinitions; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.querybuilder.Clause; -import com.datastax.driver.core.querybuilder.QueryBuilder; -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; -import com.google.gson.reflect.TypeToken; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.core.Logger; -import org.ekstep.ep.samza.domain.Event; -import org.ekstep.ep.samza.task.UserCacheUpdaterSink; -import org.ekstep.ep.samza.task.UserCacheUpdaterSource; -import org.ekstep.ep.samza.task.UserCacheUpdaterConfig; -import org.ekstep.ep.samza.util.CassandraConnect; -import org.ekstep.ep.samza.util.RedisConnect; -import org.ekstep.ep.samza.core.BaseCacheUpdaterService; - -import java.lang.reflect.Type; -import java.util.*; - -public class UserCacheUpdaterService extends BaseCacheUpdaterService { - - private static Logger LOGGER = new Logger(UserCacheUpdaterService.class); - private JobMetrics metrics; - private UserCacheUpdaterConfig userCacheUpdaterConfig; - private int userStoreDb; - private Gson gson = new Gson(); - private Type mapType = new TypeToken>() { - }.getType(); - - public UserCacheUpdaterService(UserCacheUpdaterConfig config, RedisConnect redisConnect, CassandraConnect cassandraConnect, JobMetrics metrics) { - super(redisConnect, cassandraConnect); - this.metrics = metrics; - userCacheUpdaterConfig = config; - this.userStoreDb = config.userStoreDb(); - } - - public void process(UserCacheUpdaterSource source, UserCacheUpdaterSink sink) { - try { - Event event = source.getEvent(); - String userId = event.objectUserId(); - if (null != userId ) { - updateUserCache(event, userId, sink); - } else { - sink.markSkipped(); - } - } catch (JsonSyntaxException e) { - e.printStackTrace(); - LOGGER.error(null, "INVALID EVENT: " + source.getMessage()); - sink.error(); - } - } - - public Map updateUserCache(Event event, String userId, UserCacheUpdaterSink sink) { - Map userCacheData = new HashMap<>(); - String userState = event.getUserStateValue(); - if("Update".equals(userState)) { - String data = readFromCache(userId, userStoreDb); - if (data != null && !data.isEmpty()) { - userCacheData = gson.fromJson(data, mapType); - } - userCacheData.putAll(getUserLoginType(event)); - ArrayList userUpdatedList = event.getUserMetdataUpdatedList(); - if (userCacheData.containsKey("usersignintype") && !"Anonymous".equals(userCacheData.get("usersignintype")) - && null != userUpdatedList && !userUpdatedList.isEmpty()) { - Clause userDataClause = QueryBuilder.eq("id",userId); - List userDetails = readFromCassandra(userCacheUpdaterConfig.cassandra_db(), userCacheUpdaterConfig.cassandra_user_table(), userDataClause); - Map userMetadataInfoMap = getUserMetaDataInfo(userDetails); - - if (!userMetadataInfoMap.isEmpty()) - userCacheData.putAll(userMetadataInfoMap); - - if (userUpdatedList.contains("id") && null != userCacheData.get("locationids")) { - List locationIds = (List) userCacheData.get("locationids"); - Clause locationDataClause = QueryBuilder.in("id",locationIds); - - List locationDetails = readFromCassandra(userCacheUpdaterConfig.cassandra_db(), userCacheUpdaterConfig.cassandra_location_table(), locationDataClause); - Map userLocationMap = getLocationInfo(locationDetails); - - if (!userLocationMap.isEmpty()) - userCacheData.putAll(userLocationMap); - } - } - } - else if("Create".equals(userState)) { - userCacheData.putAll(getUserSigninType(event)); - } - - if (!userCacheData.isEmpty()) { - addToCache(userId, gson.toJson(userCacheData), userStoreDb); - LOGGER.info(userId, "Updated in cache"); - sink.success(); - } else { - sink.markSkipped(); - } - return userCacheData; - } - - private Map getUserSigninType(Event event) { - Map userData = new HashMap<>(); - String signInType = null != event.getUserSignInType() ? event.getUserSignInType() : userCacheUpdaterConfig.getUserSignInTypeDefault(); - if (!(userCacheUpdaterConfig.getUserSignInTypeDefault().equalsIgnoreCase(signInType))) { - if (userCacheUpdaterConfig.getUserSelfSignedInTypeList().contains(signInType)) { - userData.put("usersignintype", userCacheUpdaterConfig.getUserSelfSignedKey()); - } else if (userCacheUpdaterConfig.getUserValidatedTypeList().contains(signInType)) { - userData.put("usersignintype", userCacheUpdaterConfig.getUserValidatedKey()); - } - } - return userData; - } - - private Map getUserLoginType(Event event) { - Map userData = new HashMap<>(); - String loginInType = null != event.getUserLoginType() ? event.getUserLoginType() : userCacheUpdaterConfig.getUserLoginInTypeDefault(); - if (!(userCacheUpdaterConfig.getUserLoginInTypeDefault().equalsIgnoreCase(loginInType))) - userData.put("userlogintype", loginInType); - return userData; - } - - public Map getUserMetaDataInfo(List userDetails) { - Map result = new HashMap<>(); - if( null != userDetails && userDetails.size() > 0) { - Row row= userDetails.get(0); - ColumnDefinitions columnDefinitions = row.getColumnDefinitions(); - int columnCount = columnDefinitions.size(); - for(int i=0; i getLocationInfo(List locationDetails) { - Map result = new HashMap<>(); - if(null != locationDetails && !locationDetails.isEmpty()) { - locationDetails.forEach(record -> { - String name = record.getString("name"); - String type = record.getString("type"); - if(type.toLowerCase().equals("state")) { - result.put("state", name); - } else if(type.toLowerCase().equals("district")) { - result.put("district", name); - } - }); - } - return result; - } -} diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterConfig.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterConfig.java deleted file mode 100644 index d8351b5cb1..0000000000 --- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterConfig.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.config.Config; - -import java.util.Arrays; -import java.util.List; - -public class UserCacheUpdaterConfig { - - private final String JOB_NAME = "UserCacheUpdater"; - - private String userSignInTypeDefault; - private String userLoginInTypeDefault; - private List userSelfSignedInTypeList; - private List userValidatedTypeList; - private String userSelfSignedKey; - private String userValidatedKey; - private String cassandra_db; - private String cassandra_user_table; - private String cassandra_location_table; - private int userStoreDb; - - public UserCacheUpdaterConfig(Config config) { - this.userSignInTypeDefault = config.get("user.signin.type.default", "Anonymous"); - this.userLoginInTypeDefault = config.get("user.login.type.default", "NA"); - this.userSelfSignedInTypeList = config.getList("user.selfsignedin.typeList", Arrays.asList("google", "self")); - this.userValidatedTypeList = config.getList("user.validated.typeList", Arrays.asList("sso")); - this.userSelfSignedKey = config.get("user.self-siginin.key", "Self-Signed-In"); - this.userValidatedKey = config.get("user.valid.key", "Validated"); - this.cassandra_db = config.get("middleware.cassandra.keyspace", "sunbird"); - this.cassandra_user_table = config.get("middleware.cassandra.user_table", "user"); - this.cassandra_location_table = config.get("middleware.cassandra.location_table", "location"); - this.userStoreDb = config.getInt("redis.database.userStore.id", 4); - } - - public String getUserSignInTypeDefault() { return userSignInTypeDefault; } - - public String getUserLoginInTypeDefault() { return userLoginInTypeDefault; } - - public String getUserSelfSignedKey() { return userSelfSignedKey; } - - public String getUserValidatedKey() { return userValidatedKey; } - - public List getUserSelfSignedInTypeList() { return userSelfSignedInTypeList; } - - public List getUserValidatedTypeList() { return userValidatedTypeList; } - - public String cassandra_db() { return cassandra_db; } - - public String cassandra_user_table() { return cassandra_user_table; } - - public String cassandra_location_table() { return cassandra_location_table; } - - public String jobName() { return JOB_NAME; } - - public int userStoreDb() { return userStoreDb; } - -} diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSink.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSink.java deleted file mode 100644 index 02e612da99..0000000000 --- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSink.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.ekstep.ep.samza.task; - -import org.apache.samza.task.MessageCollector; -import org.ekstep.ep.samza.core.BaseSink; -import org.ekstep.ep.samza.core.JobMetrics; - -public class UserCacheUpdaterSink extends BaseSink { - - public UserCacheUpdaterSink(MessageCollector collector, JobMetrics metrics) { - super(collector, metrics); - } - - public void success() { - metrics.incSuccessCounter(); - } - - public void error() { - metrics.incErrorCounter(); - } - - public void markSkipped() { metrics.incSkippedCounter(); } - -} diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSource.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSource.java deleted file mode 100644 index 1e8817f40e..0000000000 --- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSource.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.ekstep.ep.samza.task; - -import com.google.gson.Gson; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.ekstep.ep.samza.domain.Event; - -import java.util.Map; - -public class UserCacheUpdaterSource { - - private IncomingMessageEnvelope envelope; - - public UserCacheUpdaterSource(IncomingMessageEnvelope envelope) { - this.envelope = envelope; - } - - public Event getEvent() { - String message = (String) envelope.getMessage(); - @SuppressWarnings("unchecked") - Map jsonMap = (Map) new Gson().fromJson(message, Map.class); - return new Event(jsonMap); - } - - public String getMessage() { - return envelope.toString(); - } -} diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterTask.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterTask.java deleted file mode 100644 index 773145260d..0000000000 --- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterTask.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.ekstep.ep.samza.task; - -import java.util.Arrays; -import java.util.List; - -import org.apache.samza.config.Config; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.UserCacheUpdaterService; -import org.ekstep.ep.samza.util.CassandraConnect; -import org.ekstep.ep.samza.util.RedisConnect; - -public class UserCacheUpdaterTask extends BaseSamzaTask { - - private JobMetrics metrics; - private UserCacheUpdaterConfig config; - private UserCacheUpdaterService service; - - public UserCacheUpdaterTask(Config config, TaskContext context, CassandraConnect cassandraConnect, RedisConnect redisConnect) { - init(config, context, cassandraConnect, redisConnect); - } - - public UserCacheUpdaterTask() { - } - - @Override - public void init(Config config, TaskContext context) { - init(config, context, null, null); - } - - public void init(Config config, TaskContext context, CassandraConnect cassandraConnect, RedisConnect redisConnect) { - this.config = new UserCacheUpdaterConfig(config); - metrics = new JobMetrics(context, this.config.jobName()); - List cassandraHosts = Arrays.asList(config.get("middleware.cassandra.host", "127.0.0.1").split(",")); - cassandraConnect = null != cassandraConnect ? cassandraConnect : new CassandraConnect(cassandraHosts, config.getInt("middleware..cassandra.port", 9042)); - redisConnect = null != redisConnect? redisConnect: new RedisConnect(config); - service = new UserCacheUpdaterService(this.config, redisConnect, cassandraConnect, metrics); - this.initTask(config, metrics); - } - - @Override - public void process(IncomingMessageEnvelope envelope, MessageCollector collector, - TaskCoordinator taskCoordinator) throws Exception { - - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelope); - UserCacheUpdaterSink sink = new UserCacheUpdaterSink(collector, metrics); - service.process(source, sink); - } - -} diff --git a/data-pipeline/user-cache-updater/src/main/resources/log4j.xml b/data-pipeline/user-cache-updater/src/main/resources/log4j.xml deleted file mode 100644 index 7e0a688c9b..0000000000 --- a/data-pipeline/user-cache-updater/src/main/resources/log4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java b/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java deleted file mode 100644 index 5e86146f4a..0000000000 --- a/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java +++ /dev/null @@ -1,404 +0,0 @@ -package org.ekstep.ep.samza.service.Fixtures; - -public class EventFixture { - - public static final String AUDIT_EVENT = "{\"eid\":\"AUDIT\",\"ets\":1.573121861118E12,\"ver\":\"3.0\",\"mid\":\"1573121861118.40f9136b-1cc3-458d-a04a-4459606dfdd6\",\"actor\":{\"id\":\"627a431d-4f5c-4adc-812d-1f01c5588555\",\"type\":\"User\"},\"context\":{\"channel\":\"01285019302823526477\",\"pdata\":{\"id\":\"dev.sunbird.portal\",\"pid\":\"learner-service\",\"ver\":\"2.5.0\"},\"env\":\"User\",\"did\":\"2bcfc645e27e64625f7bad6ce282f9d0\",\"cdata\":[{\"id\":\"25cb0530-7c52-ecb1-cff2-6a14faab7910\",\"type\":\"SignupType\"}],\"rollup\":{\"l1\":\"01285019302823526477\"}},\"object\":{\"id\":\"627a431d-4f5c-4adc-812d-1f01c5588555\",\"type\":\"User\"},\"edata\":{\"state\":\"Update\",\"props\":[\"recoveryEmail\",\"recoveryPhone\",\"userId\",\"id\",\"externalIds\",\"updatedDate\",\"updatedBy\"]},\"syncts\":1.573121861125E12,\"@timestamp\":\"2019-11-07T10:17:41.125Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\",\"ts\":\"2019-11-07T10:17:41.118+0000\"}"; - public static final String AUDIT_EVENT2 = "{\"eid\":\"AUDIT\",\"ets\":1.573121861118E12,\"ver\":\"3.0\",\"mid\":\"1573121861118.40f9136b-1cc3-458d-a04a-4459606df\",\"actor\":{\"id\":\"5609876543234567890987654345678\",\"type\":\"Request\"},\"context\":{\"channel\":\"01285019302823526477\",\"pdata\":{\"id\":\"dev.sunbird.portal\",\"pid\":\"learner-service\",\"ver\":\"2.5.0\"},\"env\":\"User\",\"did\":\"2bcfc645e27e64625f7bad6ce282f9d0\",\"cdata\":[{\"id\":\"25cb0530-7c52-ecb1-cff2-6a14faab7910\",\"type\":\"SignupType\"}],\"rollup\":{\"l1\":\"01285019302823526477\"}},\"object\":{\"id\":\"5609876543h2fd34h5678jf909876af54345678\"},\"edata\":{\"state\":\"Update\",\"props\":[\"recoveryEmail\",\"recoveryPhone\",\"userId\",\"id\",\"externalIds\",\"updatedDate\",\"updatedBy\"]},\"syncts\":1.573121861125E12,\"@timestamp\":\"2019-11-07T10:17:41.125Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\",\"ts\":\"2019-11-07T10:17:41.118+0000\"}"; - public static final String AUDIT_EVENT3 = "{\"eid\":\"AUDIT\",\"ets\":1.573121861118E12,\"ver\":\"3.0\",\"mid\":\"1573121861118.40f9136b-1cc3-458d-a04a-4459606df\",\"actor\":{\"id\":\"5609876543234567890987654345678\",\"type\":\"Request\"},\"context\":{\"channel\":\"01285019302823526477\",\"pdata\":{\"id\":\"dev.sunbird.portal\",\"pid\":\"learner-service\",\"ver\":\"2.5.0\"},\"env\":\"User\",\"did\":\"2bcfc645e27e64625f7bad6ce282f9d0\",\"rollup\":{\"l1\":\"01285019302823526477\"}},\"object\":{\"type\":\"User\",\"id\":\"5609876543h2fd34h5678jf909876af54345678\"},\"edata\":{\"state\":\"Create\",\"props\":[\"recoveryEmail\",\"recoveryPhone\",\"userId\",\"id\",\"externalIds\",\"updatedDate\",\"updatedBy\"]},\"syncts\":1.573121861125E12,\"@timestamp\":\"2019-11-07T10:17:41.125Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\",\"ts\":\"2019-11-07T10:17:41.118+0000\"}"; - - public static final String AUDIT_EVENT_SIGINTYPE="{\n" + - " \"actor\": {\n" + - " \"type\": \"Consumer\",\n" + - " \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" + - " },\n" + - " \"eid\": \"AUDIT\",\n" + - " \"edata\": {\n" + - " \"state\": \"Create\",\n" + - " \"props\": [\n" + - " \"firstName\",\n" + - " \"email\",\n" + - " \"emailVerified\",\n" + - " \"id\",\n" + - " \"userId\",\n" + - " \"createdBy\",\n" + - " \"rootOrgId\",\n" + - " \"channel\",\n" + - " \"userType\",\n" + - " \"roles\",\n" + - " \"phoneVerified\",\n" + - " \"isDeleted\",\n" + - " \"createdDate\",\n" + - " \"status\",\n" + - " \"userName\",\n" + - " \"loginId\",\n" + - " \"externalIds\"\n" + - " ]\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1561739226844,\n" + - " \"context\": {\n" + - " \"channel\": \"0126684405014528002\",\n" + - " \"pdata\": {\n" + - " \"pid\": \"learner-service\",\n" + - " \"ver\": \"2.0.0\",\n" + - " \"id\": \"prod.diksha.learning.service\"\n" + - " },\n" + - " \"env\": \"User\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"34881c3a-8b92-4a3c-a982-7f946137cb09\"\n" + - " },\n" + - " {\n" + - " \"type\": \"SignupType\",\n" + - " \"id\": \"google\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Source\",\n" + - " \"id\": \"android\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Request\",\n" + - " \"id\": \"91f3c280-99c1-11e9-956e-6b6ef71ed575\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0126684405014528002\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8\",\n" + - " \"object\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" + - " },\n" + - " \"syncts\": 1561739243532,\n" + - " \"@timestamp\": \"2019-06-28T16:27:23.532Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String AUDIT_EVENT_SIGIN_TYPE="{\n" + - " \"actor\": {\n" + - " \"type\": \"Consumer\",\n" + - " \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" + - " },\n" + - " \"eid\": \"AUDIT\",\n" + - " \"edata\": {\n" + - " \"state\": \"Created\",\n" + - " \"props\": [\n" + - " \"firstName\",\n" + - " \"email\",\n" + - " \"emailVerified\",\n" + - " \"id\",\n" + - " \"userId\",\n" + - " \"createdBy\",\n" + - " \"rootOrgId\",\n" + - " \"channel\",\n" + - " \"userType\",\n" + - " \"roles\",\n" + - " \"phoneVerified\",\n" + - " \"isDeleted\",\n" + - " \"createdDate\",\n" + - " \"status\",\n" + - " \"userName\",\n" + - " \"loginId\",\n" + - " \"externalIds\"\n" + - " ]\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1561739226844,\n" + - " \"context\": {\n" + - " \"channel\": \"0126684405014528002\",\n" + - " \"pdata\": {\n" + - " \"pid\": \"learner-service\",\n" + - " \"ver\": \"2.0.0\",\n" + - " \"id\": \"prod.diksha.learning.service\"\n" + - " },\n" + - " \"env\": \"User\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"34881c3a-8b92-4a3c-a982-7f946137cb09\"\n" + - " },\n" + - " {\n" + - " \"type\": \"SignupType\",\n" + - " \"id\": \"sso\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Source\",\n" + - " \"id\": \"android\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Request\",\n" + - " \"id\": \"91f3c280-99c1-11e9-956e-6b6ef71ed575\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0126684405014528002\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8\",\n" + - " \"object\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" + - " },\n" + - " \"syncts\": 1561739243532,\n" + - " \"@timestamp\": \"2019-06-28T16:27:23.532Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String AUDIT_EVENT_NULL_USERID ="{\n" + - " \"actor\": {\n" + - " \"type\": \"Consumer\",\n" + - " \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" + - " },\n" + - " \"eid\": \"AUDIT\",\n" + - " \"edata\": {\n" + - " \"state\": \"Create\",\n" + - " \"props\": [\n" + - " \"firstName\",\n" + - " \"email\",\n" + - " \"emailVerified\",\n" + - " \"id\",\n" + - " \"userId\",\n" + - " \"createdBy\",\n" + - " \"rootOrgId\",\n" + - " \"channel\",\n" + - " \"userType\",\n" + - " \"roles\",\n" + - " \"phoneVerified\",\n" + - " \"isDeleted\",\n" + - " \"createdDate\",\n" + - " \"status\",\n" + - " \"userName\",\n" + - " \"loginId\",\n" + - " \"externalIds\"\n" + - " ]\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1561739226844,\n" + - " \"context\": {\n" + - " \"channel\": \"0126684405014528002\",\n" + - " \"pdata\": {\n" + - " \"pid\": \"learner-service\",\n" + - " \"ver\": \"2.0.0\",\n" + - " \"id\": \"prod.diksha.learning.service\"\n" + - " },\n" + - " \"env\": \"User\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"34881c3a-8b92-4a3c-a982-7f946137cb09\"\n" + - " },\n" + - " {\n" + - " \"type\": \"SignupType\",\n" + - " \"id\": \"sso\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Source\",\n" + - " \"id\": \"android\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Request\",\n" + - " \"id\": \"91f3c280-99c1-11e9-956e-6b6ef71ed575\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0126684405014528002\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8\",\n" + - " \"object\": {\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"syncts\": 1561739243532,\n" + - " \"@timestamp\": \"2019-06-28T16:27:23.532Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String AUDIT_EVENT_SIGN_IN="{\n" + - " \"actor\": {\n" + - " \"type\": \"Consumer\",\n" + - " \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" + - " },\n" + - " \"eid\": \"AUDIT\",\n" + - " \"edata\": {\n" + - " \"state\": \"Create\",\n" + - " \"props\": [\n" + - " \"firstName\",\n" + - " \"email\",\n" + - " \"emailVerified\",\n" + - " \"id\",\n" + - " \"userId\",\n" + - " \"createdBy\",\n" + - " \"rootOrgId\",\n" + - " \"channel\",\n" + - " \"userType\",\n" + - " \"roles\",\n" + - " \"phoneVerified\",\n" + - " \"isDeleted\",\n" + - " \"createdDate\",\n" + - " \"status\",\n" + - " \"userName\",\n" + - " \"loginId\",\n" + - " \"externalIds\"\n" + - " ]\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1561739226844,\n" + - " \"context\": {\n" + - " \"channel\": \"0126684405014528002\",\n" + - " \"pdata\": {\n" + - " \"pid\": \"learner-service\",\n" + - " \"ver\": \"2.0.0\",\n" + - " \"id\": \"prod.diksha.learning.service\"\n" + - " },\n" + - " \"env\": \"User\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"34881c3a-8b92-4a3c-a982-7f946137cb09\"\n" + - " },\n" + - " {\n" + - " \"type\": \"SignupType\",\n" + - " \"id\": \"sso\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Source\",\n" + - " \"id\": \"android\"\n" + - " },\n" + - " {\n" + - " \"type\": \"Request\",\n" + - " \"id\": \"91f3c280-99c1-11e9-956e-6b6ef71ed575\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"0126684405014528002\"\n" + - " }\n" + - " },\n" + - " \"mid\": \"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8\",\n" + - " \"object\": {\n" + - " \"type\": \"User\",\n" + - " \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" + - " },\n" + - " \"syncts\": 1561739243532,\n" + - " \"@timestamp\": \"2019-06-28T16:27:23.532Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - public static final String AUDIT_EVENT_LOGININTYPE ="{\n" + - " \"actor\": {\n" + - " \"type\": \"System\",\n" + - " \"id\": \"3b46b4c9-3a10-439a-a2cb-feb5435b3a0d\"\n" + - " },\n" + - " \"eid\": \"AUDIT\",\n" + - " \"edata\": {\n" + - " \"state\": \"Update\",\n" + - " \"props\": [\n" + - " \"medium\",\n" + - " \"board\",\n" + - " \"grade\",\n" + - " \"syllabus\",\n" + - " \"gradeValue\"\n" + - " ]\n" + - " },\n" + - " \"ver\": \"3.0\",\n" + - " \"ets\": 1561739240727,\n" + - " \"context\": {\n" + - " \"pdata\": {\n" + - " \"pid\": \"sunbird.app\",\n" + - " \"ver\": \"2.1.92\",\n" + - " \"id\": \"prod.diksha.app\"\n" + - " },\n" + - " \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" + - " \"env\": \"sdk\",\n" + - " \"did\": \"010612971a80a7677d0a3e849ab35cb4a83157de\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"type\": \"UserRole\",\n" + - " \"id\": \"student\"\n" + - " }\n" + - " ],\n" + - " \"sid\": \"ea68a05e-0843-4c06-9a84-9b98cd974724\"\n" + - " },\n" + - " \"mid\": \"0268860e-76b0-4b4e-b99b-ebf543e7a9d8\",\n" + - " \"object\": {\n" + - " \"id\": \"3b46b4c9-3a10-439a-a2cb-feb5435b3a0d\",\n" + - " \"type\": \"User\",\n" + - " \"version\": \"\",\n" + - " \"rollup\": {}\n" + - " },\n" + - " \"syncts\": 1561739245463,\n" + - " \"@timestamp\": \"2019-06-28T16:27:25.463Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true\n" + - " },\n" + - " \"type\": \"events\"\n" + - "}"; - - public static final String AUDIT_EVENT_METADATA_UPDATED = "{\n" + - " \"eid\": \"AUDIT\",\n" + - " \"ets\": 1571297660511,\n" + - " \"ver\": \"3.0\",\n" + - " \"mid\": \"1571297660511.32f5024a-aa30-4c82-abd8-bb8d8914ed2d\",\n" + - " \"actor\": {\n" + - " \"id\": \"ef70da5a-bb99-4785-b970-1d6d6ee75aad\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"context\": {\n" + - " \"channel\": \"01285019302823526477\",\n" + - " \"pdata\": {\n" + - " \"id\": \"dev.sunbird.portal\",\n" + - " \"pid\": \"learner-service\",\n" + - " \"ver\": \"2.4.0\"\n" + - " },\n" + - " \"env\": \"User\",\n" + - " \"did\": \"bfaf115f65a2086b062735885f4d2f1a\",\n" + - " \"cdata\": [\n" + - " {\n" + - " \"id\": \"1b1392bc-39d8-6e47-d7d6-5781a2f1481a\",\n" + - " \"type\": \"Request\"\n" + - " }\n" + - " ],\n" + - " \"rollup\": {\n" + - " \"l1\": \"01285019302823526477\"\n" + - " }\n" + - " },\n" + - " \"object\": {\n" + - " \"id\": \"52226956-61d8-4c1b-b115-c660111866d3\",\n" + - " \"type\": \"User\"\n" + - " },\n" + - " \"edata\": {\n" + - " \"state\": \"Updated\",\n" + - " \"props\": [\n" + - " \"firstName\",\n" + - " \"userId\",\n" + - " \"id\",\n" + - " \"externalIds\",\n" + - " \"locationIds\",\n" + - " \"updatedDate\",\n" + - " \"updatedBy\"\n" + - " ]\n" + - " },\n" + - " \"syncts\": 1571297660521,\n" + - " \"@timestamp\": \"2019-10-17T07:34:20.521Z\",\n" + - " \"flags\": {\n" + - " \"tv_processed\": true,\n" + - " \"dd_processed\": true\n" + - " },\n" + - " \"type\": \"events\",\n" + - " \"ts\": \"2019-10-17T07:34:20.511+0000\"\n" + - "}"; - - public static final String ANY_STRING = "Any String"; -} diff --git a/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/UserCacheUpdaterServiceTest.java b/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/UserCacheUpdaterServiceTest.java deleted file mode 100644 index b08e26eb68..0000000000 --- a/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/UserCacheUpdaterServiceTest.java +++ /dev/null @@ -1,285 +0,0 @@ -package org.ekstep.ep.samza.service; - -import com.datastax.driver.core.Row; -import com.fiftyonred.mock_jedis.MockJedis; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.apache.samza.Partition; -import org.apache.samza.config.Config; -import org.apache.samza.metrics.Counter; -import org.apache.samza.metrics.MetricsRegistry; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.task.MessageCollector; -import org.apache.samza.task.TaskContext; -import org.apache.samza.task.TaskCoordinator; -import org.ekstep.ep.samza.core.JobMetrics; -import org.ekstep.ep.samza.service.Fixtures.EventFixture; -import org.ekstep.ep.samza.task.UserCacheUpdaterConfig; -import org.ekstep.ep.samza.task.UserCacheUpdaterSink; -import org.ekstep.ep.samza.task.UserCacheUpdaterSource; -import org.ekstep.ep.samza.task.UserCacheUpdaterTask; -import org.ekstep.ep.samza.core.BaseCacheUpdaterService; -import org.ekstep.ep.samza.util.CassandraConnect; -import org.ekstep.ep.samza.util.RedisConnect; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import redis.clients.jedis.Jedis; - -import java.lang.reflect.Type; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.*; -import static org.mockito.Mockito.times; - -public class UserCacheUpdaterServiceTest { - - private RedisConnect redisConnectMock; - private CassandraConnect cassandraConnectMock; - private IncomingMessageEnvelope envelopeMock; - private TaskContext contextMock; - private JobMetrics jobMetricsMock; - private TaskCoordinator taskCoordinator; - private MessageCollector messageCollector; - private Jedis jedisMock = new MockJedis("test"); - private UserCacheUpdaterService userCacheUpdaterService; - private BaseCacheUpdaterService basecacheUpdater; - private UserCacheUpdaterSink userCacheUpdaterSinkMock; - private UserCacheUpdaterConfig userCacheUpdaterConfig; - private MetricsRegistry metricsRegistry; - private Counter counter; - private Config configMock; - private Integer userStoreId = 4; - private UserCacheUpdaterTask userCacheUpdaterTask; - - @Before - public void setUp() { - redisConnectMock = mock(RedisConnect.class); - userCacheUpdaterSinkMock = mock(UserCacheUpdaterSink.class); - configMock = mock(Config.class); - cassandraConnectMock = mock(CassandraConnect.class); - contextMock = mock(TaskContext.class); - jobMetricsMock = mock(JobMetrics.class); - metricsRegistry = Mockito.mock(MetricsRegistry.class); - taskCoordinator = mock(TaskCoordinator.class); - messageCollector = mock(MessageCollector.class); - counter = mock(Counter.class); - - stub(redisConnectMock.getConnection(userStoreId)).toReturn(jedisMock); - stub(redisConnectMock.getConnection()).toReturn(jedisMock); - envelopeMock = mock(IncomingMessageEnvelope.class); - stub(configMock.getInt("redis.database.userStore.id", userStoreId)).toReturn(userStoreId); - stub(configMock.getInt("location.db.redis.key.expiry.seconds", 86400)).toReturn(86400); - stub(configMock.get("user.signin.type.default", "Anonymous")).toReturn("Anonymous"); - stub(configMock.getList("user.selfsignedin.typeList", Arrays.asList("google", "self"))).toReturn(Arrays.asList("google", "self")); - stub(configMock.getList("user.validated.typeList", Arrays.asList("sso"))).toReturn(Arrays.asList("sso")); - stub(configMock.get("user.login.type.default", "NA")).toReturn("NA"); - stub(configMock.get("user.self-siginin.key", "Self-Signed-In")).toReturn("Self-Signed-In"); - stub(configMock.get("user.valid.key", "Validated")).toReturn("Validated"); - stub(configMock.get("middleware.cassandra.host", "127.0.0.1")).toReturn(""); - stub(configMock.get("middleware.cassandra.port", "9042")).toReturn("9042"); - stub(configMock.get("middleware.cassandra.keyspace", "sunbird")).toReturn("sunbird"); - stub(configMock.get("middleware.cassandra.user_table","user")).toReturn("user"); - stub(configMock.get("middleware.cassandra.location_table","location")).toReturn("location"); - - stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter); - stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry); - - userCacheUpdaterConfig=new UserCacheUpdaterConfig(configMock); - userCacheUpdaterService = new UserCacheUpdaterService(userCacheUpdaterConfig, redisConnectMock, cassandraConnectMock, jobMetricsMock); - jedisMock.flushAll(); - } - - @Test - public void shouldNotUpdateCacheForInvalidEvent() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING); - userCacheUpdaterTask = new UserCacheUpdaterTask(configMock, contextMock, cassandraConnectMock, redisConnectMock); - userCacheUpdaterTask.process(envelopeMock,messageCollector,taskCoordinator); - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - - verify(userCacheUpdaterSinkMock, times(1)).error(); - } - - @Test - public void shouldAddLocationDetailsToCache() { - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_METADATA_UPDATED); - Gson gson = new Gson(); - String userId = "52226956-61d8-4c1b-b115-c660111866d3"; - jedisMock.select(userStoreId); - jedisMock.set(userId, "{\"channel\":\"dikshacustodian\",\"phoneverified\":false,\"usersignintype\":\"Self-Signed-In\",\"userlogintype\":\"NA\", \"locationids\":[\'8952478975387\']}"); - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - stub(envelopeMock.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1))); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - - String cachedData = jedisMock.get(userId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals(5, parsedData.size()); - assertEquals("dikshacustodian",parsedData.get("channel")); - assertEquals(false, parsedData.get("phoneverified")); - assertEquals("Self-Signed-In", parsedData.get("usersignintype")); - assertEquals( "NA", parsedData.get("userlogintype")); - assertEquals(Arrays.asList("8952478975387"), parsedData.get("locationids")); - } - - @Test - public void shouldNotUpdateCacheForAnonymousUsers() throws Exception { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT3); - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - } - - @Test - public void shouldMarkEventSkippedForNullUserId() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_NULL_USERID); - userCacheUpdaterTask = new UserCacheUpdaterTask(configMock, contextMock, cassandraConnectMock,redisConnectMock); - userCacheUpdaterTask.process(envelopeMock,messageCollector,taskCoordinator); - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - verify(userCacheUpdaterSinkMock, times(1)).markSkipped(); - } - - @Test public void shouldupdateLocationDetails() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT); - userCacheUpdaterTask = new UserCacheUpdaterTask(configMock, contextMock, cassandraConnectMock,redisConnectMock); - userCacheUpdaterTask.process(envelopeMock,messageCollector,taskCoordinator); - - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - } - - @Test - public void shouldUpdateUserSignInDetailsinCacheFORAUDIT() { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_SIGINTYPE); - Gson gson = new Gson(); - String userId = "89490534-126f-4f0b-82ac-3ff3e49f3468"; - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1))); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - Map userData = userCacheUpdaterService.updateUserCache(source.getEvent(), userId, userCacheUpdaterSinkMock); - - jedisMock.set(userId, gson.toJson(userData)); - - String cachedData = jedisMock.get(userId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals("Self-Signed-In", parsedData.get("usersignintype")); - } - - @Test - public void shouldUpdateUserDetailsinCacheFORAUDIT() { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_SIGN_IN); - Gson gson = new Gson(); - String userId = "89490534-126f-4f0b-82ac-3ff3e49f3468"; - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1))); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - Map userData = userCacheUpdaterService.updateUserCache(source.getEvent(), userId, userCacheUpdaterSinkMock); - jedisMock.set(userId, gson.toJson(userData)); - - String cachedData = jedisMock.get(userId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals("Validated", parsedData.get("usersignintype")); - } - - @Test - public void shouldUpdateUserLoginInTYpeDetailsinCacheFORAUDIT() { - - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_LOGININTYPE); - Gson gson = new Gson(); - String userId = "3b46b4c9-3a10-439a-a2cb-feb5435b3a0d"; - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - stub(envelopeMock.getSystemStreamPartition()) - .toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1))); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - Map userData = userCacheUpdaterService.updateUserCache(source.getEvent(), userId, userCacheUpdaterSinkMock); - jedisMock.set(userId, gson.toJson(userData)); - - String cachedData = jedisMock.get(userId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals("student", parsedData.get("userlogintype")); - } - - @Test - public void shouldNotUpdateCacheWithMetadataChangesAndLocationFORAUDIT() { - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_METADATA_UPDATED); - Gson gson = new Gson(); - String userId = "52226956-61d8-4c1b-b115-c660111866d3"; - jedisMock.set(userId, "{\"channel\":\"dikshacustodian\",\"phoneverified\":false}"); - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - stub(envelopeMock.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1))); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - - String cachedData = jedisMock.get(userId); - Map parsedData = new HashMap<>(); - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals(0, parsedData.size()); - verify(cassandraConnectMock, times(0)).find(anyString()); - } - - @Test - public void shouldUpdateCache() throws Exception { - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_SIGINTYPE); - userCacheUpdaterTask = new UserCacheUpdaterTask(configMock, contextMock, cassandraConnectMock,redisConnectMock); - userCacheUpdaterTask.process(envelopeMock,messageCollector,taskCoordinator); - - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_SIGIN_TYPE); - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - } - - @Test - public void shouldUpdateCacheWithMetadataChangesAndLocationFORAUDIT() { - stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_METADATA_UPDATED); - Gson gson = new Gson(); - String userId = "52226956-61d8-4c1b-b115-c660111866d3"; - jedisMock.select(userStoreId); - jedisMock.set(userId, "{\"channel\":\"dikshacustodian\",\"phoneverified\":false,\"usersignintype\":\"Self-Signed-In\",\"userlogintype\":\"NA\"}"); - UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock); - stub(envelopeMock.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1))); - userCacheUpdaterService.process(source, userCacheUpdaterSinkMock); - - String cachedData = jedisMock.get(userId); - Map parsedData = null; - if (cachedData != null) { - Type type = new TypeToken>() { - }.getType(); - parsedData = gson.fromJson(cachedData, type); - } - assertEquals(4, parsedData.size()); - assertEquals(parsedData.get("channel"), "dikshacustodian"); - assertEquals(parsedData.get("phoneverified"), false); - assertEquals(parsedData.get("usersignintype"), "Self-Signed-In"); - assertEquals(parsedData.get("userlogintype"), "NA"); - } -} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..178d2e1d9e --- /dev/null +++ b/docs/README.md @@ -0,0 +1,1244 @@ +# TOC +1. [Architecture Overview](#architecture-overview) +2. [Telemetry](#telemetry) +3. [Kafka](#kafka) +4. [Flink](#flink) +5. [Secor](#secor) +6. [Spark, Data-products](#spark-data-products) +7. [Druid](#druid) +8. [Monitoring](#monitoring) +9. [DevOps](#devops) + +# Architecture Overview + +## Sunbird Architecture +Refer to [Sunbird Tech-Architecture pdf](Sunbird%20-%20Tech%20Architecture%20.pdf) + +## iGoT data-pipeline architecture +Refer to [Sunbird Tech-Architecture pdf](Sunbird%20-%20Tech%20Architecture%20.pdf) + +## Why Sunbird? +Why are we using Sunbird for iGoT project? + +- Microservices architecture - enables decoupling and allows various teams to collaborate on different features of the platform simultaneously +- Scaling - each of the services can be horizontally scaled separately +- Open source +- Employs Cutting edge open-source technologies +- Proven and battle-tested + +## Motivation Behind data-pipeline architecture +Motivation behind data-pipeline architecture - what are we trying to achieve and how does this architecture enable it? + +- Need for real-time usage data is enabled by stream processing, enabling real-time data to be displayed on dashboards +- Need for exhaustive reporting and dashboards, while interacting with multiple data-sources is enabled by batch processing +- Architecture is designed to be decoupled with multiple microservices, each of which can be horizontally scaled separately + +## Intro to data-pipeline technologies +- Telemetry Service - iGOT service responsible for producing telemetry events to Kafka +- Apache Kafka - Transport for async inter-service communication +- Apache Flink - Distributed stream processing framework +- Apache Spark - Distributed batch processing framework +- Secor - library by Pinterest for efficient and robust Kafka topic backups +- Apache Druid - OLAP store for ad-hoc analytical queries +- RAIN - Tarento's in-house accelerator, provides functionality for querying multiple data-sources and data-visualizations +- Monitoring Technologies - Prometheus, Grafana, etc. +- DevOps Technologies - Jenkins, Ansible, Kubernetes, Helm + + +# Telemetry + +Telemetry enables automatic collection of data from real-world, real-time use. + +- Event driven architecture - Instead of having to query application data-sources for data, we have the services emmit data in real-time +- Front end telemetry events are emitted by frontend services when a user action is performed +- Back end telemetry events are emitted by backend services when a particular event occurs during backend processing +- Sunbird provides a Standalone JS Library as well as an HTML Interface Library to simplify telemetry generation +- Both of the above call Telemetry Service APIs which can also be called directly + +## Sunbird Telemetry Event Structure +```js + +{ + // About the event + "eid": , // Required. + "ets": , // Required. Epoch timestamp of event (time in milli-seconds. For ex: 1442816723) + "ver": , // Required. Version of the event data structure, currently "3.0" + "mid": , // Required. Unique message ID. Used for deduplication, replay and update indexes + + // Who did the event + "actor": { // Required. Actor of the event. + "id": , // Required. Can be blank. Id of the actor. For ex: uid incase of an user + "type": // Required. Can be blank. User, System etc. + }, + + // Context of the event + "context": { // Required. Context in which the event has occured. + "channel": , // Required. Channel which has produced the event + "pdata": { // Optional. Producer of the event + "id": , // Required. unique id assigned to that component + "pid": , // Optional. In case the component is distributed, then which instance of that component + "ver": // Optional. version number of the build + }, + "env": , // Required. Unique environment where the event has occured. + "sid": , // Optional. session id of the requestor stamped by portal + "did": , // Optional. uuid of the device, created during app installation + "cdata": [{ // Optional. correlation data + "type":"", // Required. Used to indicate action that is being correlated + "id": "" // Required. The correlation ID value + }], + "rollup": { // Optional. Context rollups example: Organization heirarchy("L1": "Parent org id", "l2": "sub org id") + "l1": "", + "l2": "", + "l3": "", + "l4": "" + } + }, + + // What is the target of the event + "object": { // Optional. Object which is the subject of the event. + "id": , // Required. Id of the object. For ex: content id incase of content + "type": , // Required. Type of the object. For ex: "Content", "Community", "User" etc. + "ver": , // Optional. version of the object + "rollup": { // Optional. Rollups to be computed of the object. Only 4 levels are allowed. example: Textbook heirarchy("L1": "Parent node id", "l2": "Child node id") + "l1": "", + "l2": "", + "l3": "", + "l4": "" + } + }, + + // What is the event data + "edata": {} // Required. + + // Tags + "tags": [""] // Optional. Encrypted dimension tags passed by respective channels +} + +``` + +## Common Telemetry Events + +Front-end events +* START - Start of a multi step activity. For ex: Session, App, Tools, Community etc would have start and end +* END - End of a work flow +* IMPRESSION - A visit to a specific page by an user +* INTERACT - User interaction on the page (such as search, click, preview, move, resize, configure) + +Back-end events +* AUDIT - When an object is changed. This includes lifecycle changes as well. +* LOG - Generic log event. API call, Service call, app update etc +* SEARCH - A search is triggered (content, item, asset etc.) +* SUMMARY - A summary event + +[Full telemetry spec](telemetry-v3.md) + +## Implementing new Telemetry events +- Based on the telemetry specification decide if the new event type fits in with the existing defined telemetry types +- If a new telemetry event type is needed, provisions for its processing would need to be made + - New specification must be defined - [example](../data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/cb_audit.json) + - New kafka topics would need to be created and data-pipeline components would need to be modified to accommodate the new event's processing +- Any schema changes required to validate the events must be made [here](../data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0) +- Create clear document on what each of the field's allowed values are, and what are the expected values are in different situations/portals +- Communicate with the concerned external team + + +# Kafka + +## What is a message queue? +A message queue is a form of asynchronous service-to-service communication used in serverless and microservices +architectures. Messages are stored on the queue until they are processed and deleted + +Examples:- RabbitMQ, AWS Simple Queue Service (SQS), Kafka + + +## Introduction to Apache Kafka + +https://kafka.apache.org/28/documentation.html + +Apache Kafka is a distributed data streaming platform that can publish, subscribe to, store, and process streams of +records in real time + +Can act as a - +- Message queue - at-least-once delivery, which means that each message is delivered at least once +- FIFO queue - exactly-once processing, which means that each message is delivered once and remains available until a consumer processes it and deletes it +- Pub/Sub messaging system - the publisher sends messages to a topic, and multiple subscribers can listen to the topic +- Real-time streaming platform +- Kafka can also be used as a database due to its durable storage capability + +### Architecture - + +![kafka architecture](images/apache-kafka-architecture3.png) + +#### Kafka Cluster +A Kafka cluster is a system that consists of different brokers, topics, and their respective partitions. Data is written +to the topic within the cluster and read by the cluster itself. + +#### Producers +A producer sends or writes data/messages to the topic within the cluster. In order to store a huge amount of data, +different producers within an application send data to the Kafka cluster. + +#### Consumers +A consumer is the one that reads or consumes messages from the Kafka cluster. There can be several consumers consuming +different types of data form the cluster. Each consumer knows from where it needs to consume the data. + +#### Brokers +A Kafka server is known as a broker. A broker is a bridge between producers and consumers + +#### Topics +Kafka topics are the categories used to organize messages. Each topic has a name that is unique across the entire Kafka cluster. +Messages are sent to and read from specific topics +- Partitions - divides a singe topic log into multiple ones, each of which can live on a separate node in the Kafka cluster +- Replication factor - How many replicas of each partition exist on the cluster +- Retention - The period till which kafka must store the messages in the topic (from time of production) +- Offsets - it is a position within a partition for the next message to be sent to a consumer + +#### ZooKeeper +A ZooKeeper is a service used to store information about the Kafka cluster and details of the consumer clients. +- It manages brokers by maintaining a list of them. Also, a ZooKeeper is responsible for choosing a leader for the partitions. +- If any changes like a broker die, new topics, etc., occurs, the ZooKeeper sends notifications to Apache Kafka. +- A user does not directly interact with the Zookeeper, but via brokers. No Kafka server can run without a zookeeper server. + +## Kafka in iGOT +We have 2 kafka clusters running on iGOT +- KP Kafka - for Knowledge Platform services - ingestion-cluster +- DP Kafka - for Data Pipeline services - processing-cluster + +### DP Kafka + +Ansible role - [setup-kafka](../ansible/roles/setup-kafka) + +[Default topic configurations](../ansible/roles/setup-kafka/defaults/main.yml) + +#### Important Topics in DP kafka + +- {env}.telemetry.ingest - raw batch events appear here +- {env}.telemetry.raw - raw events +- {env}.telemetry.unique - unique events +- {env}.telemetry.derived - summary events +- {env}.druid.events.telemetry - druid telemetry events +- {env}.druid.events.summary - druid summary events + +## Basic commands for Kafka + +```shell +# Start the ZooKeeper service +bin/zookeeper-server-start.sh config/zookeeper.properties + +# Start the Kafka broker service +bin/kafka-server-start.sh config/server.properties + +# to create a topic +/opt/kafka/bin/kafka-topics.sh --create --topic --replication-factor 1 --partitions 1 --bootstrap-server localhost:9092 + +# to create topic for older version of kafka +/opt/kafka/bin/kafka-topics.sh --create --topic --replication-factor 1 --partitions 1 --zookeeper localhost:2181 + +# to list topics +/opt/kafka/bin/kafka-topics.sh --list --zookeeper localhost:2181 + +# get topic details +./bin/kafka-topics.sh --zookeeper localhost:2181 --describe --topic prod.telemetry.raw + +# console consumer: start consumer add `--from-beginning` to get all messages from start +/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic + +# console producer +/opt/kafka/bin/kafka-console-producer.sh --topic --broker-list localhost:9092 + +# modify a topic +/opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic --config retention.ms=172800000 +/opt/kafka/bin/kafka-configs.sh --bootstrap-server localhost:9092 --entity-type topics --entity-name --alter --add-config retention.ms=172800000 + +# empty a topic +/opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic --config retention.ms=0 +/opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic --config retention.ms=172800000 + +# Delete a topic +bin/kafka-topics.sh --delete --zookeeper localhost:2181 --topic + +# get offsets for a topic +/opt/kafka/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list localhost:9092 --topic + +# get offsets for a consumer-group +/opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group --describe + +``` + +# Flink + +## Stream Processing + +Stream processing is a computing paradigm used to process and analyze continuous streams of data in real-time or near-real-time. +Instead of storing data and processing it later, stream processing systems handle data as it arrives, making them suitable +for applications that require immediate insights or actions based on incoming data. + +- Real-time Data Processing +- Parallel Processing +- Event-Driven Architecture +- Low Latency +- Scalability (horizontal) + +### Use Cases +Anything that requires real-time Analytics on data streams from various sources, such as sensors, social media feeds, click-stream data, logs, and financial transactions +- Fraud Detection: to analyze transactions, identify suspicious activities, and trigger alerts or actions in real-time +- Internet of Things (IoT) Applications: real-time analysis of sensor data for applications like predictive maintenance, smart cities, and environmental monitoring. +- Real-time Monitoring and Alerting: real-time monitoring of system performance, network traffic, application logs, and security events. +- Recommendation Systems: (Netflix, Youtube, Amazon) + +### Why Flink? + +- Exactly-Once Semantics + - Flink ensures each event is processed and results are computed reliably, without duplication or loss. + - This level of reliability is critical for many real-time applications, such as financial transactions and fraud detection. + +- Stateful Stream Processing + - Flink natively supports stateful stream processing, allowing applications to maintain and update state across event streams. + - This is essential for complex computations and analytics that require contextual information or aggregation over time windows. + +- Advanced Windowing and Time Handling + - Flink provides various types of time windows, such as tumbling, sliding, and session windows to aggregate data over time intervals + - Beneficial for time-based analysis and aggregation tasks. + +- High Performance and Scalability + - Support for parallel processing and distributed execution + - Can efficiently handle large volumes of data and scale horizontally to accommodate growing workloads by leveraging features like task parallelism and data partitioning. + + +## Flink Intro +Basic concepts, checkpointing, only once, stream tasks, process functions, windows, watermarking etc. + +- DataStream and DataSet + - DataStream represents a stream of data elements that are continuously processed in real-time + - DataSet represents a static, bounded collection of data elements that are processed in batch mode + +- Transformation + - Transformations are operations applied to DataStreams or DataSets to modify, filter, aggregate, or analyze data + - map, filter, reduce, join, and window, enabling developers to manipulate data streams and datasets efficiently. + +- Windowing + - partition data streams into finite, non-overlapping or overlapping time-based segments called windows + - Flink supports various types of windows, including tumbling windows, sliding windows, and session windows + +- Stateful Processing + - Allows applications to maintain and update state across event streams. + - State can be used to store intermediate results, maintain session information, or perform aggregations over time windows + - Flink provides built-in mechanisms for managing state, including keyed state and operator state. + +- Checkpointing and Fault Tolerance + - Checkpointing is a mechanism in Flink for ensuring fault tolerance and exactly-once processing semantics + - Flink periodically takes snapshots of the application state and metadata, which can be used to restore the application's state in case of failures + - Checkpointing is essential for maintaining data consistency and reliability in distributed stream processing applications + +- Event Time and Processing Time + - Event time refers to the time at which events occur in the real world + - Processing time refers to the time at which events are processed by the system + - Flink supports event time processing for accurate and consistent results, even in the presence of delayed or out-of-order events. + +- Watermarks + - Watermarks are markers emitted by Flink to indicate progress in event time + - They are used to track the completeness of event streams and to trigger window computations + - Watermarks help Flink handle out-of-order events and late data by providing a notion of progress in event time. + +- Execution Environment + - Local execution for development and testing + - Standalone clusters + - Apache YARN + - Apache Mesos + - Kubernetes + +## Hello World + +Installation and local run: + +https://nightlies.apache.org/flink/flink-docs-release-1.13/docs/try-flink/local_installation/ + + +## iGoT data-pipeline Flink jobs + +![iGoT Stream processing architecture](images/Stream%20processing%20architecture.drawio.svg) + +## Understanding iGoT Flink jobs +- [Intro to dp-core](flink-dp-3.6.md) +- Example code Walkthrough - How a Flink project is structured/coded in iGoT Data-pipeline +- Local Build - + - `cd data-pipeline-flink` + - `mvn3.6 clean install -DskipTests` + - `cd sunbird-dp-distribution` + - `mvn3.6 package -Pbuild-docker-image -Drelease-version=` +- to only build one of the modules + - `mvn3.6 clean install -pl -am -DskipTests` + +## DevOps +- Configurations - [values.j2](../kubernetes/helm_charts/datapipeline_jobs/values.j2) +- Build - [check file - kubernetes/pipelines/build/Jenkinsfile](../kubernetes/pipelines/build/Jenkinsfile) +- Deployment - [check file - kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile](../kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile) + +## Common Operations +```sh +# ssh to kubernetes server, for pre-prod +# ssh admin-192.168.3.215@10.194.181.118 + +# list pods flink-dev namespace +kubectl get po -n flink-dev # flink-prod for prod and BM + +# get logs for a pod +kubectl logs -n flink-dev # add -f to follow the logs + +# get configurations +kubectl get cm -n flink-dev + +# view/edit configurations +kubectl edit cm -n flink-dev + +# to restart a pod +kubectl delete po -n flink-dev + +# restart all pods in the namespace +kubectl delete po --all -n flink-dev + +# to remove a deployment completely +kubectl delete deploy -n flink-dev + +# to remove a job completely +kubectl delete jobs -n flink-dev +``` + +### Check lag, reset offset + +First obtain the consumer group id and kafka cluster ip from config, then + +```sh +# ssh to kafka cluster +# get offsets for a consumer-group +/opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group --describe + +# reset offset +/opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group --reset-offsets --to-earliest --topic -execute +/opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group --reset-offsets --to-latest --topic -execute + +/opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group --reset-offsets --to-datetime 2020-12-20T00:00:00.000 --topic --execute + +# other options +# --shift-by +# --to-current +# --to-latest +# --to-offset +# --to-datetime +# --by-duration +``` + +### Common issues +- One or more pods are not present + - maybe an intermittent error due to some service shutting down for a small period (redis, cassandra, kafka etc.) + - re-deploy the last image + - if pod is getting repeatedly restarted + - check logs to see what the error is + - kafka error - broker not available, wrong cluster is configured, topic missing, kafka VM disk space is full + - validation error - faulty message is present (or some service is still producing them), reset offset shift by + - snappy error - a snappy compressed message is in some topic + - checkpointing error - ceph could be down + + +# Secor + +Secor is an efficient kafka backup utility developed by Pinterest +- Secor process runs continuously and creates back-up files on local disk +- Secor uploads these local backups to blob storage after a configured amount of time has passed + + +```shell +# check logs on blob storage +# ssh to jenkins server (251 for prod) + +# all backups +s3cmd ls s3://igotlogs/secor-prod/ + +# unique backups +s3cmd ls s3://igot/secor-prod/unique/raw/ +``` + +```shell +# ssh to secor server +sudo su analytics + +# start and stop the secor process +/home/analytics/sbin/secor unique-telemetry-backup stop +/home/analytics/sbin/secor unique-telemetry-backup start + +# check logs +cd /mount/secor/logs +ls -ltrh +``` + +#### Repos: +https://github.com/sunbird-cb/secor/tree/secor-0.25 - secor-0.25 ====> to build and generate the artifacts. +https://github.com/sunbird-cb/sunbird-data-pipeline - release-3.7.0 ===> for deployment + +https://github.com/project-sunbird/sunbird-devops - release-3.7.0 =====> for jenkins job. + +### Secor Deploy +- Deploy - [Jenkinsfile.flink](../pipelines/deploy/secor/Jenkinsfile.flink) + + +# Spark, Data-products + +## Spark Intro +Apache Spark is an open-source distributed computing system that is designed for big data processing and analytics. +It provides an interface for programming entire clusters with implicit data parallelism and fault tolerance. + +- Performance: Spark's in-memory computation and optimization techniques often lead to faster processing times compared to other batch processing frameworks, especially for iterative algorithms or jobs requiring multiple passes over the data. +- Ease of Use: Spark provides high-level APIs in multiple languages like Java, Scala, Python, and R, which makes it easier for developers to write and maintain code. Additionally, Spark's rich ecosystem includes libraries for SQL, machine learning, graph processing, and streaming data, reducing the need for developers to reinvent the wheel. +- Unified Platform: Spark is a unified analytics engine, meaning it supports multiple workloads such as batch processing, interactive queries, streaming analytics, and machine learning within a single framework. This simplifies the architecture and reduces the need for managing multiple systems. +- Fault Tolerance: Spark's resilient distributed datasets (RDDs) provide built-in fault tolerance, allowing it to recover from failures gracefully without manual intervention. This is crucial for large-scale data processing where failures are common. +- Scalability: Spark is designed to scale from a single server to thousands of machines, making it suitable for processing large datasets. It can leverage distributed computing resources efficiently, providing linear scalability with the size of the cluster. +- Flexibility: Spark can run on various cluster managers like Apache Mesos, Hadoop YARN, and Kubernetes, giving users the flexibility to deploy it in different environments and integrate with existing infrastructure seamlessly. +- Community and Ecosystem: Spark has a vibrant open-source community and a rich ecosystem of tools and libraries built around it. This includes integration with popular data sources, connectors to databases, visualization tools, and third-party extensions, making it easier to solve complex data processing tasks. + +## Spark Architecture + +![Spark architecture](images/spark-architecture.png) + +- Driver: The driver is the main entry point of a Spark application. It runs the user's main function and coordinates the execution of the Spark jobs. The driver communicates with the cluster manager to acquire resources and schedule tasks. +- Cluster Manager: Spark supports multiple cluster managers such as Apache Mesos, Hadoop YARN, and Kubernetes. The cluster manager allocates resources (CPU, memory) across the worker nodes in the cluster and manages their lifecycles. It's responsible for launching and monitoring the executors. +- Executors: Executors are worker nodes in the Spark cluster responsible for executing tasks and storing data in memory or disk. Each executor runs multiple tasks concurrently and communicates with the driver program. Executors cache data in memory to improve performance and provide fault tolerance through replication. +- Worker Nodes: Worker nodes host one or more executors and are responsible for executing tasks and storing data. They are managed by the cluster manager and can be added or removed dynamically based on the workload. +- RDD (Resilient Distributed Dataset): RDD is the fundamental data abstraction in Spark. It represents an immutable, distributed collection of objects partitioned across the worker nodes. RDDs support fault tolerance through lineage and can be rebuilt if a partition is lost. +- Data Partitioning: RDDs are divided into partitions, which are the basic units of parallelism in Spark. Partitions are distributed across the worker nodes, and tasks are executed in parallel on these partitions. +- Scheduler: Spark's scheduler determines how tasks are scheduled and executed across the worker nodes. It divides the job into stages and tasks and optimizes the execution plan based on data locality and task dependencies. +- DAG (Directed Acyclic Graph) Scheduler: The DAG scheduler breaks down the job into stages of tasks and constructs a DAG representing the data flow dependencies. It optimizes the execution plan by pipelining transformations and minimizing data shuffling. +- Task Execution: Tasks are the smallest units of work in Spark and are executed on the worker nodes. Tasks process data partitions and perform transformations or actions defined by the user. Spark supports pipelining tasks to minimize data movement and maximize parallelism. +- Caching and Persistence: Spark allows data to be cached in memory or disk across multiple operations, improving performance by avoiding recomputation. Caching can be controlled using RDD.persist() or DataFrame.cache(). + +### Other basic concepts + +- Transformations: Transformations are operations that are applied to RDDs to create a new RDD. Examples include map, filter, reduceByKey, join, etc. Transformations are lazy, meaning they are not executed immediately but build up a computation plan (DAG) that is executed only when an action is called. +- Actions: Actions are operations that trigger the execution of the computation plan built by transformations and return results to the driver program or write data to an external storage system. Examples include collect, count, saveAsTextFile, etc. +- Spark Context: Spark Context is the main entry point for interacting with Spark. It represents the connection to a Spark cluster and is used to create RDDs, broadcast variables, and accumulators, as well as to set configuration parameters. +- SparkSession: SparkSession is the entry point for Spark SQL functionality. It allows you to interact with structured data using DataFrames and SQL queries. SparkSession encapsulates SparkContext, SQLContext, and HiveContext. +- DataFrames and Datasets: DataFrames and Datasets are higher-level abstractions introduced in Spark for working with structured data. They provide a more user-friendly API compared to RDDs and offer optimizations through the Catalyst query optimizer. +- Broadcast Variables and Accumulators: Broadcast variables allow efficient distribution of read-only data to all worker nodes, while accumulators are variables that can be added to by tasks and are typically used for aggregating results or collecting statistics. + +## Hello World +https://spark.apache.org/docs/2.4.0/ + +## iGoT data-pipeline Spark jobs + +![iGoT Batch processing architecture](images/batch-processing-architecture.drawio.svg) + +### Configuration +[Model config](../ansible/roles/data-products-deploy/templates/model-config.j2) + +WFS +```json +{ + "search": { + "type": "{{ dp_object_store_type }}", + "queries": [ + { + "bucket": "'$bucket'", + "prefix": "{{ dp_raw_telemetry_backup_location }}", + "endDate": "'$endDate'", + "delta": 0 + } + ] + }, + "model": "org.ekstep.analytics.model.WorkflowSummary", + "modelParams": { + "storageKeyConfig": "{{ dp_storage_key_config }}", + "storageSecretConfig": "{{ dp_storage_secret_config }}", + "apiVersion": "v2", + "parallelization": 200 + }, + "output": [ + { + "to": "kafka", + "params": { + "brokerList": "'$brokerList'", + "topic": "'$topic'", + "compression": "{{ dashboards_broker_compression }}" + } + } + ], + "parallelization": 200, + "appName": "Workflow Summarizer", + "deviceMapping": true +} +``` + +Dashboard Jobs +```json +{ + "search": { + "type": "none" + }, + "model": "org.ekstep.analytics.dashboard.report.acbp.UserACBPReportJob", + "modelParams": { + "debug": "false", + "validation": "false", + "redisHost": "{{ dashboards_redis_host }}", + "redisPort": "{{ dashboards_redis_port }}", + "redisDB": "{{ dashboards_redis_db }}", + "sparkCassandraConnectionHost": "{{ core_cassandra_host }}", + "sparkDruidRouterHost": "{{ druid_router_host }}", + "sparkElasticsearchConnectionHost": "{{ single_node_es_host }}", + "fracBackendHost": "{{ dashboards_frac_backend_host }}", + "cassandraUserKeyspace": "{{ user_table_keyspace }}", + "cassandraCourseKeyspace": "{{ course_keyspace }}", + "cassandraHierarchyStoreKeyspace": "{{ hierarchy_store_keyspace }}", + "cassandraUserTable": "{{ dashboards_cassandra_user_table }}", + "cassandraUserRolesTable": "{{ dashboards_cassandra_user_roles_table }}", + "cassandraOrgTable": "{{ dashboards_cassandra_org_table }}", + "cassandraUserEnrolmentsTable": "{{ dashboards_cassandra_user_enrolments_table }}", + "cassandraContentHierarchyTable": "{{ dashboards_cassandra_content_hierarchy_table }}", + "cassandraRatingSummaryTable": "{{ dashboards_cassandra_rating_summary_table }}", + "cassandraRatingsTable": "{{ dashboards_cassandra_ratings_table }}", + "cassandraUserAssessmentTable": "{{ dashboards_cassandra_user_assessment_table }}", + "cassandraOrgHierarchyTable": "{{ dashboards_cassandra_org_hierarchy_table }}", + "cassandraCourseBatchTable": "{{ dashboards_cassandra_course_batch_table }}", + "cassandraLearnerStatsTable": "{{ dashboards_cassandra_learner_stats_table }}", + "cassandraAcbpTable": "{{ dashboards_cassandra_acbp_table }}", + "cassandraKarmaPointsLookupTable": "{{ dashboards_cassandra_karma_points_lookup_table }}", + "cassandraKarmaPointsTable": "{{ dashboards_cassandra_karma_points_table }}", + "cassandraHallOfFameTable": "{{ dashboards_cassandra_mdo_karma_points_table }}", + "appPostgresHost": "{{ app_postgres_host }}", + "appPostgresSchema": "sunbird", + "appPostgresUsername": "sunbird", + "appPostgresCredential": "sunbird", + "appOrgHierarchyTable": "org_hierarchy_v4", + "dwPostgresHost": "{{ dw_postgres_host }}", + "dwPostgresSchema": "warehouse", + "dwPostgresUsername": "postgres", + "dwPostgresCredential": "{{ dw_postgres_credential }}", + "dwUserTable": "user_detail", + "dwCourseTable": "content", + "dwEnrollmentsTable": "user_enrolments", + "dwOrgTable": "org_hierarchy", + "dwAssessmentTable": "assessment_detail", + "dwBPEnrollmentsTable": "bp_enrolments", + "key": "{{ dp_storage_key_config }}", + "secret": "{{ dp_storage_secret_config }}", + "store": "{{ report_storage_type }}", + "container": "{{ s3_storage_container }}", + "mdoIDs": "'$reportMDOIDs'", + "cutoffTime": "60.0", + "userReportPath": "{{ user_report_path }}", + "userEnrolmentReportPath": "{{ user_enrolment_report_path }}", + "courseReportPath": "{{ course_report_path }}", + "cbaReportPath": "{{ cba_report_path }}", + "standaloneAssessmentReportPath": "{{ standalone_assessment_report_path }}", + "taggedUsersPath": "{{ tagged_users_path }}", + "blendedReportPath": "{{ blended_report_path }}", + "orgHierarchyReportPath": "{{ org_hierarchy_report_path }}", + "commsConsoleReportPath": "{{ comms_console_report_path }}", + "acbpReportPath": "{{ acbp_report_path }}", + "acbpMdoEnrolmentReportPath": "{{ acbp_mdo_enrolment_report_path }}", + "acbpMdoSummaryReportPath": "{{ acbp_mdo_summary_report_path }}", + "commsConsolePrarambhEmailSuffix": "{{ comms_console_prarambh_email_suffix }}", + "commsConsoleNumDaysToConsider": "{{ comms_console_num_days_to_consider }}", + "commsConsoleNumTopLearnersToConsider": "{{ comms_console_num_top_learners_to_consider }}", + "commsConsolePrarambhTags": "{{ comms_console_prarambh_tags }}", + "commsConsolePrarambhCbpIds": "{{ comms_console_prarambh_cbp_ids }}", + "commsConsolePrarambhNCount": "{{ comms_console_prarambh_n_count }}", + "sideOutput": { + "brokerList": "'$brokerList'", + "compression": "{{ dashboards_broker_compression }}", + "topics": { + "roleUserCount": "{{ dashboards_role_count_topic }}", + "orgRoleUserCount": "{{ dashboards_org_role_count_topic }}", + "allCourses": "{{ dashboards_courses_topic }}", + "userCourseProgramProgress": "{{ dashboards_user_course_program_progress_topic }}", + "fracCompetency": "{{ dashboards_frac_competency_topic }}", + "courseCompetency": "{{ dashboards_course_competency_topic }}", + "expectedCompetency": "{{ dashboards_expected_competency_topic }}", + "declaredCompetency": "{{ dashboards_declared_competency_topic }}", + "competencyGap": "{{ dashboards_competency_gap_topic }}", + "userOrg": "{{ dashboards_user_org_topic }}", + "org": "{{ dashboards_org_topic }}", + "userAssessment": "{{ dashboards_user_assessment_topic }}", + "assessment": "{{ dashboards_assessment_topic }}", + "acbpEnrolment": "{{ dashboards_acbp_enrolment_topic }}" + } + } + }, + "output": [], + "parallelization": 16, + "appName": "ACBP Report Job", + "deviceMapping": false +} +``` + +### How are the jobs being run? +Data products jobs are scheduled in crontab, (Walk-through with WFS as an example) + +[run job script](../ansible/roles/data-products-deploy/templates/run-job.j2) + +### Dashboard Jobs + +https://github.com/sunbird-cb/sunbird-core-dataproducts/tree/cbrelease-4.8.11/batch-models/src/main/scala/org/ekstep/analytics/dashboard + +- DashboardUtil +- DataUtil +- TestUtil + +## DevOps +Analytics Core +- build - https://github.com/sunbird-cb/sunbird-analytics-core/blob/release-4.8.0-nic/Jenkinsfile +- deploy - [Jenkinsfile](../pipelines/deploy/data-products/Jenkinsfile) + +Data Products +- build - https://github.com/sunbird-cb/sunbird-core-dataproducts/blob/cbrelease-4.8.11/Jenkinsfile +- deploy - [Jenkinsfile](../pipelines/deploy/data-products/Jenkinsfile) + +## Debugging + +```shell + +# ssh to spark server +sudo su analytics + +# check crontab +crontab -l + +# running spark shell +cd /mount/data/analytics/spark-2.4.4-bin-hadoop2.7/ +bin/spark-shell --master local[*] --packages redis.clients:jedis:4.2.3,com.datastax.spark:spark-cassandra-connector_2.11:2.5.0,org.mongodb.spark:mongo-spark-connector_2.11:2.4.2,org.apache.spark:spark-avro_2.11:2.4.0 --conf spark.cassandra.connection.host=192.168.3.246 --conf spark.sql.caseSensitive=true --jars /mount/data/analytics/models-2.0/analytics-framework-2.0.jar,/mount/data/analytics/models-2.0/scruid_2.11-2.4.0.jar +# :load /home/analytics/spark-scripts/DashboardUtil.scala +# :load /home/analytics/spark-scripts/DataUtil.scala +# :load /home/analytics/spark-scripts/TestUtil.scala +# :load /home/analytics/spark-scripts/DataExhaustModel.scala +# TestUtil.main(DataExhaustModel) + +# convert avro files to json +cd /mount/data/analytics/cache +wget https://repo1.maven.org/maven2/org/apache/avro/avro-tools/1.9.1/avro-tools-1.9.1.jar # may already be present +java -jar avro-tools-1.9.1.jar tojson some.avro > some.json +# we have multiple partitions of these files so we extract in a loop +ls orgHierarchy/*.avro | while read LINE; do java -jar avro-tools-1.9.1.jar tojson "$LINE" >> orgHierarchy.json; done + +# check logs +cd /mount/data/analytics/scripts/logs +tail -f joblog.log + +# detailed spark logs +cd /mount/data/analytics/logs/data-products +ls -ltrh + +``` + +# Druid + +Apache Druid is an open-source distributed data store designed for real-time analytics on large datasets. It provides low-latency queries, scalable data ingestion, and flexible data exploration capabilities, making it suitable for a wide range of use cases in modern data-driven applications. Here are some key use cases for Apache Druid: + +- Real-Time Analytics: Apache Druid is well-suited for real-time analytics use cases where low-latency queries on large volumes of data are required. It can ingest and query streaming data with millisecond-level latency, enabling organizations to make timely and informed decisions based on up-to-date data. +- Event Monitoring and Dashboards: Druid can be used to build interactive dashboards and monitoring systems for tracking events, metrics, and key performance indicators (KPIs) in real-time. It allows users to slice and dice data dynamically, drill down into details, and visualize insights through various chart types. +- Time Series Data Analysis: Druid excels at analyzing time-series data, such as logs, metrics, sensor data, and IoT (Internet of Things) data. It supports efficient storage and querying of time-stamped data, enabling users to perform time-based aggregations, filtering, and transformations at scale. +- Operational Intelligence: Druid is commonly used for operational intelligence use cases, such as monitoring system performance, detecting anomalies, and troubleshooting issues in real-time. It can ingest data from various sources, including application logs, infrastructure metrics, and system events, and provide actionable insights for operational teams. +- Ad Hoc Data Exploration: Druid enables ad hoc data exploration and analysis by providing fast query response times and interactive querying capabilities. Users can explore large datasets, perform aggregations, apply filters, and visualize results in real-time using SQL-like queries or intuitive GUI tools. +- Personalization and Recommendation Systems: Druid can power personalization and recommendation systems by ingesting and querying user behavior data, such as clicks, views, and purchases. It allows organizations to analyze user interactions, identify patterns, and deliver personalized recommendations or content in real-time. +- Fraud Detection and Security Analytics: Druid can be used for detecting fraud, anomalies, and security threats by analyzing large volumes of transactional data, user behavior logs, and network traffic data in real-time. It enables organizations to identify suspicious patterns, detect deviations from normal behavior, and take proactive measures to mitigate risks. +- Customer Analytics and Marketing Attribution: Druid can help organizations analyze customer behavior, track marketing campaigns, and measure the effectiveness of advertising channels. It enables marketers to gain insights into customer interactions, segment audiences, and attribute conversions across various touchpoints. +- Business Intelligence and Reporting: Druid can serve as a backend data store for business intelligence (BI) and reporting tools, providing fast query performance and high concurrency for interactive dashboards and ad hoc queries. It allows users to analyze historical and real-time data, generate reports, and derive actionable insights to drive business decisions. +- Machine Learning and AI Applications: Druid can be integrated with machine learning (ML) and artificial intelligence (AI) platforms to build predictive models, perform advanced analytics, and automate decision-making processes. It provides the necessary data infrastructure for ingesting, storing, and querying large datasets used in ML and AI workflows. + +## Druid Concepts +- Segments: In Druid, data is stored in segments, which are immutable, append-only files that contain pre-aggregated data. Segments are created by ingesting raw data and processing it into a summarized format that can be efficiently queried. Each segment typically represents a time interval (e.g., one hour, one day) and contains aggregated data for that period. +- Data Source: A data source in Druid represents a collection of data ingested into the system. It corresponds to a specific dataset or table from which data can be queried. Each data source is associated with one or more segments that contain the actual data. +- Dimensions: Dimensions in Druid are the attributes or categorical variables on which data can be grouped, filtered, and aggregated. Examples of dimensions include user ID, product category, country, etc. Dimensions provide context and granularity to the data and are used extensively in queries to analyze and aggregate data. +- Metrics: Metrics in Druid are the numerical values that are aggregated and analyzed during queries. Examples of metrics include counts, sums, averages, minimums, maximums, etc. Metrics provide quantitative measurements of the data and are often aggregated based on dimensions to produce meaningful insights. +- Granularity: Granularity in Druid refers to the level of detail or precision at which data is aggregated and stored in segments. Druid supports various granularities, such as seconds, minutes, hours, days, and months, allowing users to choose the appropriate level of aggregation based on their data and query requirements. +- Rollup: Rollup is a data optimization technique used in Druid to reduce storage and improve query performance. It involves pre-aggregating data at ingestion time based on specified dimensions and metrics. Rollup reduces the number of unique values for dimensions and eliminates low-cardinality data, resulting in smaller segments and faster queries. +- Deep Storage: Persistent storage system where data segments are stored for long-term retention and durability. Deep storage serves as the primary storage layer for historical nodes, allowing them to store and serve data segments efficiently for query execution. + +## Druid Services +- Zookeeper: Distributed coordination service, providing cluster management, configuration synchronization, and leader election capabilities. +- Coordinator: The Coordinator service is responsible for managing the cluster's metadata and coordinating segment assignments across nodes. It maintains information about available segments, segment versions, and segment replicas, ensuring data availability, consistency, and load balancing. The Coordinator also handles data retention policies and segment lifecycle management. +- Overlord: The Overlord service is responsible for managing ingestion tasks and coordinating the ingestion process. It receives ingestion requests, schedules tasks, and monitors the progress of data ingestion jobs. Overlord ensures that data is ingested into the cluster efficiently and reliably, coordinating real-time and batch ingestion workflows. +- Historical: The Historical service is responsible for storing and serving data segments to query nodes. It hosts one or more segments and handles read-only queries by fetching and serving data from local disk storage. Historical nodes are horizontally scalable, allowing them to scale out to accommodate growing data volumes and query loads. +- Broker: The Broker service acts as a query router and coordinator for query execution. It receives query requests from clients, determines the optimal data nodes to execute the query, and forwards the requests to the appropriate nodes. Brokers aggregate results from multiple nodes and return them to the client, providing a single entry point for querying the cluster. +- MiddleManager: The MiddleManager service is responsible for managing the ingestion process on individual nodes. It runs ingestion tasks, processes data streams, and converts them into segments that can be queried. MiddleManagers distribute the workload of data ingestion across nodes, ensuring efficient resource utilization and fault tolerance. +- Router: Service responsible for routing query requests from clients to the appropriate nodes in the cluster for query execution. The router acts as an intermediary between clients and the various nodes (historical, broker, and coordinator) in the Druid cluster, ensuring that queries are executed efficiently and results are returned to the client. +- Indexing Service: The Indexing Service is responsible for creating and managing inverted indexes for Druid segments. It runs indexing tasks to build indexes from raw data, optimize query performance, and support fast filtering and aggregation. Indexing tasks are executed asynchronously and can be scheduled to run on demand or on a recurring basis. +- Metadata Storage: Metadata storage is crucial for maintaining information about the cluster, including data sources, segments, configurations, and runtime state +- Druid Console: Web-based interface for managing and monitoring the Druid cluster. It provides a graphical user interface for configuring data sources, managing segments, monitoring ingestion tasks, and viewing cluster metrics and status. The Coordinator Console enables administrators to monitor the health and performance of the cluster and perform administrative tasks easily. + +## Druid Ingestion + +https://druid.apache.org/docs/latest/ingestion/ingestion-spec/ + +[Sample](../ansible/roles/druid-ingestion/templates/raw_dashboards_user_org) + +## Druid Querying +SQL API + +https://druid.apache.org/docs/latest/api-reference/sql-api/ + +https://druid.apache.org/docs/latest/querying/sql + +Native Queries + +https://druid.apache.org/docs/latest/querying/ + +Datasketches + +https://druid.apache.org/docs/latest/development/extensions-core/datasketches-extension/ + +## DevOps +- Provisioning [Jenkinsfile](../pipelines/provision/druid/Jenkinsfile) +- Configurations [config](../ansible/roles/analytics-druid/templates) +- Druid Ingestion [druid-ingestion.yml](../ansible/druid-ingestion.yml) + +## Druid Console demo + +- Submitting ingestion spec +- Suspending supervisors +- Hard Reset +- Querying +- Setting retention rules +- Deleting segments + + +### Change log level for druid services + +to change log level for any of the druid services edit their respective `log4j2.xml` file. +for example to set broker log level to warn, edit it's `log4j2.xml` - + +Note: setting loglevel above `WARN` (e.g. `INFO`) will make logs very busy, and log files would inflate to +MBs in a couple of minutes, set loglevel back to `ERROR` as soon as done with debugging + +```xml + + + + + + "%d{ISO8601} %p [%t] %c - %m%n" + + + + + + + + + + + + + +``` + +#### for s3 compatible deep storage + +to use s3 as deep storage make sure `common.runtime.properties` contains following config + +``` +druid.extensions.loadList=["druid-s3-extensions"] +druid.extensions.directory=/data/druid/extensions + +druid.storage.type=s3 +druid.storage.bucket= +druid.storage.baseKey=druid/segments + +druid.s3.accessKey= +druid.s3.secretKey= + +# set protocol and endpoint together +druid.s3.endpoint.url=:// + +# or separately as +# druid.s3.endpoint.url= +# druid.s3.endpoint.protocol= +``` + +for non-aws s3-like stores (like ceph), we might have to add additional config +``` +# enable access of bucket from any region +druid.s3.forceGlobalBucketAccessEnabled=true + +# to enable path like access +# if true, url=:/// +# if false, url=://. +druid.s3.enablePathStyleAccess=true +``` + +to allow Druid to publish task logs to s3 add following config +``` +druid.indexer.logs.type=s3 +druid.indexer.logs.s3Bucket= +# path to logs within the bucker +druid.indexer.logs.s3Prefix=druid/stage/indexing_logs +``` + +additional config for s3 deep storage (optional) +``` +# uncomment to enable server side encryption for s3 +# druid.storage.sse.type=s3 + +# uncomment to enable v4 signing of requests +# druid.s3.endpoint.signingRegion= + +# uncomment to disable chunk encoding +# druid.s3.disableChunkedEncoding=true +``` + +#### S3 bucket policy + +Druid should have permissions to read and write from `druid` dir of the bucket +For S3, we would require `GetObject`, `PutObject`, `GetObjectAcl`, `PutObjectAcl` permissions + +Example policy might look like- + +`policy.json` +```json +{ + "Statement": [ + { + "Action": [ + "s3:ListAllMyBuckets" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::*" + }, + { + "Action": [ + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:GetBucketLocation", + "s3:AbortMultipartUpload", + "s3:GetObjectAcl", + "s3:GetObjectVersion", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:GetObject", + "s3:PutObjectAcl", + "s3:PutObject", + "s3:GetObjectVersionAcl" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::/druid", + "arn:aws:s3:::/druid/*" + ] + } + ] +} +``` + +to update bucket policy using `s3cmd`, first install s3cmd and configure using `s3cmd --configure`, then run + +```sh +s3cmd setpolicy policy.json s3:// +``` + +Depending on the library being used to upload files to storage, to make files accessible publicly we might need to set acl to public using `s3cmd` + +```sh +s3cmd setacl s3:////* --acl-public +``` + +#### for azure deep storage + +to use azure as deep storage make sure `common.runtime.properties` contains following config + +``` +druid.extensions.loadList=["druid-azure-extensions"] +druid.extensions.directory=/data/druid/extensions + +druid.storage.type=azure +druid.azure.account= +druid.azure.key= +druid.azure.container= +``` + +to allow Druid to publish task logs to azure add following config +``` +druid.indexer.logs.type=azure +druid.indexer.logs.container= +druid.indexer.logs.prefix= +``` + +#### misc config +``` +# uncomment to disable acl for deep storage +# druid.storage.disableAcl=true + +# uncomment to disable acl for only logs +# druid.indexer.logs.disableAcl=true +``` + +### Druid graceful restart / rolling update + +For configurations to take effect Druid services for which config has changed must be restarted. +All Druid services except for `middlemanager` can be restarted safely through `systemctl` + +```sh +# ssh to druid +systemctl restart druid_broker.service +systemctl restart druid_coordinator.service +systemctl restart druid_historical.service +systemctl restart druid_overlord.service +``` + +to gracefully restart `middlemanager` first we have to suspend all running supervisors. this publishes +segments which have not been published yet + +```sh +# ssh to druid +# get running supervisor names +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor -i + +# do this for all running supervisors +# suspend supervisor (stop running tasks and publish segments) +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//suspend + +# restart middlemanager service +systemctl restart druid_middlemanager.service + +# resume suspended supervisors +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//resume +``` + +### Druid API + +ports - to find out what ports each of the services are running check `runtime.properties` file in `/data/druid/conf/druid//` + +default ports - +```sh +# coordinator - 8081 +# broker - 8082 +# historical - 8083 +# overlord - 8090 +# middlemanager - 8091 +``` + +Check status, get data sources +```sh +# check status of overlord service +curl -X GET http://localhost:8090/status + +# show data sources +curl -X GET http://localhost:8081/druid/coordinator/v1/datasources -i +``` + +Manage Ingestion +```sh +# get running supervisor names +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor -i + +# inspect particular supervisor ingestion config +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor/ -i + +# inspect particular supervisor status +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor//status -i + +# inspect particular supervisor task stats +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor//stats -i + +# inspect tasks +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor/tasks -i + +# inspect pending tasks +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor/pendingTasks -i + +# inspect running tasks +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor/runningTasks -i + + +# add new supervisor +curl -X POST -H 'Content-Type: application/json' -d @spec.json http://localhost:8090/druid/indexer/v1/supervisor + +# stop and delete supervisor +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//terminate -i + +# suspend supervisor (stop running tasks and publish segments) +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//suspend + +# resume supervisor +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//resume + +``` + +# Monitoring + +## Why? +- Data Quality Assurance: Data pipelines are responsible for moving data from various sources to destinations, transforming it along the way. Monitoring ensures that data quality is maintained throughout this process. By tracking metrics such as data completeness, accuracy, consistency, and timeliness, organizations can ensure that the data being processed is reliable and fit for use in analytics, decision-making, and other applications. +- Performance Optimization: Monitoring helps identify bottlenecks, inefficiencies, and other performance issues within data pipelines. By tracking metrics such as processing time, resource utilization, and throughput, organizations can optimize pipeline performance to ensure timely data delivery and efficient resource utilization. +- Fault Detection and Troubleshooting: Data pipelines are complex systems that can encounter various issues such as software bugs, hardware failures, network issues, data schema changes, and data source changes. Monitoring helps detect these issues in real-time or near real-time, allowing for prompt troubleshooting and resolution to minimize downtime and data loss. +- Compliance and Governance: Many industries are subject to regulatory requirements regarding data handling, storage, and processing (e.g., GDPR, HIPAA). Monitoring helps ensure compliance with these regulations by tracking data access, usage, and security-related metrics, thus enabling organizations to demonstrate adherence to regulatory standards and mitigate risks associated with non-compliance. +- Cost Optimization: Data processing and storage can be costly, especially at scale. Monitoring helps organizations track resource utilization and costs associated with data pipelines, allowing them to identify opportunities for optimization, such as optimizing resource allocation, reducing unnecessary data processing, or choosing more cost-effective data storage options. +- Business Insights and Decision-Making: Monitoring data pipelines provides valuable insights into data usage patterns, trends, and anomalies, which can inform strategic decision-making, business process improvements, and product enhancements. By analyzing monitoring data, organizations can gain a better understanding of their data assets and how they are being utilized, enabling them to make data-driven decisions and derive actionable insights. + +## Flink +- Instrumentation: Kubernetes and the applications running on it need to be instrumented to expose metrics in a format that Prometheus can scrape. This typically involves adding libraries or exporters to the codebase of your applications or deploying Prometheus-compatible exporters for Kubernetes components (e.g., kube-state-metrics for cluster state, node-exporter for node metrics, etc.). +- Deployment of Prometheus: Prometheus needs to be deployed within the Kubernetes cluster. This can be done using Helm charts, YAML manifests, or Kubernetes operators. Prometheus should be configured with appropriate scrape configurations to collect metrics from the instrumented applications and Kubernetes components. +- Service Discovery: Prometheus needs to be configured to discover the endpoints of the applications and Kubernetes components it needs to monitor. Kubernetes provides service discovery mechanisms that Prometheus can leverage, such as DNS-based service discovery or Kubernetes service discovery. +- Alerting Rules: Prometheus allows you to define alerting rules based on the collected metrics. You can define rules to trigger alerts when certain conditions are met (e.g., CPU usage exceeds a threshold, number of error responses exceeds a threshold, etc.). These alerting rules are written in Prometheus Query Language (PromQL). +- Alertmanager Configuration: Alertmanager is a component that handles alerts sent by Prometheus and manages the routing, grouping, and notification of alerts. You need to configure Alertmanager to define notification channels (e.g., email, Slack, PagerDuty, etc.) and routing rules for different types of alerts. +- Grafana Integration: Grafana is a popular visualization tool that can be integrated with Prometheus to create dashboards and visualize metrics. You can deploy Grafana within the Kubernetes cluster and configure it to connect to Prometheus as a data source. Grafana allows you to create custom dashboards to visualize the collected metrics and monitor the health and performance of your Kubernetes cluster and applications. + +### deploying Monitoring +[Monitoring Jenkinsfile](../kubernetes/pipelines/monitoring/Jenkinsfile) + +### Adding monitoring config for new flink jobs +[dp_alertrules.yaml](../kubernetes/ansible/roles/sunbird-monitoring/templates/dp_alertrules.yaml) + +[dp_prometheus-adapter.yaml](../kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-adapter.yaml) + +[flinkjobs.yaml](../kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs.yaml) + +## Secor +Secor monitoring is done using cron jobs + +[How monitoring script is copied over](../ansible/roles/secor-deploy) + +[How monitoring cron jobs are scheduled](../ansible/roles/secor-telemetry-backup-deploy) + +## Spark +Spark monitoring setup - not enabled + +## Druid +Druid monitoring setup - through Graphite, does not seem to be configured in Graphana + +## Grafana +Data-pipeline Grafana Dashboards in production + +[DPLag Dashboard](https://igotkarmayogi.gov.in/grafana/d/vfdQRdtMDp/dplag-dashboard?orgId=1&refresh=5s) +- flink lag +- secor lag + +[Data Pipeline Metrics Flink](https://igotkarmayogi.gov.in/grafana/d/Ynghyskz/data-pipeline-metrics-flink?orgId=1&refresh=30s) + +[Lag Dashboard](https://igotkarmayogi.gov.in/grafana/d/vfdQRdtMh/lag-dashboard?orgId=1&refresh=5s) + +[Server Metrics](https://igotkarmayogi.gov.in/grafana/d/Ne61eqpSz/node-exporter-server-metrics-v1?orgId=1&var-node=192.168.3.119:9100&from=now-2d&to=now) + +# DevOps + +## Devops intro + +Environments +- Pre-prod +- QA +- BM +- Prod + +Default Configuration +- Common defaults [all.yml](../ansible/inventory/env/group_vars/all.yml) +- Role specific defaults [example](../ansible/roles/analytics-druid/defaults/main.yml) + +### Private repo overrides + +[inventory](../ansible/inventory) + +`common.yml` file +- configuration defined in yml format +- need to override the entire dictionary + +`hosts` file: +```shell +[druid-postgres] +10.0.0.0 + +[raw-zookeeper] +10.0.0.1 +10.0.0.2 +``` + +```yaml +druid_zookeeper_host: "{{ groups[cluster+'-zookeeper']|join(':2181,')}}:2181" +druid_postgres_host: "{{ groups['druid-postgres'][0] }}" +``` + +## Jenkins +Jenkins is an open-source automation server widely used for continuous integration (CI) and continuous delivery (CD) pipelines + +### Important Jenkins Jobs for data-pipeline + +#### Kafka +- deployment - KafkaSetup + +#### Flink +- build +- artifact upload +- deploy + +### Secor +- build +- artifact upload +- deploy + +#### Spark +- spark provisioning +- analytics-core + - build + - artifact upload + - deploy +- core-dataproducts + - build + - artifact upload + - deploy + +#### Druid +- Provisioning +- druid-ingestion deploy + +## Ansible +Ansible is an open-source automation tool used for configuration management, application deployment, infrastructure provisioning, and orchestration. It simplifies the management and automation of IT infrastructure by allowing users to define tasks in simple, human-readable YAML format, known as playbooks, which are then executed against remote servers or infrastructure nodes. + +### Core Concepts: +- Agentless Architecture: Ansible operates in an agentless manner, meaning it doesn't require any software agents to be installed on managed nodes. Instead, it uses SSH (Secure Shell) protocol to establish connections and execute tasks remotely, making it lightweight and easy to deploy. +- Inventory: The inventory file is a list of managed hosts or nodes that Ansible will operate on. It can be defined in a simple text file or dynamically generated using scripts or plugins. Ansible uses this inventory to determine the target hosts for executing tasks. +- Playbooks: Playbooks are the heart of Ansible automation. They are YAML files that define a series of tasks to be executed on remote hosts. Playbooks can contain multiple plays, each consisting of one or more tasks. Tasks define the actions to be performed, such as installing packages, copying files, starting services, etc. +- Tasks: Tasks are individual units of work defined within playbooks. They represent the actions that Ansible will perform on remote hosts. Tasks can include modules, which are Ansible's building blocks for performing specific actions (e.g., yum for package management, copy for file transfer, service for managing services, etc.). +- Modules: Modules are small programs that Ansible executes on remote hosts to perform specific tasks. Ansible ships with a large number of built-in modules covering various aspects of system administration, networking, cloud management, and more. Users can also create custom modules to extend Ansible's functionality. +- Roles: Roles are a way of organizing and reusing Ansible content. They provide a structured approach for grouping related tasks, variables, files, and handlers into reusable units. Roles promote modularity, scalability, and maintainability of Ansible playbooks. + +### Key Features: +- Idempotent Execution: Ansible ensures idempotent execution, meaning running the same playbook multiple times has the same effect as running it once. This helps in maintaining the desired state of the infrastructure and prevents unintended changes. +- Declarative Syntax: Ansible playbooks use a declarative syntax, allowing users to specify the desired state of the system rather than imperatively listing the steps to achieve it. This makes playbooks easier to read, write, and understand. +- Extensibility: Ansible is highly extensible and can be integrated with other tools and technologies. It supports plugins, custom modules, dynamic inventories, and can be extended using various programming languages. +- Parallel Execution: Ansible can execute tasks in parallel across multiple hosts simultaneously, improving the speed and efficiency of automation workflows. +- Integration with Cloud Providers: Ansible provides modules and plugins for managing cloud resources on popular cloud platforms such as AWS, Azure, Google Cloud, and OpenStack, allowing users to automate cloud infrastructure provisioning and management. +- Community and Ecosystem: Ansible has a vibrant community with extensive documentation, tutorials, and support resources. It also integrates with configuration management frameworks, monitoring tools, version control systems, and CI/CD pipelines, enabling seamless integration into existing DevOps workflows. + +### Use Cases: +- Configuration Management: Ansible can be used to enforce and maintain the desired configuration of servers, applications, and networking devices across an IT infrastructure. +- Application Deployment: Ansible automates the deployment of applications and services, ensuring consistency and repeatability across development, testing, and production environments. +- Infrastructure Provisioning: Ansible facilitates the provisioning and lifecycle management of infrastructure resources, including virtual machines, containers, cloud instances, and network devices. +- Orchestration: Ansible orchestrates complex workflows and tasks involving multiple systems, services, and dependencies, enabling end-to-end automation of IT processes. +- Continuous Delivery: Ansible integrates with CI/CD pipelines to automate the deployment, testing, and release of software applications, accelerating the delivery of software updates and features. + +## Kubernetes +Kubernetes is an open-source container orchestration platform designed to automate the deployment, scaling, and management of containerized applications. It was originally developed by Google and is now maintained by the Cloud Native Computing Foundation (CNCF). Kubernetes provides a flexible and scalable platform for deploying and managing containerized workloads across a cluster of machines. + +### Core Concepts: +- Containerization: Kubernetes is built around the concept of containers, lightweight and portable units of software that package an application and its dependencies. Kubernetes supports popular container runtimes such as Docker and containerd. +- Cluster: A Kubernetes cluster is a set of nodes (servers) that run containerized applications orchestrated by Kubernetes. A cluster typically consists of one or more master nodes (control plane) and multiple worker nodes (compute nodes). +- Master Node: The master node is responsible for managing the Kubernetes cluster and its resources. It includes several components: + - API Server: Exposes the Kubernetes API, which allows users and external components to interact with the cluster. + - Scheduler: Assigns pods (groups of containers) to nodes based on resource availability and workload requirements. + - Controller Manager: Manages various controllers responsible for maintaining the desired state of the cluster, such as replication controller, endpoint controller, and namespace controller. + - etcd: Consistent and highly available key-value store used for storing cluster state and configuration data. +- Worker Node: Worker nodes are responsible for running containerized workloads (pods) and providing the necessary compute resources. Each worker node includes: + - Kubelet: An agent that runs on each node and communicates with the master node. It manages the pods and containers running on the node. + - Container Runtime: Software responsible for running containers, such as Docker or containerd. + - Kube-proxy: A network proxy that maintains network rules and enables communication between pods and services. +- Pod: The smallest deployable unit in Kubernetes. A pod encapsulates one or more containers that share networking and storage resources. Pods are ephemeral and can be scheduled, scaled, and managed independently. +- Deployment: A Kubernetes Deployment is a higher-level abstraction that manages the deployment and scaling of replica sets. It allows users to define and manage the desired state of applications, including the number of replicas, update strategy, and rolling updates. +- Service: Kubernetes Service is an abstraction that defines a logical set of pods and a policy by which to access them. Services provide a stable endpoint (IP address and port) for accessing applications running within the cluster, enabling load balancing and service discovery. + +### Key Features: +- Automatic Scaling: Kubernetes can automatically scale applications based on resource utilization metrics, such as CPU and memory usage, ensuring optimal performance and resource utilization. +- Self-Healing: Kubernetes monitors the health of applications and automatically restarts or reschedules containers that fail or become unresponsive, improving application availability and reliability. +- Rolling Updates and Rollbacks: Kubernetes supports rolling updates for deploying new versions of applications without downtime. It also allows for rollback to previous versions in case of deployment failures or issues. +- Service Discovery and Load Balancing: Kubernetes provides built-in service discovery and load balancing for distributing traffic across multiple instances of an application, improving scalability and reliability. +- Storage Orchestration: Kubernetes offers flexible storage options for persisting data, including local storage, network-attached storage (NAS), and cloud storage solutions. It supports dynamic provisioning and management of storage volumes. +- Configuration Management: Kubernetes enables the declarative management of application configuration and environment variables, allowing users to define the desired state of applications and infrastructure using YAML or JSON manifests. +- Multi-Tenancy and Security: Kubernetes provides features for securing and isolating workloads within a cluster, including role-based access control (RBAC), network policies, and pod security policies (PSPs). + +### Use Cases: +- Microservices Architecture: Kubernetes is well-suited for deploying and managing microservices-based applications, allowing teams to develop, deploy, and scale individual services independently. +- Continuous Integration/Continuous Deployment (CI/CD): Kubernetes integrates seamlessly with CI/CD pipelines, enabling automated testing, deployment, and delivery of applications with high velocity and efficiency. +- Hybrid and Multi-Cloud Deployments: Kubernetes provides a consistent platform for deploying and managing applications across on-premises data centers, public clouds, and hybrid cloud environments, offering flexibility and portability. +- Big Data and Machine Learning Workloads: Kubernetes can orchestrate complex data processing and analytics workloads, including big data frameworks (e.g., Apache Spark, Hadoop) and machine learning frameworks (e.g., TensorFlow, PyTorch), allowing organizations to scale and manage these workloads efficiently. +- Edge Computing: Kubernetes can be used to deploy and manage containerized applications at the network edge, bringing compute resources closer to end-users and devices for low-latency processing and improved performance. diff --git a/docs/Sunbird - Tech Architecture .pdf b/docs/Sunbird - Tech Architecture .pdf new file mode 100644 index 0000000000..7ca3a29c2b Binary files /dev/null and b/docs/Sunbird - Tech Architecture .pdf differ diff --git a/docs/dp-debug.md b/docs/dp-debug.md new file mode 100644 index 0000000000..2402332db9 --- /dev/null +++ b/docs/dp-debug.md @@ -0,0 +1,337 @@ +# Data-pipeline troubleshooting + +To verify if data-pipeline is working properly +- Ensure that if a unique, valid, recent event is ingested from kafka ingestion topic (e.g. `dev.telemetry.ingest`), it + is reaching the topic druid ingests from (e.g. `dev.druid.events.telemetry`) + - get golden data set from `test-data/flink_golden_dataset.json` + - update `data.params.msgid` to a unique uuid + - for each event in `data.events`, update `mid` to a unique uuid + - update time fields to a recent time, because `druid-event-validator` job drops older events (3 months) silently +- When the event is confirmed to be reaching druid ingestion topic, check if a data-source is created in Druid, and the + message is added + +## Kafka + +```sh +# ssh to kafka server (KP) +cd opt/kafka/bin + +# to create a topic +./kafka-topics.sh --create --topic --replication-factor 1 --partitions 1 --bootstrap-server localhost:9092 + +# to create topic for older version of kafka +./kafka-topics.sh --create --topic --replication-factor 1 --partitions 1 --zookeeper localhost:2181 + +# to list topics +./kafka-topics.sh --list --zookeeper localhost:2181 + +# start consumer add `--from-beginning` to get all messages from start +./kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic + +# to start a producer +./kafka-console-producer.sh --topic --broker-list localhost:9092 + +``` + +## Flink jobs + +### check logs for flink jobs +```sh +# ssh to kubernetes server (jenkins in our case) +# export Kubernetes config file environment variable +export KUBECONFIG=/path/to/kube-config.yaml + +# list pods flink-dev namespace +kubectl get po -n flink-dev + +# get logs for a pod +kubectl logs -n flink-dev +``` + +if logs contain errors because of missing topics +check if config for flink jobs is correct (`ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2`) +or, to create missing topics, ssh to KP(kafka) server and create the topic +```sh +# ssh to kafka server (KP) +cd opt/kafka/bin +./kafka-topics.sh --create --topic --replication-factor 1 --partitions 1 --bootstrap-server localhost:9092 +# for older version of kafka +./kafka-topics.sh --create --topic --replication-factor 1 --partitions 1 --zookeeper localhost:2181 +``` + +### flink state backend +if flink is unable to connect to state backend, check your state-backend config +or, optionally you can turn off the state backend (flink will store state in memory, but will not be able to recover state in case pod crashed) +```sh + +``` + +Note: for more detailed info on different kafka topics and flink jobs click [here](https://github.com/shobhit-vashistha/sb-dp-doc#readme "Data-pipeline doc") + +## Druid + +### Change log level for druid services + +to change log level for any of the druid services edit their respective `log4j2.xml` file. +for example to set broker log level to warn, edit it's `log4j2.xml` - + +Note: setting loglevel above `WARN` (e.g. `INFO`) will make logs very busy, and log files would inflate to +MBs in a couple of minutes, set loglevel back to `ERROR` as soon as done with debugging + +```xml + + + + + + "%d{ISO8601} %p [%t] %c - %m%n" + + + + + + + + + + + + + +``` + +### Druid config + +common druid config for services is present at `/data/druid/conf/druid/_common/common.runtime.properties` +config for individual services is present at `/data/druid/conf/druid//runtime.properties` + +#### for s3 compatible deep storage + +to use s3 as deep storage make sure `common.runtime.properties` contains following config + +``` +druid.extensions.loadList=["druid-s3-extensions"] +druid.extensions.directory=/data/druid/extensions + +druid.storage.type=s3 +druid.storage.bucket= +druid.storage.baseKey=druid/segments + +druid.s3.accessKey= +druid.s3.secretKey= + +# set protocol and endpoint together +druid.s3.endpoint.url=:// + +# or separately as +# druid.s3.endpoint.url= +# druid.s3.endpoint.protocol= +``` + +for non-aws s3-like stores (like ceph), we might have to add additional config +``` +# enable access of bucket from any region +druid.s3.forceGlobalBucketAccessEnabled=true + +# to enable path like access +# if true, url=:/// +# if false, url=://. +druid.s3.enablePathStyleAccess=true +``` + +to allow Druid to publish task logs to s3 add following config +``` +druid.indexer.logs.type=s3 +druid.indexer.logs.s3Bucket= +# path to logs within the bucker +druid.indexer.logs.s3Prefix=druid/stage/indexing_logs +``` + +additional config for s3 deep storage (optional) +``` +# uncomment to enable server side encryption for s3 +# druid.storage.sse.type=s3 + +# uncomment to enable v4 signing of requests +# druid.s3.endpoint.signingRegion= + +# uncomment to disable chunk encoding +# druid.s3.disableChunkedEncoding=true +``` + +#### S3 bucket policy + +Druid should have permissions to read and write from `druid` dir of the bucket +For S3, we would require `GetObject`, `PutObject`, `GetObjectAcl`, `PutObjectAcl` permissions + +Example policy might look like- + +`policy.json` +```json +{ + "Statement": [ + { + "Action": [ + "s3:ListAllMyBuckets" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::*" + }, + { + "Action": [ + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:GetBucketLocation", + "s3:AbortMultipartUpload", + "s3:GetObjectAcl", + "s3:GetObjectVersion", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:GetObject", + "s3:PutObjectAcl", + "s3:PutObject", + "s3:GetObjectVersionAcl" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::/druid", + "arn:aws:s3:::/druid/*" + ] + } + ] +} +``` + +to update bucket policy using `s3cmd`, first install s3cmd and configure using `s3cmd --configure`, then run + +```sh +s3cmd setpolicy policy.json s3:// +``` + +Depending on the library being used to upload files to storage, to make files accessible publicly we might need to set acl to public using `s3cmd` + +```sh +s3cmd setacl s3:////* --acl-public +``` + +#### for azure deep storage + +to use azure as deep storage make sure `common.runtime.properties` contains following config + +``` +druid.extensions.loadList=["druid-azure-extensions"] +druid.extensions.directory=/data/druid/extensions + +druid.storage.type=azure +druid.azure.account= +druid.azure.key= +druid.azure.container= +``` + +to allow Druid to publish task logs to azure add following config +``` +druid.indexer.logs.type=azure +druid.indexer.logs.container= +druid.indexer.logs.prefix= +``` + +#### misc config +``` +# uncomment to disable acl for deep storage +# druid.storage.disableAcl=true + +# uncomment to disable acl for only logs +# druid.indexer.logs.disableAcl=true +``` + +### Druid graceful restart / rolling update + +For configurations to take effect Druid services for which config has changed must be restarted. +All Druid services except for `middlemanager` can be restarted safely through `systemctl` + +```sh +# ssh to druid +systemctl restart druid_broker.service +systemctl restart druid_coordinator.service +systemctl restart druid_historical.service +systemctl restart druid_overlord.service +``` + +to gracefully restart `middlemanager` first we have to suspend all running supervisors. this publishes +segments which have not been published yet + +```sh +# ssh to druid +# get running supervisor names +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor -i + +# do this for all running supervisors +# suspend supervisor (stop running tasks and publish segments) +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//suspend + +# restart middlemanager service +systemctl restart druid_middlemanager.service + +# resume suspended supervisors +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//resume +``` + +### Druid API + +ports - to find out what ports each of the services are running check `runtime.properties` file in `/data/druid/conf/druid//` + +default ports - +```sh +# coordinator - 8081 +# broker - 8082 +# historical - 8083 +# overlord - 8090 +# middlemanager - 8091 +``` + +Check status, get data sources +```sh +# check status of overlord service +curl -X GET http://localhost:8090/status + +# show data sources +curl -X GET http://localhost:8081/druid/coordinator/v1/datasources -i +``` + +Manage Ingestion +```sh +# get running supervisor names +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor -i + +# inspect particular supervisor ingestion config +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor/ -i + +# inspect particular supervisor status +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor//status -i + +# inspect particular supervisor task stats +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor//stats -i + +# inspect tasks +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor/tasks -i + +# inspect pending tasks +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor/pendingTasks -i + +# inspect running tasks +curl -X GET http://localhost:8081/druid/indexer/v1/supervisor/runningTasks -i + + +# add new supervisor +curl -X POST -H 'Content-Type: application/json' -d @spec.json http://localhost:8090/druid/indexer/v1/supervisor + +# stop and delete supervisor +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//terminate -i + +# suspend supervisor (stop running tasks and publish segments) +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//suspend + +# resume supervisor +curl -X POST http://localhost:8090/druid/indexer/v1/supervisor//resume + +``` \ No newline at end of file diff --git a/docs/flink-dp-3.6.md b/docs/flink-dp-3.6.md new file mode 100644 index 0000000000..48d49a1eab --- /dev/null +++ b/docs/flink-dp-3.6.md @@ -0,0 +1,450 @@ +# Sunbird Data Pipeline (branch: release-3.6.0) + +The purpose of data-pipeline is to process the Telemetry (+ some other) data, and store it for use by +anyone who requires it, in a scalable manner. + +**Note:** this document assumes default configuration of flink jobs, actual configuration for each of the flink jobs is present [here](../kubernetes/helm_charts/datapipeline_jobs/values.j2) + +**Note:** To troubleshoot your data-pipeline setup see doc [here](dp-debug.md "Data-pipeline debug") + +## Overview + +Batch events are sent to Kafka Topic `env.telemetry.ingest` by the Telemetry API. From there a series of Flink jobs +transform this stream, first by de-duplicating and extracting individual events, generating `AUDIT` meta events, routing +different event types to different Kafka topics, de-normalizing data present in the stream, validating the data stream +for Druid ingestion, and sending this processed stream to `env.druid.*` Kafka topics. From there Druid ingests +the data using its Kafka-Ingestion service into *raw* and *rollup* clusters. The data stored in Druid is then available +to be viewed/visualized/analyzed using Superset. Each of the Kafka topics is backed up to Azure Blob Storage using a +Secor Process. + +## Components + +Sunbird Data Pipeline majorly consists of following Components +- [Data Source - Telemetry API - Node.js](#telemetry-api---nodejs) emits individual or batch events to Kafka Ingestion Topic +- [Transport - Apache Kafka](#apache-kafka---transport) for reliably moving data among various services +- [Transport Backup - Azure Blob Storage](#backup---secor---azure-blob-store) to back-up Kafka topics to Azure Blob Storage using a Secor Process +- [Stream Processing - Apache Flink](#apache-flink---stream-processing) i.e. De-duplication, De-normalization, Routing etc. +- [Batch Processing - Apache Spark](#apache-spark---batch-processing) for batch processing +- [OLAP Store - Apache Druid](#druid---olap-store) for storing processed data in raw and rollup clusters +- [Data Visualization - Superset](#superset) to access, visualize and analyze data present in Druid + +### Telemetry API - Node.js + +REST API implemented in Node.js, and used to send individual/batch events to Kafka Ingestion Topic. More details +regarding this can be found [here](http://docs.sunbird.org/latest/developer-docs/telemetry/specification/ "Telemetry Specs") + +### Apache Kafka - Transport + +Kafka is utilized to move data among different data-pipeline services, following table shows the various Kafka topics +and their purpose + +#### Kafka Topics + +Kafka Topic (env.) | Topic Content | Topic Consumers | Topic Producers +--------------------------------|---------------------------|---------------------------|---------------------------- +telemetry.ingest | Batch-Events | telemetry-extractor | Telemetry API +telemetry.extractor.duplicate | Duplicate Batch-Events | | telemetry-extractor +telemetry.extractor.failed | Failed Batch-Events | | telemetry-extractor +telemetry.raw | Unique Events | pipeline-preprocessor | telemetry-extractor +telemetry.assess.raw | ASSES/RESPONSE Events | | telemetry-extractor +telemetry.error | ERROR Events | | pipeline-preprocessor +telemetry.audit | AUDIT Events | user-cache-updater | pipeline-preprocessor +telemetry.duplicate | Duplicate Events | | pipeline-preprocessor,
de-normalization:Summary,
druid-events-validator +telemetry.unique | Unique Events | de-normalization | pipeline-preprocessor +telemetry.unique.secondary | Secondary Unique Events | | pipeline-preprocessor +telemetry.unique.primary | Unique Events | | pipeline-preprocessor +telemetry.denorm | De-normalized Events | druid-events-validator | de-normalization +druid.events.summary | Summary Events | Druid | de-normalization:Summary,
druid-events-validator +telemetry.derived | Summary Events | de-normalization:Summary | Spark +telemetry.derived.unique | Unique Summary Events | | de-normalization:Summary +druid.events.telemetry | Validated Events | Druid | druid-events-validator +telemetry.assess? | ASSES/RESPONSE Events? | assessment-aggregator | ? +telemetry.assess.failed | | | assessment-aggregator +issue.certificate.request | | | assessment-aggregator +learning.graph.events | | content-cache-updater | ? +events.deviceprofile | | device-profile-updater | ? +telemetry.failed | Failed Events | | telemetry-extractor,
pipeline-preprocessor,
de-normalization,
druid-events-validator +druid.events.log | LOG events | Druid | telemetry-extractor,
pipeline-preprocessor + +#### Backup - Secor - Azure Blob Store + +Kafka topics are backed up to Azure Blob Store using Secor Processes +[expand] + +### Apache Flink - Stream Processing + +#### Core Components `dp-core` + +Common components shared among all other components of Flink Data Pipeline + +`domain` - package containing event related classes + +`reader` - telemetry reading utilities + +`serde` - package containing Serialization/De-Serialization utilities for `String`, `Byte`, `Map` and `Event` + +`cache` - package containing utilities for connecting to and querying redis store +- `RedisConnect` - A Utility for providing connection to redis via `Jedis` +- `DataCache` - Utility to store and retrieve objects from a redis store of given index +- `DeDupeEngine` - De-Duplication Utility - provides helper methods to query de-duplication redis store to check/store a checksum + +`job` - package containing job related base classes and Flink-Kafka connection utility +- `BaseDuplication` - Base class (trait) providing de-duplication functionality, +- `BaseJobConfig` - Base class for job config +- `BaseProcessFunction` - Contains abstract classes for `ProcessFunction`s in the data-pipeline + - `JobMetrics` - Trait providing functionality to modify/retrieve job metrics in a thread safe manner + - `BaseProcessFunction` - abstract class extending `ProcessFunction`, `BaseDuplication` and `JobMetrics` takes config + as input, all data-pipeline `ProcessFunction`s inherit from this + - `WindowBaseProcessFunction` - abstract class extending `WindowProcessFunction`, `BaseDuplication` and `JobMetrics` takes config + as input, all data-pipeline `WindowProcessFunction`s inherit from this +- `FlinkKafkaConnector` - A Utility providing methods for creating Kafka sources and sinks for flink jobs, utilises `serde` + package for Serialization/De-Serialization functionality + +`util` +- `CassandraUtil` - Utility to connect to and query Cassandra +- `FlinkUtil` - Utility to provide `StreamExecutionEnvironment` with job specific config applied +- `JSONUtil` - Utility to help with JSON Serialization/De-Serialization +- `PostgresConnect` - Utility to connect to and query Postgres +- `RestUtil` - Utility for calling REST API + +#### Flink Jobs + +Sunbird Flink Job Projects follow the following package structure (mostly) - +1) `domain` - Contains all the models relevant to the job - POJO's or Scala case classes +2) `functions` - Contains Flink `ProcessFunction`s for user defined stream transformations +3) `task` - Contains Flink Tasks to be run and Config associated with them +4) `util` - Helper classes/functions +5) other packages + +*Note* - In the following documentation `ProcessFunction` subclasses are frequently attributed to routing messages that +satisfy a certain criterion to specific/multiple kafka topics. In reality, `ProcessFunction`s do not perform +the actual routing, they tag messages that satisfy a certain criterion with respective `OutputTag`s, and it's the +`StreamTask` that routes messages with these `OutputTag`s to respective kafka topics. This bit of detail has been +skipped to facilitate better understanding of the underlying logic. + + +#### Telemetry Extractor `telemetry-extractor` + +Config variable | Config key | Config value +------------------------|-----------------------------------|----------------------------------- +`kafkaInputTopic` | kafka.input.topic | env.telemetry.ingest +`kafkaDuplicateTopic` | kafka.output.duplicate.topic | env.telemetry.extractor.duplicate +`kafkaBatchFailedTopic` | kafka.output.batch.failed.topic | env.telemetry.extractor.failed +`kafkaSuccessTopic` | kafka.output.success.topic | env.telemetry.raw +`kafkaLogRouteTopic` | kafka.output.log.route.topic | env.druid.events.log +`kafkaFailedTopic` | kafka.output.failed.topic | env.telemetry.failed +`kafkaAssessRawTopic` | kafka.output.assess.raw.topic | env.telemetry.assess.raw +`redactEventsList` | redact.events.list | ASSESS, RESPONSE + + +`TelemetryExtractorStreamTask` + +This task reads from batch telemetry events from `kafkaInputTopic` and +- handles de-duplication of batch events by extracting message identifier and checking against a redis store +- parses unique batch events into individual events and generates `AUDIT` events +- routes `LOG`/`AUDIT` events and events in `redactEventsList` to respective kafka topics +- routes invalid/errored out events to kafka failure topics +- routes rest of events to be further processed by [`pipeline-preprocessor`](#pipeline-pre-processor-pipeline-preprocessor) + +Source - `kafkaInputTopic` + +Sink - `kafkaSuccessTopic`, `kafkaFailedTopic`, `kafkaDuplicateTopic`, `kafkaBatchFailedTopic`, `kafkaAssessRawTopic`, `kafkaLogRouteTopic` + +Transformations - +- Reads from source stream then applies `DeDuplicationFunction` + - `DeDuplicationFunction` + - routes failed batch-events to `kafkaBatchFailedTopic` + - routes duplicate batch-events to `kafkaDuplicateTopic` + - applies `ExtractionFunction` to the unique batch events + - `ExtractionFunction` + - generates `AUDIT` events (to count the events in a batch) which are routed to `kafkaLogRouteTopic` + - extracts individual events from batch-events + - routes events with size > `config.maxSize` to `kafkaFailedTopic` + - routes `LOG` events to `kafkaLogRouteTopic` + - applies `RedactorFunction` to events in `redactEventsList` + - routes other events to `kafkaSuccessTopic` + - `RedactorFunction` + - routes all events to `kafkaSuccessTopic` + - routes events with `questionType=Registration` to `kafkaAssessRawTopic` + + +#### Pipeline Pre-processor `pipeline-preprocessor` + +Config variable | Config key | Config value +--------------------------------|-------------------------------------------|--------------------------------- +`kafkaInputTopic` | kafka.input.topic | env.telemetry.raw +`kafkaFailedTopic` | kafka.output.failed.topic | env.telemetry.failed +`kafkaPrimaryRouteTopic` | kafka.output.primary.route.topic | env.telemetry.unique +`kafkaLogRouteTopic` | kafka.output.log.route.topic | env.druid.events.log +`kafkaErrorRouteTopic` | kafka.output.error.route.topic | env.telemetry.error +`kafkaAuditRouteTopic` | kafka.output.audit.route.topic | env.telemetry.audit +`kafkaDuplicateTopic` | kafka.output.duplicate.topic | env.telemetry.duplicate +`kafkaDenormSecondaryRouteTopic`| kafka.output.denorm.secondary.route.topic | env.telemetry.unique.secondary +`kafkaDenormPrimaryRouteTopic` | kafka.output.denorm.primary.route.topic | env.telemetry.unique.primary +`secondaryEvents` | kafka.secondary.events | INTERACT, IMPRESSION, SHARE_ITEM + + +`PipelinePreprocessorStreamTask` + +This task routes data to relevant Kafka topics. It also de-duplicates, validates event schema and flattens `SHARE` +events. Unique events are routed to be processed by [`de-normalization`](#de-normalization-de-normalization) + +Source - `kafkaInputTopic` + +Sink - `kafkaFailedTopic`, `kafkaPrimaryRouteTopic`, `kafkaLogRouteTopic`, `kafkaErrorRouteTopic`, +`kafkaAuditRouteTopic`, `kafkaDuplicateTopic`, `kafkaDenormSecondaryRouteTopic`, `kafkaDenormPrimaryRouteTopic` + +Transformations - +- Reads from source stream then applies `PipelinePreprocessorFunction` + - `PipelinePreprocessorFunction` + - if schema is missing or if validation fails events are routed to `kafkaFailedTopic` + - routes `LOG` events to `kafkaLogRouteTopic` + - routes duplicate events to `kafkaDuplicateTopic` + - routes unique events in `secondaryEvents` to `kafkaDenormSecondaryRouteTopic` + - routes rest of the unique events to `kafkaDenormPrimaryRouteTopic` + - also routes unique `AUDIT` events to `kafkaAuditRouteTopic` and `kafkaPrimaryRouteTopic` + - also routes unique `ERROR` events to `kafkaErrorRouteTopic` + - also routes unique `SHARE` events to `kafkaPrimaryRouteTopic` also, flattens them and routes flattened events + again to `kafkaPrimaryRouteTopic` + - also routes other unique events to `kafkaPrimaryRouteTopic` + + +#### De-Normalization `de-normalization` + +Config variable | Config key | Config value +----------------------------|---------------------------------------|--------------------------------- +`telemetryInputTopic` | kafka.input.telemetry.topic | env.telemetry.unique +`summaryInputTopic` | kafka.input.summary.topic | env.telemetry.derived +`telemetryDenormOutputTopic`| kafka.telemetry.denorm.output.topic | env.telemetry.denorm +`summaryDenormOutputTopic` | kafka.summary.denorm.output.topic | env.druid.events.summary +`summaryUniqueEventsTopic` | kafka.summary.unique.events.topic | env.telemetry.derived.unique +`failedTopic` | kafka.output.failed.topic | env.telemetry.failed +`duplicateTopic` | kafka.output.duplicate.topic | env.telemetry.duplicate +`eventsToskip` | skip.events | INTERRUPT + + +`DenormalizationStreamTask` + +This task performs de-normalization of event data (device, user, dial-code, content, location), and then routes events +to be validated for Druid Ingestion by [`druid-events-validator`](#druid-events-validator-druid-events-validator) + +Source - `telemetryInputTopic` + +Sink - `telemetryDenormOutputTopic` + +Transformations - +- Reads from source stream, keys by `event.did()`, windows by `config.windowCount` then applies `DenormalizationWindowFunction` + - `DenormalizationWindowFunction` + - filter out events older than `config.ignorePeriodInMonths` + - filter out events not suitable for de-normalization + - de-normalize device, user, dial-code, content, location for rest of the events + - route de-normalized events to `telemetryDenormOutputTopic` + + +`SummaryDenormalizationStreamTask` + +This task performs de-duplication of summary data [coming from Spark?] then further performs de-normalization +device, user, dial-code, content, location in the summary data, which it routes to be Ingested by [Druid](#druid---olap-store) + +Source - `summaryInputTopic` + +Sink - `summaryUniqueEventsTopic`, `duplicateTopic`, `telemetryDenormOutputTopic`, `summaryDenormOutputTopic` + +Transformations - +- Reads from source stream and applies `SummaryDeduplicationFunction` + - `SummaryDeduplicationFunction` + - routes duplicate summary events to `duplicateTopic` + - routes unique summary events to `summaryUniqueEventsTopic` + - applies `DenormalizationWindowFunction` to unique summary events after keying by `event.did()`, windowing by `config.windowCount` + - `DenormalizationWindowFunction` + - filter out events older than `config.ignorePeriodInMonths` + - filter out events not suitable for de-normalization + - de-normalize device, user, dial-code, content, location for rest of the events + - route de-normalized events to `summaryDenormOutputTopic` + + +#### Druid Events Validator `druid-events-validator` + +Config variable | Config key | Config value +----------------------------|---------------------------------------|--------------------------------- +`kafkaInputTopic` | kafka.input.topic | env.telemetry.denorm +`kafkaTelemetryRouteTopic` | kafka.output.telemetry.route.topic | env.druid.events.telemetry +`kafkaSummaryRouteTopic` | kafka.output.summary.route.topic | env.druid.events.summary +`kafkaFailedTopic` | kafka.output.failed.topic | env.telemetry.failed +`kafkaDuplicateTopic` | kafka.output.duplicate.topic | env.telemetry.duplicate + + +`DruidValidatorStreamTask` + +This task validates events for [Druid](#druid---olap-store) ingestion, summary and non-summary events are routed to respective kafka topics +after routing out invalid and duplicate events + +Source - `kafkaInputTopic` + +Sink - `kafkaTelemetryRouteTopic`, `kafkaSummaryRouteTopic`, `kafkaFailedTopic`, `kafkaDuplicateTopic` + +Transformations - +- Reads from source stream then applies `DruidValidatorFunction` + - `DruidValidatorFunction` + - validates events for druid ingestion and routes invalid events to `kafkaFailedTopic` + - checks if events are unique and routes duplicate events to `kafkaDuplicateTopic` + - routes unique valid summary events to `kafkaSummaryRouteTopic` + - routes unique valid non summary events to `kafkaTelemetryRouteTopic` + + +#### Assessment Aggregator `assessment-aggregator` + +Config variable | Config key | Config value +------------------------|-------------------------------|--------------------------------- +`kafkaInputTopic` | kafka.input.topic | env.telemetry.assess +`kafkaFailedTopic` | kafka.failed.topic | env.telemetry.assess.failed +`kafkaCertIssueTopic` | kafka.output.certissue.topic | env.issue.certificate.request + + +`AssessmentAggregatorStreamTask` + +This task aggregates assessment data saves it to cassandra and issues certificates + +Source - `kafkaInputTopic` + +Sink - `kafkaFailedTopic`, `kafkaCertIssueTopic`, cassandra + +Transformations - +- Reads from source stream then applies `AssessmentAggregatorFunction` + - `AssessmentAggregatorFunction` + - checks validity of assess events and routes invalid/errored-out events to `kafkaFailedTopic` + - performs aggregations on assess events and saves assessments to cassandra + - creates certificate issue events and routes them to `kafkaCertIssueTopic` + + +#### Content Cache Updater `content-cache-updater` + +Config variable | Config key | Config value +--------------------|-----------------------|--------------------------------- +`inputTopic` | kafka.input.topic | env.learning.graph.events + + +`ContentCacheUpdaterStreamTask` + +This task updates content cache and dial code properties in the redis cache + +Source - `inputTopic` + +Sink - redis + +Transformations - +- Reads from source stream then applies `ContentUpdaterFunction` + - `ContentUpdaterFunction` + - updates redis content cache + - if event has dial code properties, applies `DialCodeUpdaterFunction` + - `DialCodeUpdaterFunction` + - updates dial code properties in redis + + +#### Device Profile Updater `device-profile-updater` + +Config variable | Config key | Config value +------------------------|-----------------------|--------------------------------- +`kafkaInputTopic` | kafka.input.topic | env.events.deviceprofile + + +`DeviceProfileUpdaterStreamTask` + +This task updates/inserts device profile to redis cache and postgres database + +Source - `kafkaInputTopic` + +Sink - redis, postgres + +Transformations - +- Reads from source stream then applies `DeviceProfileUpdaterFunction` + - `DeviceProfileUpdaterFunction` + - ignores if there is no device id + - adds the device to redis cache and postgres database + + +#### User Cache Updater `user-cache-updater` + +Config variable | Config key | Config value +--------------------|-----------------------|--------------------------------- +`inputTopic` | kafka.input.topic | env.telemetry.audit + + +`UserCacheUpdaterStreamTask` + +This task creates or updates user data in redis cache, fetches user/location metadata from cassandra in case of update + +Source - `inputTopic` + +Sink - redis + +Transformations - +- Reads from source stream then applies `UserCacheUpdaterFunction` + - `UserCacheUpdaterFunction` + - either, creates user data and updates redis cache + - or, updates user data by fetching user metadata and location metadata from cassandra and then updates redis cache + + +#### User Cache Updater 2.0 `user-cache-updater-2.0` + +Config variable | Config key | Config value +--------------------|-----------------------|--------------------------------- +`inputTopic` | kafka.input.topic | env.telemetry.audit + + +`UserCacheUpdaterStreamTaskV2` + +This task creates or updates user data in redis cache, fetches user/location metadata from cassandra in case of update + +Source - `inputTopic` + +Sink - redis + +Transformations - +- Reads from source stream then applies `UserCacheUpdaterFunctionV2` + - `UserCacheUpdaterFunctionV2` + - creates user metadata or queries cassandra to get user metadata + - updates user data in redis cache + + + +### Apache Spark - Batch Processing +[expand] + +### Druid - OLAP Store + +Data Store | Type | Kafka Topic (env.) +----------------------------|-----------|------------------------ +telemetry-events | raw | events.telemetry +telemetry-log-events | raw | events.log +summary-events | raw | events.summary +telemetry-feedback-events | raw | events.telemetry +tpd-hourly-rollup-syncts | rollup | druid.events.telemetry +telemetry-rollup-ets | rollup | events.telemetry +summary-rollup-syncts | rollup | events.summary +summary-rollup-ets | rollup | events.summary +summary-distinct-counts | rollup | events.summary +error-rollup-syncts | rollup | events.error + + +#### Superset + +Visualize Druid data [expand] \ No newline at end of file diff --git a/docs/images/Stream processing architecture.drawio.svg b/docs/images/Stream processing architecture.drawio.svg new file mode 100644 index 0000000000..59473690de --- /dev/null +++ b/docs/images/Stream processing architecture.drawio.svg @@ -0,0 +1,4 @@ + + + +
ingest-router
telemery-extractor
pipeline-preprocessor
denorm-primary
denorm-secondary
From KP Kafka
telemetry.ingestion
telemetry.raw
telemetry.ingest
telemetry.raw
telemetry.unique.secondary
telemetry.unique.primary
telemetry.unique
Ceph
via Secor
user-cache-updater
Redis
telemetry.audit
WFS
denorm-summary
telemetry.derived
druid-validator
telemetry.denorm
Druid
druid.events.telemetry
druid.events.summary
content-cache-updater
device-profile-updater
From KP Kafka
learning.graph.events
From KP Kafka
events.deviceprofile
 \ No newline at end of file diff --git a/docs/images/apache-kafka-architecture3.png b/docs/images/apache-kafka-architecture3.png new file mode 100644 index 0000000000..d4abe0a1fe Binary files /dev/null and b/docs/images/apache-kafka-architecture3.png differ diff --git a/docs/images/batch-processing-architecture.drawio.svg b/docs/images/batch-processing-architecture.drawio.svg new file mode 100644 index 0000000000..7860150c7f --- /dev/null +++ b/docs/images/batch-processing-architecture.drawio.svg @@ -0,0 +1,4 @@ + + + +
competency-metrics
assessment-metrics
course-report
user-enrolment-report
user-report
wfs
summary-redis-sync
course-based-assessment-report
blended-program-report
data-warehouse-sync
weekly-claps
acbp-report
comms-console
survey-nps
hall-of-fame
Cassandra
ES
Postgres (app)
Druid Dashboard Sources
Druid Summary
Druid NPS
Blob
File System
Redis
Postgres Data Warehouse
Blob
(backups)
data-exhaust
FS-exhaust

raw data

Avro format
FS-dw

transformed
data

Avro format

for data
warehouse
Sources
Spark Jobs
(in order of execution)
Sinks
Cassandra
MongoDB
Write
Read
Note:
 
- All writes to sinks are not displayed because they  clutter the diagram
 \ No newline at end of file diff --git a/docs/images/spark-architecture.png b/docs/images/spark-architecture.png new file mode 100644 index 0000000000..cfc286ed7b Binary files /dev/null and b/docs/images/spark-architecture.png differ diff --git a/docs/telemetry-v3.md b/docs/telemetry-v3.md new file mode 100644 index 0000000000..151a33d195 --- /dev/null +++ b/docs/telemetry-v3.md @@ -0,0 +1,834 @@ +## Telemetry Envelope + +```javascript +{ + // About the event + "eid": , // Required. + "ets": , // Required. Epoch timestamp of event (time in milli-seconds. For ex: 1442816723) + "ver": , // Required. Version of the event data structure, currently "3.0" + "mid": , // Required. Unique message ID. Used for deduplication, replay and update indexes + + // Who did the event + "actor": { // Required. Actor of the event. + "id": , // Required. Can be blank. Id of the actor. For ex: uid incase of an user + "type": // Required. Can be blank. User, System etc. + }, + + // Context of the event + "context": { // Required. Context in which the event has occured. + "channel": , // Required. Channel which has produced the event + "pdata": { // Optional. Producer of the event + "id": , // Required. unique id assigned to that component + "pid": , // Optional. In case the component is distributed, then which instance of that component + "ver": // Optional. version number of the build + }, + "env": , // Required. Unique environment where the event has occured. + "sid": , // Optional. session id of the requestor stamped by portal + "did": , // Optional. uuid of the device, created during app installation + "cdata": [{ // Optional. correlation data + "type":"", // Required. Used to indicate action that is being correlated + "id": "" // Required. The correlation ID value + }], + "rollup": { // Optional. Context rollups example: Organization heirarchy("L1": "Parent org id", "l2": "sub org id") + "l1": "", + "l2": "", + "l3": "", + "l4": "" + } + }, + + // What is the target of the event + "object": { // Optional. Object which is the subject of the event. + "id": , // Required. Id of the object. For ex: content id incase of content + "type": , // Required. Type of the object. For ex: "Content", "Community", "User" etc. + "ver": , // Optional. version of the object + "rollup": { // Optional. Rollups to be computed of the object. Only 4 levels are allowed. example: Textbook heirarchy("L1": "Parent node id", "l2": "Child node id") + "l1": "", + "l2": "", + "l3": "", + "l4": "" + } + }, + + // What is the event data + "edata": {} // Required. + + // Tags + "tags": [""] // Optional. Encrypted dimension tags passed by respective channels +} + +``` + +## Telemetry Events + +* [START](#start) - Start of a multi step activity. For ex: Session, App, Tools, Community etc would have start and end +* [END](#end) - End of a work flow +* [IMPRESSION](#impression) - A visit to a specific page by an user +* [INTERACT](#interact) - User interaction on the page (such as search, click, preview, move, resize, configure) +* [ASSESS](#assess) - An assessment has happened during content play +* [RESPONSE](#response) - To capture an user response. For ex: Responded to a poll or a calendar event or a question +* [INTERRUPT](#interrupt) - An interrupt trigged to the app (such as mobile app sent to background) +* [FEEDBACK](#feedback) - Feedback provided +* [SHARE](#share) - When any share happens. For ex: Share content, telemetry data, link, file etc +* [AUDIT](#audit) - When an object is changed. This includes lifecycle changes as well. +* [ERROR](#error) - When an error has occured +* [HEARTBEAT](#heartbeat) - An heart beat event to denote that the process is running. +* [LOG](#log) - Generic log event. API call, Service call, app update etc +* [SEARCH](#search) - A search is triggered (content, item, asset etc) +* [METRICS](#metrics) - Service business metrics (also accessible via health API) +* [SUMMARY](#summary) - A summary event +* [EXDATA](#exdata) - A generic wrapper event to capture encrypted/serialized data + +## Standard Flow + +### Mobile App Standard Flow +Here is the standard flow of events in the stack for a mobile application: + +``` +START(type: "app") + ...| --> app events such as IMPRESSION, FEEDBACK, etc may happen + START(type: "session") + ... + | --> IMPRESSION - For the pages that the user visits + | --> INTERACT --> one of the content is clicked + | --> START(type: "player") --> events generated by specific content + ...| + ...| --> in-content events such as ASSESS, INTERACT, IMPRESSION, LEVEL_SET etc. + ...| + | --> END(type: "player") + | --> IMPRESSION - Returned back to mobile app for content player + ...| --> app events such as IMPRESSION, INTERACT, etc. may happen + | --> INTERACT --> one of the content is clicked + | --> START(type: "player") --> events generated by specific content + ...| + ...| --> in-content events such as ASSESS, INTERACT, IMPRESSION, LEVEL_SET etc. + ...| + | --> END(type: "player") + | --> IMPRESSION - Returned back to mobile app for content player + ...| --> app events such as IMPRESSION, INTERACT, etc. may happen + END(type: "session") + ...| --> app events such as APP_UPDATE, FEEDBACK, etc may happen +END(type: "app") +``` + +### Portal Standard Flow +Here is the standard flow of events in the portal stack: + +``` +AUDIT (object: user) --> (Optional if a user is created for the first time) +START(type: "session") --> User session starts + ... + | --> IMPRESSION - For the pages that the user visits + | --> INTERACT - For the interactions on the page + // there is no explicit logout/timeout +``` + +### Editor Standard Flow +Sample flow to find out when assets are added or removed + +``` +START (type: "session") - User logs in + ... + | --> IMPRESSION (Portal) - User visits content creation page (cdata session) + | --> INTERACT (Portal) - User initiates content creation + | --> AUDIT (Platform) - User creates a new content + | --> START (type: "editor", mode: "content") + ... + | --> INTERACT (Editor) - In editor, user is loading the asset browser + | --> SEARCH (Platform) - In AT, user is searching for assets (cdata session, search result id) + | --> PLUGIN_LIFECYCLE (Editor) - User selects the asset for content (which search result was used) + | --> PLUGIN_LIFECYCLE (Editor) - User removes the asset from content + | --> INTERACT (Editor) - User clicks save in AT + | --> ACCESS (Platform) - Platform save API is called + | --> INTERACT (Editor) - User clicks on submit to review + | --> AUDIT (object: Content, state: "Review", prevstate: "Draft") - Platform sends the content to review state + | --> END (type: "editor", mode: "content") - User closes the editor and goes back to portal +``` + +### Back-end Services Standard Flow + +``` +AUDIT (object: Service, state: "Ready") --> State transition to READY + ...| --> ACCESS events for API requests + ACCESS + LOG --> Log events in the context of the incoming request (by request correlation ID) + ... + METRICS --> Health/business metrics (e.g. number of jobs executed) +AUDIT (object: Service, state: "Stopped") --> State transition to STOPPED +``` + +### Analytics Jobs Standard Flow +Here is the standard flow of events in the analytics job stack: + +``` +JOB_START --> Data product job start + ... + | --> JOB_LOG - To log some data while Data product job is in progress + ... +JOB_END --> Data product job end +``` + +## Event Definitions +Following sections provide exact event specific structures ```edata``` for all events. Generic envelope is defined [here](#telemetry-envelope). + +## Defined Structures + +```javascript + +// Device Specification +DSPEC = { + "os": "", // OS name and version + "make": "", // device make and model + "id": "", // physical device id if available from OS + "mem": , // total mem in MB, -1 if unknown + "idisk": , // total internal disk in GB, -1 if unknown + "edisk": , // total external disk (card) in GB, -1 if unknown + "scrn": , // screen size in inches, -1 if unknown + "camera": "13,1.3", // primary and secondary camera specs + "cpu": "", // processor name + "sims" , // number of sim cards, -1 if unknown + "cap": [] +} + +UASPEC = { + "agent": "Mozilla", // which user agent (mozilla, chrome, safari, ie) + "ver": "5.0", // Agent version number + "system": "iPad; U; CPU OS 3_2_1 like Mac OS X; en-us" // System identification + "platform": "AppleWebKit/531.21.10" // client platform, + "raw": "Mozilla\/5.0 (X11; Linux x86_64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/57.0.2987.98 Safari\/537.36" // Raw user agent of server +} + +OBJECT = { + "id": "", // Required. unique id for the object type + "type": "", // Required. Object Type - Tag, User, Partner, Content, Item, Asset + "ver": "", // Required. Version of the object + "subtype": "", // Optional. Sub type for each respective type. For ex: for Content - Story/Worksheet/Resource etc. + "name": "", // Optional. Name of the object eg: Name of the partner or user or content + "code": "", // Optional. unique code for the object + "parent": { + "id": "", // Optional. parent id of the object + "type": "" // Optional. parent type of the object. Required if parentid is present. + } +} + +TARGET = { + "id": "", // Required. unique id for the target + "ver": "", // Required. version of the target + "type": "", // Required. Type of the target + "parent": { + "id": "", // Optional. parent id of the object + "type": "" // Optional. parent type of the object. Required if parentid is present. + } +} + +PLUGIN = { + "id": "", // Required. unique id for the plugin. For ex: org.ekstep.text + "ver": "", // Required. version of the plugin + "category": "" // Optional. Category of the plugin. For ex: "core", "library", "numeracy" etc +} + +VISIT = { + "objid": "", // Required. unique id for the object visited + "objtype": "", // Required. Type of the object visited + "objver": "", // Optional. version of the object visited + "section": "", // Optional. Free flowing text + "index": // Optional. Index of the object within a given list +} + +QUESTION = { + "id": "", // unique assessment question id. its an required property. + "type": "", // type of the question. its an optional property. + "maxscore", // user defined score to this assessment/question. + "exlength": , // expected time (decimal number) in seconds that ideally child should take + "params": [ // Array of parameter tuples + {"id":"value"} // for ex: if var1 is substituted with 5 apples the parameter is {"var1":"5"} + ], + "uri": "", // Unique external resource identifier if any (for recorded voice, image, etc.) + "desc": "short description", + "title": "title", + "mmc": [], // User defined missing micros concepts + "mc": [] // micro concepts list +} + +``` + +### START + +```javascript +{ + "edata": { + "type": "", // Required. app, session, editor, player, workflow, assessment + "dspec": DSPEC, // Optional. Device spec + "uaspec": UASPEC, // Optional. User agent spec + "loc": "", // Optional. Location of the device + "mode": "", // Optional. Mode of start. For "player" it would be "play/edit/preview". For Workflow it would be Review/Flag/Publish. For editor it could be "content", "textbook", "generic", "lessonplan" etc + "duration": , // Optional. Time taken to initialize/start + "pageid": "" // Optional. Page/Stage id where the start has happened. + } +} +``` + +### END + +```javascript +{ + "edata": { + "type": , // Required. app, session, editor, player, workflow, assessment + "mode": "", // Optional. Mode of start. For "player" it would be "play/edit/preview". For Workflow it would be Review/Flag/Publish. For editor it could be "content", "textbook", "generic", "lessonplan" etc + "duration": , // Optional. Total duration from start to end in seconds + "pageid": "", // Optional. Page/Stage id where the end has happened. + "summary": [{"key":"value"}] // Optional. Summary of the actions done between start and end. For ex: "progress" for player session, "nodesModified" for collection editor + } +} +``` + +### IMPRESSION + +```javascript +{ + "edata": { + "type": "", // Required. Impression type (list, detail, view, edit, workflow, search) + "subtype": "", // Optional. Additional subtype. "Paginate", "Scroll" + "pageid": "", // Required. Unique page id + "uri": "", // Required. Relative URL of the content + "duration": , // Optional. Time taken to render the page/stage in seconds + "visits": [VISIT] // Optional. Capture the object visits + } +} +``` + +### INTERACT + +```javascript +{ + "edata": { + "type": "", // Required. Type of interaction CLICK,TOUCH,DRAG,DROP,PINCH,ZOOM,SHAKE,ROTATE,SPEAK,LISTEN,WRITE,DRAW,START,END,CHOOSE,ACTIVATE,SHOW,HIDE,SCROLL,HEARTBEAT,OTHER + "subtype": "", // Optional. Additional types for a global type. For ex: for an audio the type is LISTEN and thesubtype can be one of PLAY,PAUSE,STOP,RESUME,END + "id": "", // Required. Resource (button, screen, page, etc) id on which the interaction happened - use systemidentifiers when reporting device events + "pageid": "", // Optional. Stage or page id on which the event happened + "target": TARGET, // Optional. Target context where the interaction has happened + "duration": , // Optional. Time taken to interact on object in seconds + "plugin": PLUGIN, // Optional. Plugin on which the interaction has happend + "extra": { // Optional. Extra attributes for an interaction + "pos": [{"x":,"y":,"z":}], // Array of positional attributes. For ex: Drag and Drop has two positional attributes. One where the drag has started and the drop point + "values": [], // Array of values, e.g. for timestamp of audio interactions + } + } +} +``` + +### ASSESS + +```javascript +{ + "edata": { + "item": QUESTION, // Required. Question Data + "index": , // Optional. Index of the question within a content. + "pass": "", // Required. Yes, No. This is case-sensitive. default value: No. + "score": , // Required. Evaluated score (Integer or decimal) on answer(between 0 to 1), default is 1 if pass=YES or 0 if pass=NO. + "resvalues": [{"id":"value"}], // Required. Array of key-value pairs that represent child answer (result of this assessment) + "duration": // Required. time taken (decimal number) for this assessment in seconds + } +} +``` + +### RESPONSE + +```javascript +{ + "edata": { + "target": TARGET, // Required. Target of the response + "type": "", // Required. Type of response. CHOOSE, DRAG, SELECT, MATCH, INPUT, SPEAK, WRITE + "values": [{"key":"value"}] // Required. Array of response tuples. For ex: if lhs option1 is matched with rhs optionN - [{"lhs":"option1"}, {"rhs":"optionN"}] + } +} +``` + +**Examples** + +1. Responding to a question + +```javascript +{ + "object": { + "id": "content101", + "type": "content", + "hrchy": { + "l1": "textbook101", + "l2": "course101", + "l3": "curriculumA" + } + } + "edata": { + "target": { + "id": "q-101", + "type": "AssessmentItem", + "parent": { + + } + }, // Required. Question Data + "type": "SELECT", // Required. Type of response. CHOOSE, DRAG, SELECT, MATCH, INPUT, SPEAK, WRITE + "values": [{"state":"selected", "option": "A"}] // Required. Array of response tuples. For ex: if lhs option1 is matched with rhs optionN - [{"lhs":"option1"}, {"rhs":"optionN"}] + } +} +``` + +2. Responding to a poll + +```javascript +{ + "object": { + "id": "community101", + "type": "Community", + "hrchy": { + "l1": "poll101", + "l2": "thread202", + "l3": "forum211" + } + } + "edata": { + "target": { + "id": "p101", + "type": "Poll", + "parent": { + "id": "thread202", + "type": "ForumThread" + } + }, // Required. Question Data + "type": "MATCH", // Required. Type of response. CHOOSE, DRAG, SELECT, MATCH, INPUT, SPEAK, WRITE + "values": [{"state":"selected", "option": "A"}] // Required. Array of response tuples. For ex: if lhs option1 is matched with rhs optionN - [{"lhs":"option1"}, {"rhs":"optionN"}] + } +} +``` + +3. Responding to a calendar event (rsvp) + +```javascript +{ + "object": { + "id": "community101", + "type": "Community" + "hrchy": { + "l1": "cal101" + } + }, + "edata": { + "target": { + "id": "event101", + "type": "Event", + "parent": { + "id": "cal101", + "type": "Calendar" + } + }, // Required. Question Data + "type": "CHOOSE", // Required. Type of response. CHOOSE, DRAG, SELECT, MATCH, INPUT, SPEAK, WRITE + "values": [{"state": "selected", "option":"Rejected/Decline", "reason": "Conflicting schedule"}] // Required. Array of response tuples. For ex: if lhs option1 is matched with rhs optionN - [{"lhs":"option1"}, {"rhs":"optionN"}] + } +} +``` + +### INTERRUPT + +```javascript +{ + "edata": { + "type": "", // Required. [m:background, m:resume] + "pageid": "" // Optional. Page id where the interrupt has happened + } +} +``` + +### FEEDBACK + +```javascript +{ + "edata": { + "rating": 3, // Optional. Numeric score (+1 for like, -1 for dislike, or 4.5 stars given in a rating) + "comments": "User entered feedback" // Optional. Text feedback (if any) + } +} +``` + +### SHARE + +```javascript +{ + "edata": { + "dir": "", // In/Out + "type": "", // File/Link/Message + "items": [{ // Required. array of items shared + "id": "", + "type": "", + "ver": "" + "params": [ + {"key": "value"} + }], + "origin": { // Origin of the share file/link/content + "id": "", // Origin id + "type": "" // Origin type + }, + "to": { + "id": "", + "type": "" + } + }] + } +} +``` + +**Examples** + +1. Export content on app + +```javascript +{ + "edata": { + "dir": "out", + "type": "File", + "items": [{ + "id": "numeracy_377", + "type": "Content", + "ver": "11", + "params": [ + {"transfers": "3"}, + {"size": "23456.0"} + ], + "origin": { + "id": "0fca12d5-9b4f-48ea-89bc-31220f78f14a", + "type": "Device" + } + }, { + "id": "numeracy_398", + "type": "Content", + "ver": "5", + "params": [ + {"transfers": "7"}, + {"size": "23456.0"} + ], + "origin": { + "id": "0fca12d5-9b4f-48ea-89bc-31220f78f14a", + "type": "Device" + } + }] + } +} +``` + +2. Import content on app + +```javascript +{ + "edata": { + "dir": "in", + "type": "File", + "items": [{ + "id": "numeracy_377", + "type": "Content", + "ver": "11", + "origin": { + "id": "0fca12d5-9b4f-48ea-89bc-31220f78f14a", + "type": "Device" + } + }, { + "id": "numeracy_398", + "type": "Content", + "ver": "5", + "origin": { + "id": "0fca12d5-9b4f-48ea-89bc-31220f78f14a", + "type": "Device" + } + }] + } +} +``` + +3. Download content on app + +```javascript +{ + "edata": { + "dir": "in", + "type": "File", + "items": [{ + "id": "numeracy_377", + "type": "Content", + "ver": "11" + }] + } +} +``` + +3. Export profile on app + +```javascript +{ + "edata": { + "dir": "out", + "type": "File", + "items": [{ + "id": "user1", + "type": "User", + "origin": { + "id": "0fca12d5-9b4f-48ea-89bc-31220f78f14a", + "type": "Device" + } + }, { + "id": "user999", + "type": "User", + "origin": { + "id": "0fca12d5-9b4f-48ea-89bc-31220f78f14a", + "type": "Device" + } + }] + } +} +``` + +4. Import profile on app + +```javascript +{ + "edata": { + "dir": "in", + "type": "File", + "items": [{ + "id": "user1", + "type": "User", + "origin": { + "id": "0fca12d5-9b4f-48ea-89bc-31220f78f14a", + "type": "Device" + } + }, { + "id": "user999", + "type": "User", + "origin": { + "id": "0fca12d5-9b4f-48ea-89bc-31220f78f14a", + "type": "Device" + } + }] + } +} +``` + +5. Export telemetry on app + +```javascript +{ + "edata": { + "dir": "out", + "type": "File", + "items": [{ + "id": "telemetry1", + "type": "Telemetry", + "params": [ + {"count": "70"}, + {"size": "23456.0"} + ], + "origin": { + "id": "0fca12d5-9b4f-48ea-89bc-31220f78f14a", + "type": "Device" + } + }] + } +} +``` + +7. Share link + +```javascript +{ + "edata": { + "dir": "in", + "type": "Link", + "items": [{ + "obj": { + "id": "numeracy_377", + "type": "Content" + }, + "params": [ + {"link": "https://community.ekstep.in/content/numeracy_377"} + ] + }] + } +} +``` + +8. Share Post + +```javascript +{ + "edata": { + "dir": "out", + "type": "Message", + "items": [{ + "obj": { + "id": "thread101", + "type": "ForumThread" + }, + "to": { + "id": "comm202", + "type": "Community" + } + }] + } +} +``` + +### AUDIT + +```javascript +{ + "edata": { + "props": [""], // Updated properties + "state": "", // Optional. Current state + "prevstate": "", // Optional. Previous state + "duration": // Optional. Duration or time taken in seconds to change from prevstate to current state. Should only be passed if the state has truly changed + } +} +``` + +### ERROR + +```javascript +{ + "edata": { + "err": "", // Required. Error code + "errtype": "", // Required. Error type classification - "SYSTEM", "MOBILEAPP", "CONTENT" + "stacktrace": "", // Required. Detailed error data/stack trace + "pageid": "", // Optional. Page where the error has occured + "object": OBJECT, // Optional. Object on which the error occured + "plugin": PLUGIN // Optional. Plugin in which the error occured + } +} +``` + +### HEARTBEAT + +```javascript +{ + "edata": { + } +} +``` + +### LOG + +```javascript +{ + "edata": { + "type": "", // Required. Type of log (system, process, api_access, api_call, job, app_update etc) + "level": "", // Required. Level of the log. TRACE, DEBUG, INFO, WARN, ERROR, FATAL + "message": "", // Required. Log message + "pageid": "", // Optional. Page where the log event has happened + "params": [{"key":"value"}] // Optional. Additional params in the log message + } +} +``` + +**Exaples** + +1. Simple log + +```javascript +``` + +2. API log when being accessed + +```javascript +{ + "edata": { + "type": "api_access", + "level": "INFO", + "message": "", + "params": [ + {"rid": "org.ekstep.search"}, // Resource being accessed + {"uip": "10.0.0.7"}, // User IP address (if available) - server will anonymize + {"title": "Search API"}, // Title of the resource for analytics (e.g. the page title) + {"category": "contentsearch"}, // Categorization of the event + {"url": "/content/search"}, // Categorization of the event + {"size": "3333"}, // Size of the response in bytes + {"duration": "333"}, // Response time in ms + {"status": "200"}, // Response code (use standard protocol codes - e.g. 200, 404) + {"protocol": "http"}, // Protocol of access (http, or others such as messaging/Akka calls) + {"method": "GET"}, // Protocol specific method (e.g. GET/POST) + {"req": ""} // Serialized request + ] + } +} +``` + +### SEARCH + +Please note: + +1. The search is not indexed as it might contain PII +2. The search is available as an exhaust post anonymization + +```javascript +{ + "edata": { + "type": "", + "query": "", // Required. Search query string + "filters": {}, // Optional. Additional filters (see the API spec) + "sort": {}, // Optional. Additional sort parameters + "correlationid": "", // Optional. Server generated correlation id (for mobile app's telemetry) + "size": 333, // Required. Number of search results + "topn": [{}] // Required. top N (configurable) results with their score + } +} +``` + +### METRICS + +```javascript +{ + "edata": { + "metric1": Int, + "metric2": Int + /// more metrics, each is a key value + } +} +``` + +### SUMMARY + +```javascript +{ + "edata": { + "type": "", // Required. Type of summary. Free text. "session", "app", "tool" etc + "mode": "", // Optional. + "starttime": Long, // Required. Epoch Timestamp of app start. Retrieved from first event. + "endtime": Long, // Required. Epoch Timestamp of app end. Retrieved from last event. + "timespent": Double, // Required. Total time spent by visitor on app in seconds excluding idle time. + "pageviews": Long, // Required. Total page views per session(count of CP_IMPRESSION) + "interactions": Long, // Required. Count of interact events + "envsummary": [{ // Optional + "env": String, // High level env within the app (content, domain, resources, community) + "timespent": Double, // Time spent per env + "visits": Long // count of times the environment has been visited + }], + "eventssummary": [{ // Optional + "id": String, // event id such as CE_START, CE_END, CP_INTERACT etc. + "count": Long // Count of events. + }], + "pagesummary": [{ // Optional + "id": String, // Page id + "type": String, // type of page - view/edit + "env": String, // env of page + "timespent": Double, // Time taken per page + "visits": Long // Number of times each page was visited + }] + } +} +``` + +### EXDATA + +```javascript +{ + "edata": { + "type": "", // Free flowing text. For ex: partnerdata, xapi etc + "data": "" // Serialized data (can be either encrypted/encoded/stringified) + } +} +``` \ No newline at end of file diff --git a/druid/adhoc-jobs/dependency-reduced-pom.xml b/druid/adhoc-jobs/dependency-reduced-pom.xml deleted file mode 100644 index 39716f4f57..0000000000 --- a/druid/adhoc-jobs/dependency-reduced-pom.xml +++ /dev/null @@ -1,119 +0,0 @@ - - - 4.0.0 - org.ekstep - adhoc-jobs - 1.0 - - src/main/scala - src/test/scala - adhoc-jobs-1.0 - - - net.alchim31.maven - scala-maven-plugin - 3.2.2 - - - - compile - testCompile - - - - -dependencyfile - ${project.build.directory}/.scala_dependencies - - - - - - - org.scalatest - scalatest-maven-plugin - 2.0.0 - - - test - test - - test - - - - - - maven-shade-plugin - 2.3 - - - package - - shade - - - - - - - META-INF/services/org.jclouds.apis.ApiMetadata - - - META-INF/services/org.jclouds.providers.ProviderMetadata - - - - - *:* - - *:* - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - maven-assembly-plugin - 2.3 - - - make-assembly - package - - single - - - - - - src/main/assembly/src.xml - - - - - - - - org.scala-lang - scala-library - 2.11.8 - provided - - - org.scalatest - scalatest_2.11 - 2.2.4 - test - - - - 2.11.8 - 1.1.1 - 2.11 - 2.0.1 - - - diff --git a/druid/adhoc-jobs/pom.xml b/druid/adhoc-jobs/pom.xml deleted file mode 100644 index a418f7c4d7..0000000000 --- a/druid/adhoc-jobs/pom.xml +++ /dev/null @@ -1,232 +0,0 @@ - - - 4.0.0 - - org.ekstep - adhoc-jobs - 1.0 - - - 2.4.4 - 2.11.8 - 2.11 - 1.1.1 - - - - - org.apache.spark - spark-core_${scala.maj.version} - ${spark.version} - provided - - - jets3t - net.java.dev.jets3t - - - - - org.scala-lang - scala-library - ${scala.version} - provided - - - com.fasterxml.jackson.core - jackson-annotations - 2.7.8 - - - com.fasterxml.jackson.core - jackson-core - 2.7.8 - - - com.fasterxml.jackson.core - jackson-databind - 2.7.8 - - - com.fasterxml.jackson.module - jackson-module-scala_${scala.maj.version} - 2.7.8 - - - org.apache.commons - commons-text - 1.6 - - - com.datastax.spark - spark-cassandra-connector_${scala.maj.version} - 2.0.8 - - - org.apache.spark - spark-mllib_${scala.maj.version} - ${spark.version} - - - org.apache.spark - spark-core_2.11 - - - - - org.scalatest - scalatest_${scala.maj.version} - 2.2.4 - test - - - - com.typesafe - config - 1.3.3 - - - - org.elasticsearch - elasticsearch-spark-20_2.11 - 6.6.2 - - - - com.redislabs - spark-redis - 2.3.1-RC1 - - - - redis.clients - jedis - 3.0.1 - - - - org.apache.spark - spark-streaming_2.11 - 2.1.0 - provided - - - - org.sunbird - cloud-store-sdk - 1.2.6 - - - com.microsoft.azure - azure-storage - - - com.fasterxml.jackson.core - jackson-core - - - org.apache.httpcomponents - httpclient - - - com.google.guava - guava - - - - - com.microsoft.azure - azure-storage - 3.0.0 - - - org.apache.hadoop - hadoop-client - 2.7.3 - provided - - - org.apache.hadoop - hadoop-common - 2.7.3 - provided - - - org.apache.avro - avro - - - - - org.apache.hadoop - hadoop-aws - 2.7.3 - provided - - - - - - adhoc-jobs-1.0 - src/main/scala - src/test/scala - - - - net.alchim31.maven - scala-maven-plugin - 3.2.2 - - - - compile - testCompile - - - - -dependencyfile - ${project.build.directory}/.scala_dependencies - - - - - - - org.scalatest - scalatest-maven-plugin - 2.0.0 - - - test - test - - test - - - - - - - maven-assembly-plugin - 2.3 - - - src/main/assembly/src.xml - - - - - make-assembly - package - - single - - - - - - - - \ No newline at end of file diff --git a/druid/adhoc-jobs/src/main/assembly/src.xml b/druid/adhoc-jobs/src/main/assembly/src.xml deleted file mode 100644 index 66dc0a42c8..0000000000 --- a/druid/adhoc-jobs/src/main/assembly/src.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - distribution - - tar.gz - - - - ${basedir}/src/main/resources - /resources - - *.conf - - - - ${basedir}/src/scripts - /scripts - - *.sh - - - - target - / - - *.jar - - - - - - ../ingestion-spec/content_index_batch.json - /druid_models - true - - - - - lib - true - - - diff --git a/druid/adhoc-jobs/src/main/resources/DeviceProfile.conf b/druid/adhoc-jobs/src/main/resources/DeviceProfile.conf deleted file mode 100644 index 4e6eb88351..0000000000 --- a/druid/adhoc-jobs/src/main/resources/DeviceProfile.conf +++ /dev/null @@ -1,4 +0,0 @@ -spark.cassandra.connection.host="__cassandra__host" -cassandra.keyspace="__env__" -cassandra.deviceprofileOldTable="device_profile" -cassandra.deviceprofileNewTable="device_profile_temp" diff --git a/druid/adhoc-jobs/src/main/resources/DialcodeRedisIndexer.conf b/druid/adhoc-jobs/src/main/resources/DialcodeRedisIndexer.conf deleted file mode 100644 index 9f60abc672..0000000000 --- a/druid/adhoc-jobs/src/main/resources/DialcodeRedisIndexer.conf +++ /dev/null @@ -1,15 +0,0 @@ -# redis -redis.host=__redis_host__ -redis.port="6379" -redis.connection.max=20 -redis.connection.idle.max=20 -redis.connection.idle.min=10 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.max.pipeline.size="1000" -redis.dialcode.database.index=6 - -cloudStorage.accountName="" -cloudStorage.accountKey="" -cloudStorage.container="dev-data-store" -cloudStorage.dialCodeDataFile="dialcode-data/dial_code.csv" \ No newline at end of file diff --git a/druid/adhoc-jobs/src/main/resources/ESCloudUploader.conf b/druid/adhoc-jobs/src/main/resources/ESCloudUploader.conf deleted file mode 100644 index dd1a482c2b..0000000000 --- a/druid/adhoc-jobs/src/main/resources/ESCloudUploader.conf +++ /dev/null @@ -1,11 +0,0 @@ -elasticsearch.host="localhost" -elasticsearch.port="9200" -elasticsearch.scroll.size="1000" -elasticsearch.query.index="compositesearch" -elasticsearch.query.jsonString="{\"query\":{\"match_all\":{}}}" -outputFilePath="/home/ops/tmp/compositeSearchBatchData" -cloudStorage.container="telemetry-data-store" -cloudStorage.objectKey="druid-content-snapshot/snapshot.txt" -cloudStorage.provider="azure" -cloudStorage.accountName="" -cloudStorage.accountKey="" \ No newline at end of file diff --git a/druid/adhoc-jobs/src/main/resources/ESContentIndexer.conf b/druid/adhoc-jobs/src/main/resources/ESContentIndexer.conf deleted file mode 100644 index a4d7aa897e..0000000000 --- a/druid/adhoc-jobs/src/main/resources/ESContentIndexer.conf +++ /dev/null @@ -1,22 +0,0 @@ -# redis -redis.host=__redis_host__ -redis.port="6379" -redis.connection.max=20 -location.db.redis.key.expiry.seconds=3600 -redis.connection.idle.max=20 -redis.connection.idle.min=10 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.max.pipeline.size="1000" - -# EStoRedis config, all values should be in string format -redis.es.database.index="5" -# to which Redis database (0 - 15) data should be indexed? -elasticsearch.host=__es_host__ -elasticsearch.port="9200" -elasticsearch.scroll.size="1000" -elasticsearch.index.source.keys=["identifier"] # this will be used as key for redis -elasticsearch.index.source.keyDelimiter="" -elasticsearch.query.index="compositesearch" -# query to fetch all 'Live' status contents of objectType 'Content' -elasticsearch.query.jsonString="{\"query\":{\"bool\":{\"must\":[{\"match\":{\"status\":{\"query\":\"Live\"}}},{\"match\":{\"objectType\":{\"query\":\"Content\"}}}]}}}" diff --git a/druid/adhoc-jobs/src/main/resources/ESDialcodeIndexer.conf b/druid/adhoc-jobs/src/main/resources/ESDialcodeIndexer.conf deleted file mode 100644 index c24cf9392a..0000000000 --- a/druid/adhoc-jobs/src/main/resources/ESDialcodeIndexer.conf +++ /dev/null @@ -1,22 +0,0 @@ -# redis -redis.host=__redis_host__ -redis.port="6379" -redis.connection.max=20 -location.db.redis.key.expiry.seconds=3600 -redis.connection.idle.max=20 -redis.connection.idle.min=10 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.max.pipeline.size="1000" - -# EStoRedis config, all values should be in string format - -redis.es.database.index="6" -# to which Redis database (0 - 15) data should be indexed? -elasticsearch.host=__es_host__ -elasticsearch.port="9200" -elasticsearch.scroll.size="1000" -elasticsearch.index.source.keys=["identifier"] # this will be used as key for redis -elasticsearch.index.source.keyDelimiter="" -elasticsearch.query.index="dialcode" -elasticsearch.query.jsonString="{\"query\":{\"match_all\":{}}}" diff --git a/druid/adhoc-jobs/src/main/resources/application.conf b/druid/adhoc-jobs/src/main/resources/application.conf deleted file mode 100644 index 216a34592f..0000000000 --- a/druid/adhoc-jobs/src/main/resources/application.conf +++ /dev/null @@ -1 +0,0 @@ -# do not delete this file \ No newline at end of file diff --git a/druid/adhoc-jobs/src/main/resources/cassandraRedis.conf b/druid/adhoc-jobs/src/main/resources/cassandraRedis.conf deleted file mode 100644 index 3cebe8942b..0000000000 --- a/druid/adhoc-jobs/src/main/resources/cassandraRedis.conf +++ /dev/null @@ -1,18 +0,0 @@ -# redis -redis.host=__redis_host__ -redis.port="6379" -redis.connection.max=20 -location.db.redis.key.expiry.seconds=3600 -redis.connection.idle.max=20 -redis.connection.idle.min=10 -redis.connection.minEvictableIdleTimeSeconds=120 -redis.connection.timeBetweenEvictionRunsSeconds=300 -redis.max.pipeline.size="1000" - - -#CassandraToRedis Config -spark.cassandra.connection.host="localhost" -cassandra.user.keyspace="sunbird" -cassandra.user.table="user" -redis.user.database.index="4" -redis.user.index.source.key="id" # this will be used as key for redis diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CSVToRedisIndexer.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CSVToRedisIndexer.scala deleted file mode 100644 index 4cbf94c790..0000000000 --- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CSVToRedisIndexer.scala +++ /dev/null @@ -1,51 +0,0 @@ -package org.ekstep.analytics.jobs - -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.spark.{SparkConf, SparkContext} -import com.redislabs.provider.redis._ -import org.apache.spark.sql.SparkSession -import org.ekstep.analytics.util.JSONUtils - -object CSVToRedisIndexer { - - private val config: Config = ConfigFactory.load - - def main(args: Array[String]): Unit = { - - val container = config.getString("cloudStorage.container") - val dialCodeDataFile = config.getString("cloudStorage.dialCodeDataFile") - val cloudStorageAccount = config.getString("cloudStorage.accountName") - - val filePath = s"wasbs://$container@$cloudStorageAccount.blob.core.windows.net/$dialCodeDataFile" - val conf = new SparkConf() - .setAppName("SparkCSVtoRedisIndexer") - .setMaster("local[*]") - - // redis settings - .set("spark.redis.host", config.getString("redis.host")) - .set("spark.redis.port", config.getString("redis.port")) - .set("spark.redis.db", config.getString("redis.dialcode.database.index")) - .set("spark.redis.max.pipeline.size", config.getString("redis.max.pipeline.size")) - - val sc = new SparkContext(conf) - val spark = SparkSession.builder.config(conf).getOrCreate() - - spark.sparkContext.hadoopConfiguration.set("fs.azure", "org.apache.hadoop.fs.azure.NativeAzureFileSystem") - spark.sparkContext.hadoopConfiguration.set("fs.azure.account.key." + cloudStorageAccount + ".blob.core.windows.net", config.getString("cloudStorage.accountKey")) - - val data = spark.read.option("header", "true").option("inferSchema", "true").csv(filePath).toJSON.rdd.map(f => f) - val finalData = data.map(f => (getKey(f), f)) - finalData.foreach(f => println(f)) - - spark.sparkContext.toRedisKV( - finalData - ) - } - - def getKey(str: String) ={ - val dataMap = JSONUtils.deserialize[Map[String, AnyRef]](str) - val key = dataMap.get("identifier") - key.mkString - } -} - diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CassandraRedisIndexer.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CassandraRedisIndexer.scala deleted file mode 100644 index 74467b39fc..0000000000 --- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CassandraRedisIndexer.scala +++ /dev/null @@ -1,43 +0,0 @@ -package org.ekstep.analytics.jobs - -import com.typesafe.config.{Config, ConfigFactory} -import org.ekstep.analytics.util.JSONUtils -import com.datastax.spark.connector._ -import com.redislabs.provider.redis._ -import org.apache.spark.SparkConf -import org.apache.spark.SparkContext - - -object CassandraRedisIndexer { - - private val config: Config = ConfigFactory.load - - def main(args: Array[String]): Unit = { - - val conf = new SparkConf() - .setAppName("CassandraToRedisIndexer") - .setMaster("local[*]") - // Cassandra settings - .set("spark.cassandra.connection.host", config.getString("spark.cassandra.connection.host")) - // redis settings - .set("spark.redis.host", config.getString("redis.host")) - .set("spark.redis.port", config.getString("redis.port")) - .set("spark.redis.db", config.getString("redis.user.database.index")) - .set("spark.redis.max.pipeline.size", config.getString("redis.max.pipeline.size")) - - val sc = new SparkContext(conf) - - val userKeyspace = config.getString("cassandra.user.keyspace") - val userTableName = config.getString("cassandra.user.table") - val redisKeyProperty: String = config.getString("redis.user.index.source.key") - - val userData = sc.cassandraTable(userKeyspace, userTableName) - val userMap = userData.map(f => f.toMap) - val mappedData = userMap.map { obj => - (obj.getOrElse(redisKeyProperty, "").asInstanceOf[String] , JSONUtils.serialize(obj)) - } - sc.toRedisKV(mappedData) - } - - -} diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/DeviceProfileUpdateCassandra.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/DeviceProfileUpdateCassandra.scala deleted file mode 100644 index b517e6926e..0000000000 --- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/DeviceProfileUpdateCassandra.scala +++ /dev/null @@ -1,36 +0,0 @@ -package org.ekstep.analytics.jobs -import org.apache.spark.sql.functions.{row_number} -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.spark.sql.expressions.Window -import com.datastax.spark.connector._ -import org.apache.spark.sql.SparkSession -import org.apache.spark.SparkConf -import org.apache.spark.SparkContext -import org.apache.spark.sql._ - -object DeviceProfileUpdateCassandra { - -private val config: Config = ConfigFactory.load - -def main(args: Array[String]): Unit = { - -val conf = new SparkConf() - .setAppName("DeviceProfileUpdateCassandra") - .setMaster("local[*]") - .set("spark.cassandra.connection.host", config.getString("spark.cassandra.connection.host")) - -val sc = new SparkContext(conf) -val spark = SparkSession.builder().master("local[*]").appName("DeviceProfileUpdateCassandra").getOrCreate() -import spark.implicits._ - -val deviceProfileDf = spark.read.format("org.apache.spark.sql.cassandra").options(Map( "table" -> config.getString("cassandra.deviceprofileOldTable") , "keyspace" -> config.getString("cassandra.keyspace") )).load() - -val windowUpdatedDate = Window.partitionBy($"device_id").orderBy($"updated_date".desc) - -val deviceProfileTempDf = deviceProfileDf.filter(deviceProfileDf("state").isNotNull).withColumn("latest_updatedDate", row_number.over(windowUpdatedDate)).where($"latest_updatedDate" === 1).drop("latest_updatedDate","channel") - -deviceProfileTempDf.write.format("org.apache.spark.sql.cassandra").options(Map( "table" -> config.getString("cassandra.deviceprofileNewTable"), "keyspace" -> config.getString("cassandra.keyspace") )).mode(SaveMode.Append).save() - -} - -} diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESCloudUploader.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESCloudUploader.scala deleted file mode 100644 index d5a138b4ec..0000000000 --- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESCloudUploader.scala +++ /dev/null @@ -1,54 +0,0 @@ -package org.ekstep.analytics.jobs - -import java.io.File -import java.util.Date - -import scala.reflect.io.Directory -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.spark.SparkConf -import org.apache.spark.sql.SparkSession -import org.elasticsearch.spark._ -import org.sunbird.cloud.storage.factory.{StorageConfig, StorageServiceFactory} - -object ESCloudUploader { - private val now: Date = new Date() - private val config: Config = ConfigFactory.load - case class Source(timestamp: Long, data: String) - - def main(args: Array[String]): Unit = { - - val index = config.getString("elasticsearch.query.index") - val query = config.getString("elasticsearch.query.jsonString") - val outputFilePath = config.getString("outputFilePath") - - println(s"[$index] query ===> $query") - - require(!index.isEmpty && !query.isEmpty, "require valid inputs! index name and query cannot be empty!") - - val conf = new SparkConf() - .setAppName("SparkEStoJSON") - .setMaster("local[*]") - // Elasticsearch settings - .set("es.nodes", config.getString("elasticsearch.host")) - .set("es.port", config.getString("elasticsearch.port")) - .set("es.scroll.size", config.getString("elasticsearch.scroll.size")) - .set("es.query", query) - - val sparkSession = SparkSession.builder.config(conf).getOrCreate - val sc = sparkSession.sparkContext - - println(s"deleting output folder if exist! : $outputFilePath") - val directory = new Directory(new File(outputFilePath)) - if (directory.exists) directory.deleteRecursively() - - sc.esJsonRDD(index).map(data => s"""{ "timestamp": ${now.getTime}, "data": ${data._2} }""") - .coalesce(1) - .saveAsTextFile(outputFilePath) - - // backup the output file to cloud - val storageService = StorageServiceFactory.getStorageService(StorageConfig(config.getString("cloudStorage.provider"), config.getString("cloudStorage.accountName"), config.getString("cloudStorage.accountKey"))) - storageService.upload(config.getString("cloudStorage.container"), outputFilePath + "/part-00000", config.getString("cloudStorage.objectKey"), isDirectory = Option(false)) - println("successfully backed up file to cloud!") - System.exit(0) - } -} diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESRedisIndexer.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESRedisIndexer.scala deleted file mode 100644 index 48e4b4a607..0000000000 --- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESRedisIndexer.scala +++ /dev/null @@ -1,52 +0,0 @@ -package org.ekstep.analytics.jobs - -import com.redislabs.provider.redis._ -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.spark.{SparkConf, SparkContext} -import org.ekstep.analytics.util.JSONUtils -import org.elasticsearch.spark._ -import scala.collection.JavaConversions._ -import scala.collection.Map - -object ESRedisIndexer { - - private val config: Config = ConfigFactory.load - - def main(args: Array[String]): Unit = { - - val index = config.getString("elasticsearch.query.index") - val query = config.getString("elasticsearch.query.jsonString") - - println(s"[$index] query ===> $query") - - require(!index.isEmpty && !query.isEmpty, "require valid inputs! index name and query cannot be empty!") - - val conf = new SparkConf() - .setAppName("SparkEStoRedisIndexer") - .setMaster("local[*]") - // Elasticsearch settings - .set("es.nodes", config.getString("elasticsearch.host")) - .set("es.port", config.getString("elasticsearch.port")) - .set("es.scroll.size", config.getString("elasticsearch.scroll.size")) - .set("es.query", query) - // redis settings - .set("spark.redis.host", config.getString("redis.host")) - .set("spark.redis.port", config.getString("redis.port")) - .set("spark.redis.db", config.getString("redis.es.database.index")) - .set("spark.redis.max.pipeline.size", config.getString("redis.max.pipeline.size")) - - val sc = new SparkContext(conf) - val keys = config.getStringList("elasticsearch.index.source.keys").toList - val keyDelimiter = config.getString("elasticsearch.index.source.keyDelimiter") - - // todo: log details - def getKey(data: String): String = { - val record = JSONUtils.deserialize[Map[String, AnyRef]](data) - keys.map(value => record(value)).mkString(keyDelimiter) - } - - sc.toRedisKV( - sc.esJsonRDD(index).map(data => (getKey(data._2), data._2)) - ) - } -} diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/util/JSONUtils.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/util/JSONUtils.scala deleted file mode 100644 index 80857aec0e..0000000000 --- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/util/JSONUtils.scala +++ /dev/null @@ -1,49 +0,0 @@ -package org.ekstep.analytics.util - -import java.lang.reflect.{ParameterizedType, Type} -import com.fasterxml.jackson.annotation.JsonInclude.Include -import com.fasterxml.jackson.core.`type`.TypeReference -import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature} -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import org.apache.commons.text.StringEscapeUtils - -object JSONUtils { - - @transient val mapper = new ObjectMapper(); - mapper.registerModule(DefaultScalaModule); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - mapper.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false); - mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - mapper.setSerializationInclusion(Include.NON_NULL); - - @throws(classOf[Exception]) - def serialize(obj: AnyRef): String = { - mapper.writeValueAsString(obj); - } - - @throws(classOf[Exception]) - def deserialize[T: Manifest](value: String): T = mapper.readValue(value, typeReference[T]); - - @throws(classOf[Exception]) - def unescapeJSON(string: String): String = { - StringEscapeUtils.unescapeJava(string) - } - - private[this] def typeReference[T: Manifest] = new TypeReference[T] { - override def getType = typeFromManifest(manifest[T]) - } - - - private[this] def typeFromManifest(m: Manifest[_]): Type = { - if (m.typeArguments.isEmpty) { m.runtimeClass } - // $COVERAGE-OFF$Disabling scoverage as this code is impossible to test - else new ParameterizedType { - def getRawType = m.runtimeClass - def getActualTypeArguments = m.typeArguments.map(typeFromManifest).toArray - def getOwnerType = null - } - // $COVERAGE-ON$ - } - -} \ No newline at end of file diff --git a/druid/adhoc-jobs/src/scripts/DeviceProfileScripts.sh b/druid/adhoc-jobs/src/scripts/DeviceProfileScripts.sh deleted file mode 100644 index 3e1ab8bc69..0000000000 --- a/druid/adhoc-jobs/src/scripts/DeviceProfileScripts.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -home=`echo $HOME` -jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar" -jobConfPath="${home}/adhoc-jobs-1.0/resources/DeviceProfile.conf" - -spark/bin/spark-submit \ ---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \ ---class org.ekstep.analytics.jobs.DeviceProfileUpdateCassandra \ -${jobJarPath} diff --git a/druid/adhoc-jobs/src/scripts/DruidContentIndexer.sh b/druid/adhoc-jobs/src/scripts/DruidContentIndexer.sh deleted file mode 100644 index d43f6a6d2b..0000000000 --- a/druid/adhoc-jobs/src/scripts/DruidContentIndexer.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -e -# Configurations -druidCoordinatorIP="" -dataSourceName="content-model-snapshot" -today=`date +%Y-%m-%d` -interval="2019-01-01_$today" -now=`date +%Y-%m-%d-%s` -home=`echo $HOME` -ingestionSpecFilePath="${home}/adhoc-jobs-1.0/druid_models/content_index_batch.json" -jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar" -jobConfPath="${home}/adhoc-jobs-1.0/resources/ESCloudUploader.conf" - -# get list of segments from content model snapshot datasource -bkpIFS="$IFS" -segmentIds=$(curl -X 'GET' -H 'Content-Type:application/json' http://${druidCoordinatorIP}:8081/druid/coordinator/v1/metadata/datasources/${dataSourceName}/segments) -IFS=',][' -read -r -a array <<< ${segmentIds} -IFS="$bkpIFS" - -# start the spark script to fetch Elasticsearch data and write it to a file and upload to cloud -spark/bin/spark-submit \ ---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \ ---class org.ekstep.analytics.jobs.ESCloudUploader \ -${jobJarPath} - -printf "\n>>> submit ingestion task to Druid!\n" - -# submit task to start batch ingestion -curl -X 'POST' -H 'Content-Type:application/json' -d @${ingestionSpecFilePath} http://${druidCoordinatorIP}:8090/druid/indexer/v1/task - - -for index in "${!array[@]}" -do - val=( $(eval echo ${array[index]}) ) - printf "\n>>> Disabling segment id: $val \n" - # disable older segments - curl -X 'DELETE' -H 'Content-Type:application/json' http://${druidCoordinatorIP}:8081/druid/coordinator/v1/datasources/${dataSourceName}/segments/$val -done - -printf "\n>>> Deleting segments from interval $interval \n" - -# delete older segments -curl -X 'DELETE' -H 'Content-Type:application/json' http://${druidCoordinatorIP}:8081/druid/coordinator/v1/datasources/${dataSourceName}/intervals/${interval} - -printf "\n>>> success!\n" - - diff --git a/druid/adhoc-jobs/src/scripts/RedisContentIndexer.sh b/druid/adhoc-jobs/src/scripts/RedisContentIndexer.sh deleted file mode 100644 index 0e350bcd1b..0000000000 --- a/druid/adhoc-jobs/src/scripts/RedisContentIndexer.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -home=`echo $HOME` -jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar" -jobConfPath="${home}/adhoc-jobs-1.0/resources/ESContentIndexer.conf" - -spark/bin/spark-submit \ ---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \ ---class org.ekstep.analytics.jobs.ESRedisIndexer \ -${jobJarPath} diff --git a/druid/adhoc-jobs/src/scripts/RedisDialcodeIndexer.sh b/druid/adhoc-jobs/src/scripts/RedisDialcodeIndexer.sh deleted file mode 100755 index b22cd90814..0000000000 --- a/druid/adhoc-jobs/src/scripts/RedisDialcodeIndexer.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -home=`echo $HOME` -jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar" -jobConfPath="${home}/adhoc-jobs-1.0/resources/DialcodeRedisIndexer.conf" - -spark/bin/spark-submit \ ---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \ ---class org.ekstep.analytics.jobs.CSVToRedisIndexer \ -${jobJarPath} diff --git a/druid/adhoc-jobs/src/scripts/RedisUserDataIndexer.sh b/druid/adhoc-jobs/src/scripts/RedisUserDataIndexer.sh deleted file mode 100644 index 87a62fca01..0000000000 --- a/druid/adhoc-jobs/src/scripts/RedisUserDataIndexer.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -home=`echo $HOME` -jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar" -jobConfPath="${home}/adhoc-jobs-1.0/resources/cassandraRedis.conf" - -spark/bin/spark-submit \ ---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \ ---class org.ekstep.analytics.jobs.CassandraRedisIndexer \ -${jobJarPath} diff --git a/druid/adhoc-jobs/src/test/resources/application.conf b/druid/adhoc-jobs/src/test/resources/application.conf deleted file mode 100644 index fcdc438309..0000000000 --- a/druid/adhoc-jobs/src/test/resources/application.conf +++ /dev/null @@ -1,11 +0,0 @@ -elasticsearch.host="localhost" -elasticsearch.port="9200" -elasticsearch.scroll.size="1000" -elasticsearch.query.index="compositesearch" -elasticsearch.query.jsonString="{\"query\":{\"match_all\":{}}}" -outputFilePath="" -cloudStorage.container="telemetry-data-store" -cloudStorage.objectKey="druid-content-snapshot/snapshot.txt" -cloudStorage.provider="azure" -cloudStorage.accountName="" -cloudStorage.accountKey="" \ No newline at end of file diff --git a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestCassandraRedisIndexer.scala b/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestCassandraRedisIndexer.scala deleted file mode 100644 index 679fcc6064..0000000000 --- a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestCassandraRedisIndexer.scala +++ /dev/null @@ -1,11 +0,0 @@ -package org.ekstep.analytics.jobs - -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} - -class TestCassandraRedisIndexer extends FlatSpec with Matchers with BeforeAndAfterAll { - - "cassandra indexer" should "index user data to redis" in { - println(s"======> Starting Redis indexing job <=======") - //CassandraToRedisIndexer.main(Array()) - } -} \ No newline at end of file diff --git a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESCloudUploader.scala b/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESCloudUploader.scala deleted file mode 100644 index ef443de3ee..0000000000 --- a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESCloudUploader.scala +++ /dev/null @@ -1,9 +0,0 @@ -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} - -class TestESCloudUploader extends FlatSpec with Matchers with BeforeAndAfterAll { - - "main method" should "run" in { - println(s"======> Starting Redis indexing job <=======") - //ESToCloudUploader.main(Array()) - } -} diff --git a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESRedisIndexer.scala b/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESRedisIndexer.scala deleted file mode 100644 index 58c07dbccf..0000000000 --- a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESRedisIndexer.scala +++ /dev/null @@ -1,11 +0,0 @@ -package org.ekstep.analytics.jobs - -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} - -class TestESRedisIndexer extends FlatSpec with Matchers with BeforeAndAfterAll { - - "main method" should "run" in { - println(s"======> Starting Redis indexing job <=======") - //ESToRedisIndexer.main(Array()) - } -} diff --git a/druid/ingestion-spec/content_index_batch.json b/druid/ingestion-spec/content_index_batch.json index 9a29c0b26c..c061fa0c5e 100644 --- a/druid/ingestion-spec/content_index_batch.json +++ b/druid/ingestion-spec/content_index_batch.json @@ -783,7 +783,7 @@ "path": "/druid-content-snapshot/snapshot.txt" } ], - "fetchTimeout": 120000 + "fetchTimeout": 300000 } }, "tuningConfig":{ @@ -794,4 +794,4 @@ "logParseExceptions": true } } -} \ No newline at end of file +} diff --git a/druid/ingestion-spec/rollups/ml_project_index_rollup.json b/druid/ingestion-spec/rollups/ml_project_index_rollup.json new file mode 100644 index 0000000000..57f23ec9e4 --- /dev/null +++ b/druid/ingestion-spec/rollups/ml_project_index_rollup.json @@ -0,0 +1,258 @@ +{ + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "ml-project-rollup", + "parserSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { + "type": "path", + "name": "project_title", + "expr": "$.solutionInformation.name" + }, + { + "type": "path", + "name": "area_of_improvement", + "expr": "$.categories[*].name" + }, + { + "type": "path", + "name": "tasks", + "expr": "$.tasks[*].name" + }, + { + "type": "path", + "name": "tasks_status", + "expr": "$.tasks[*].status" + }, + { + "type": "path", + "name": "designation", + "expr": "$.userRoleInformation.role" + }, + { + "type": "path", + "name": "task_evidence", + "expr": "$.tasks[*].attachments[*].sourcePath" + }, + { + "type": "path", + "name": "task_id", + "expr": "$.tasks[*]._id" + }, + { + "type": "path", + "name": "program_id", + "expr": "$.programId" + }, + { + "type": "path", + "name": "status_of_project", + "expr": "$.status" + }, + { + "type": "path", + "name": "program_name", + "expr": "$.programInformation.name" + }, + { + "type": "path", + "name": "project_updated_date", + "expr": "$.updatedAt" + }, + { + "type": "path", + "name": "createdBy", + "expr": "$.userid" + }, + { + "type": "path", + "name": "program_externalId", + "expr": "$.programExternalId" + }, + { + "type": "path", + "name": "private_program", + "expr": "$.isAPrivateProgram" + }, + { + "type": "path", + "name": "project_terms_and_condition", + "expr": "$.hasAcceptedTAndC" + }, + { + "type": "path", + "name": "solution_id", + "expr": "$.solutionInformation._id" + }, + { + "type": "path", + "name": "state_externalId", + "expr": "$.userRoleInformation.state" + }, + { + "type": "path", + "name": "block_externalId", + "expr": "$.userRoleInformation.block" + }, + { + "type": "path", + "name": "district_externalId", + "expr": "$.userRoleInformation.district" + }, + { + "type": "path", + "name": "cluster_externalId", + "expr": "$.userRoleInformation.cluster" + }, + { + "type": "path", + "name": "school_externalId", + "expr": "$.userRoleInformation.school" + } + ] + } + }, + "timestampSpec": { + "column": "project_updated_date", + "format": "iso", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "project_title" + }, + { + "type": "string", + "name": "area_of_improvement" + }, + { + "type": "string", + "name": "tasks" + }, + { + "type": "string", + "name": "tasks_status" + }, + { + "type": "string", + "name": "designation" + }, + { + "type": "string", + "name": "task_evidence" + }, + { + "type": "string", + "name": "task_id" + }, + { + "type": "string", + "name": "status_of_project" + }, + { + "type": "string", + "name": "program_name" + }, + { + "type": "string", + "name": "createdBy" + }, + { + "type": "string", + "name": "program_externalId" + }, + { + "type": "string", + "name": "private_program" + }, + { + "type": "string", + "name": "project_terms_and_condition" + }, + { + "type": "string", + "name": "state_externalId" + }, + { + "type": "string", + "name": "block_externalId" + }, + { + "type": "string", + "name": "district_externalId" + }, + { + "type": "string", + "name": "cluster_externalId" + } + ], + "dimensionExclusions": [] + }, + "metricsSpec": [ + { + "type": "HLLSketchBuild", + "name": "count_of_unique_programs", + "fieldName": "program_id" + }, + { + "type": "HLLSketchBuild", + "name": "count_of_createdBy", + "fieldName": "createdBy" + }, + { + "type": "HLLSketchBuild", + "name": "count_of_solution_id", + "fieldName": "solution_id" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "HOUR", + "rollup": true + }, + "transformSpec": { + "filter": null, + "transforms": [ + { + "type": "expression", + "name": "task_evidence_status", + "expression": "if(array_length(\"task_evidence\") > 0, 'true', 'false')" + }, + { + "type": "expression", + "name": "parent_channel", + "expression": "upper('SHIKSHALOKAM')" + }, + { + "type": "expression", + "name": "project_created_type", + "expression": "if((NULL != \"project_created_type\"), 'project imported from library', 'user created project')" + } + ] + } + }, + "ioConfig": { + "topic": "{{env}}.ml.project", + "replicas": 1, + "taskCount": "{{rollup_ml_project_taskcount}}", + "taskDuration": "PT14400S", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "useEarliestOffset": true, + "completionTimeout": "PT1800S" + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "maxRowsPerSegment": 5000000 + } + } + } diff --git a/druid/ingestion-spec/rollups/tpd_index_hourly_syncts.json b/druid/ingestion-spec/rollups/tpd_index_hourly_syncts.json new file mode 100644 index 0000000000..6ab6767584 --- /dev/null +++ b/druid/ingestion-spec/rollups/tpd_index_hourly_syncts.json @@ -0,0 +1,168 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "tpd-hourly-rollup-syncts", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "flattenSpec": { + "useFieldDiscovery": false, + "fields": [ + { + "type": "root", + "name": "eid" + }, + { + "type": "path", + "name": "collection_type", + "expr": "$.collectiondata.contenttype" + }, + { + "type": "path", + "name": "context_env", + "expr": "$.context.env" + }, + { + "type": "path", + "name": "edata_id", + "expr": "$.edata.id" + }, + { + "type": "path", + "name": "edata_mode", + "expr": "$.edata.mode" + }, + { + "type": "path", + "name": "edata_pageid", + "expr": "$.edata.pageid" + }, + { + "type": "path", + "name": "edata_type", + "expr": "$.edata.type" + }, + { + "type": "path", + "name": "user_signin_type", + "expr": "$.userdata.usersignintype" + }, + { + "type": "path", + "name": "actor_id", + "expr": "$.actor.id" + }, + { + "type": "path", + "name": "context_did", + "expr": "$.context.did" + } + ] + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "eid" + }, + { + "type": "string", + "name": "context_env" + }, + { + "type": "string", + "name": "user_signin_type" + }, + { + "type": "string", + "name": "collection_type" + }, + { + "type": "string", + "name": "edata_id" + }, + { + "type": "string", + "name": "edata_mode" + }, + { + "type": "string", + "name": "edata_pageid" + }, + { + "type": "string", + "name": "edata_type" + }, + { + "type":"string", + "name": "first_time_user" + } + ] + }, + "timestampSpec": { + "column": "syncts", + "format": "auto" + } + } + }, + "transformSpec": { + "transforms": [ + { + "type": "expression", + "name": "first_time_user ", + "expression": "if(like(\"context_cdata_type\",'FirstTimeUser') && like(\"context_cdata_id\",'true') , 'true','false')" + }, + { + "type": "expression", + "name": "is_playerstart_event", + "expression": "if((like(\"context_pdata_pid\",'%contentplayer%'), if ( \"eid\" == 'START', 'true','false') ,'true')" + } + ], + "filter": { + "type": "selector", + "dimension": "is_playerstart_event", + "value": "true" + } + }, + "metricsSpec": [ + { + "name": "total_count", + "type": "count" + }, + { + "type": "HLLSketchBuild", + "name": "unique_devices", + "fieldName": "context_did" + }, + { + "type": "HLLSketchBuild", + "name": "unique_users", + "fieldName": "actor_id" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "hour", + "queryGranularity": "hour", + "rollup": true + } + }, + "ioConfig": { + "topic": "{{env}}.druid.events.telemetry", + "consumerProperties": { + "bootstrap.servers": "{{kafka_brokers}}" + }, + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT7200S", + "completionTimeout" : "PT3600S", + "useEarliestOffset": true + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false, + "logParseExceptions": true, + "maxSavedParseExceptions": 10 + } +} diff --git a/kubernetes/ansible/bootstrap_minimal.yaml b/kubernetes/ansible/bootstrap_minimal.yaml new file mode 100644 index 0000000000..f9facd6152 --- /dev/null +++ b/kubernetes/ansible/bootstrap_minimal.yaml @@ -0,0 +1,44 @@ +--- +# tasks file for bootstrap-k8s + +- hosts: local + gather_facts: false + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + serial: 1 + vars: + helm_vars: + reloader: + watchGlobally: false + vars_files: + - "{{inventory_dir}}/secrets.yml" + pre_tasks: + - name: Creating tmp file + tempfile: + state: file + suffix: helm + register: helm_file + - name: Creating helm_charts vars file + copy: + content: "{{ helm_vars|to_yaml }}" + dest: "{{ helm_file.path }}" + - name: Creating namespace + shell: "kubectl create namespace {{ item }} " + with_items: + - "flink-{{ env }}" + - "flink-kp-{{ env }}" + - secor + - monitoring + - logging + ignore_errors: yes + + - name: Creating docker secrets + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ core_vault_docker_registry_url }} --docker-username {{ core_vault_docker_registry_user }} --docker-password {{ core_vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + with_items: + - "flink-{{ env }}" + - "flink-kp-{{ env }}" + - secor + when: imagepullsecrets|length > 0 + + - name: Installing reloader for configmaps reload + shell: helm upgrade --install --atomic reloader ../helm_charts/bootstrap/reloader --namespace secor -f "{{ helm_file.path }}" diff --git a/kubernetes/ansible/druid_provision.yml b/kubernetes/ansible/druid_provision.yml new file mode 100644 index 0000000000..a3635f03b5 --- /dev/null +++ b/kubernetes/ansible/druid_provision.yml @@ -0,0 +1,11 @@ +--- +- hosts: local + gather_facts: no + vars: + - cluster_type: "{{ cluster_type }}" + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + roles: + - druid \ No newline at end of file diff --git a/kubernetes/ansible/flink_jobs_stop_start.yml b/kubernetes/ansible/flink_jobs_stop_start.yml new file mode 100644 index 0000000000..639ca1529c --- /dev/null +++ b/kubernetes/ansible/flink_jobs_stop_start.yml @@ -0,0 +1,43 @@ +--- +- hosts: local + gather_facts: no + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + tasks: + - name: scale down the taskmanager to zero + shell: kubectl scale deployment {{ item }}-taskmanager --replicas=0 -n {{ flink_namespace }} + with_items: + - "{{ job_names.split(',')|list }}" + tags: + - scaledown_taskmanager + + - name: scale down the jobmanager + shell: kubectl get job {{ item }}-jobmanager -o json -n {{ flink_namespace }} | jq 'del(.spec.selector)' | jq 'del(.spec.template.metadata.labels."controller-uid")' |jq '.spec.parallelism = 0' | kubectl replace --force -f - + with_items: + - "{{ job_names.split(',')|list }}" + tags: + - scaledown_jobmanager + + - name: scale up the jobmanager + shell: kubectl get job {{ item }}-jobmanager -o json -n {{ flink_namespace }} | jq 'del(.spec.selector)' | jq 'del(.spec.template.metadata.labels."controller-uid")' |jq '.spec.parallelism = 1' | kubectl replace --force -f - + with_items: + - "{{ job_names.split(',')|list }}" + tags: + - scaleup_jobmanager + + - name: Scale up taskmanager + shell: kubectl scale deployment {{ item }}-taskmanager --replicas={{ flink_job_names['%s'|format(item)].replica }} -n {{ flink_namespace }} + with_items: + - "{{ job_names.split(',')|list }}" + tags: + - scaleup_taskmanager + + - name: rollout the taskmanager + shell: kubectl rollout restart deployment {{ item }}-taskmanager -n {{ flink_namespace }} + with_items: + - "{{ job_names.split(',')|list }}" + tags: + - scaleup_taskmanager + diff --git a/kubernetes/ansible/roles/druid/defaults/main.yml b/kubernetes/ansible/roles/druid/defaults/main.yml new file mode 100644 index 0000000000..84d1511a04 --- /dev/null +++ b/kubernetes/ansible/roles/druid/defaults/main.yml @@ -0,0 +1,269 @@ +druid_image: "apache/druid:0.21.1" +druid_namespace: "druid-{{ cluster_type }}" + +# Druid Template variables +druid_monitoring: false + +# postgres: +# db_url: "{{ groups['postgres'][0] }}" +# db_port: 5432 +# db_admin_user: analytics +# db_admin_password: "{{dp_vault_pgdb_admin_password}}" +# druid: +# db_name: "druid-{{ cluster_type }}" +# db_username: "druid" +# db_password: "{{dp_vault_druid_postgress_pass}}" + +# druid_user: druid +druid_directory: "/opt/druid" + +###################### Common Varibles ########################### + +druid_default_tmp_dir: "/var/tmp" +druid_log_dir: "/var/log/druid/" +druid_gc_logdir: "/var/log/druid/crash_logs" +druid_crash_logdir: "/var/log/druid/crash_logs" + + +#Writing request query logs to file +# druid_request_logging_type: "file" +druid_request_logging_type: "composing" + +#Druid Extensions + +druid_extensions_loadList : '"druid-azure-extensions", "postgresql-metadata-storage", "druid-kafka-indexing-service"' +druid_community_extensions: + - "graphite-emitter" +druid_emitter_logging_logLevel: INFO + +# End of druid_extensions + +druid_indexing_logs_type: "azure" +druid_indexing_log_dir: "/var/druid/indexing-logs" +druid_indexing_storage_type: "metadata" +druid_indexing_task_basedir: "/var/task" + +druid_common_monitors: '"com.metamx.metrics.JvmMonitor","org.apache.druid.java.util.metrics.JvmMonitor"' +# druid_common_emitters: '"logging","graphite"' +druid_common_emitters: '"logging"' + +druid_graphite_prefix: "druid.metric" +druid_whitelist_filepath: "{{ druid_directory }}/whitelist" + +#End of Common variables + +######################## Coordinator Varibles ######################### + +druid_coordinator_port: 8081 +druid_coordinator_service : druid/coordinator + +druid_coordinator_tmp_dir: "{{ druid_default_tmp_dir }}" +druid_coordinator_gc_logfile: "{{ druid_crash_logdir }}/gc.log" +druid_coordinator_heap_dump_file : "{{ druid_crash_logdir }}/coordinator.hprof" + +####################### Overlord Varibles ############################# + +druid_overlord_port: 8090 +druid_overlord_service : druid/overlord + +druid_overlord_heap_size: 256m +druid_overlord_tmp_dir: "{{ druid_default_tmp_dir }}" +druid_overlord_gc_logfile: "{{ druid_crash_logdir }}/gc_overlord.log" +druid_overlord_heap_dump_file : "{{ druid_crash_logdir }}/overlord.hprof" + +######################### Broker Varibles ############################# + +druid_broker_port: 8082 +druid_broker_service: druid/broker + +druid_broker_max_direct_size: 800m +druid_broker_tmp_dir: /var/tmp +druid_broker_gc_logdir: "{{ druid_crash_logdir }}/gc.log" +druid_broker_heap_dump_file : "{{ druid_crash_logdir }}/broker.hprof" + +####################### Historical Varibles ########################### + +druid_historical_port: 8084 +druid_historical_service: druid/historical + +druid_historical_tmp_dir: "{{ druid_default_tmp_dir }}" +druid_historical_gc_logfile: "{{ druid_crash_logdir }}/historical.gc.log" +druid_historical_heap_dump_file: "{{ druid_crash_logdir }}/historical.hprof" + +druid_historical_monitoring_monitors: '"org.apache.druid.server.metrics.HistoricalMetricsMonitor","com.metamx.metrics.JvmMonitor"' + +# druid_historical_segmentcache_path: "/var/segments/store" +druid_historical_cache_size: 256MiB +druid_historical_cache_expiry: 3600000 + +druid_historical_cache_unCacheable: '"select", "scan"' + +##################### Middlemanager Varibles ########################## + +druid_middlemanager_port: 8091 +druid_middlemanager_service: druid/middlemanager + +druid_middlemanager_tmp_dir: "{{ druid_default_tmp_dir }}" +druid_middlemanager_gc_logdir: "{{ druid_crash_logdir }}/gc.log" + +druid_middlemanager_peon_segmentcache_path: "/var/segment_cache" +druid_azure_container_name: "telemetry-data-store" +druid_mm_heap_dump_file : "{{ druid_crash_logdir }}/middlemanager.hprof" + +##################### Middlemanager Varibles ########################## + +druid_router_service: druid/router +druid_router_plaintextPort: 8888 + +enable_druid_sql: true +persistent_storage_class: managed-premium + +druid_configs: + raw: + #Druid Postgres Details + druid_postgres_db: "druid-{{ cluster_type }}" + druid_postgres_host: "{{ postgres.db_url }}" + druid_postgres_port: "{{ postgres.db_port }}" + druid_postgres_user: "druid@{{ postgres.db_url }}" + druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}" + #Druid Azure Details + druid_storge_type: "azure" + azure_account_name: "{{ sunbird_druid_storage_account_name }}" + azure_storage_secret: "{{ sunbird_druid_storage_account_key }}" + azure_container: "{{ druid_azure_container_name }}" + #Logging the indexing logs to azure + druid_log_azure_container: "{{ druid_azure_container_name }}" + druid_log_azure_folder: "druidlogs" + druid_coordinator_heap_size: 128M + druid_coordinator_period: PT30S + druid_coordinator_startDelay: PT30S + druid_coordinator_balance_strategy: diskNormalized + druid_coordinator_monitoring_monitor_enabled: false + druid_coordinator_replicas: 1 + #Druid overlord node configuration + druid_overlord_heap_size: 256M + druid_overlord_monitoring_monitor_enabled: false + druid_overlord_replicas: 1 + #Druid broker node configuration + druid_broker_min_heap_size: 128M + druid_broker_max_heap_size: 128M + druid_broker_max_direct_size: 800M + druid_broker_http_numConnections: 5 + druid_broker_server_http_numThread: 25 + druid_broker_processing_bufferBytes: 128MiB + druid_broker_processing_threads: 2 + druid_broker_monitor_enabled: false + druid_broker_replicas: 1 + #Druid historical node configuration + #druid_historical_min_heap_size: 1048m + #druid_historical_max_heap_size: 1048m + druid_historical_min_heap_size: 512M + druid_historical_max_heap_size: 512M + druid_historical_max_direct_size: 800M + druid_historical_persistent_volume_size: 20G + druid_historical_http_numConnections: 5 + druid_historical_server_http_numThread: 25 + druid_historical_processing_bufferBytes: 128MiB + druid_historical_processing_threads: 2 + druid_historical_enable_cache: false + #druid_historical_segmentcache_size: 10000000000 + druid_historical_segmentcache_size: 2000000000 + druid_historical_server_maxsize: 1000000000 + druid_historical_processing_num_merge_buffers: 2 + druid_query_ondiskstorage_enabled: true + druid_historical_maxMergingDictionarySize: 100000000 + druid_historical_segmentcache_numloadingthreads: 4 + druid_historical_segmentcache_path: "/var/segments/store" + druid_query_groupBy_maxOnDiskStorage: 1073741824 + druid_historical_monitoring_monitor_enabled: false + druid_historical_replicas: 1 + #Druid middlemanager configuration + druid_middlemanager_heap_size: 128M + druid_middlemanager_worker_cap: 2 + druid_middlemanager_peon_heap_size: 256M + druid_middlemanager_persistent_volume_size: 20G + druid_mm_java_opts_array: "-server -Xms512m -Xmx512m -XX:+UseG1GC -XX:MaxGCPauseMillis=100" + druid_middlemanager_peon_server_http_numThread: 25 + druid_middlemanager_peon_processing_bufferBytes: 25MiB + druid_middlemanager_peon_processing_threads: 2 + druid_middlemanager_peon_server_maxsize: 0 + druid_middlemanager_monitoring_monitor_enabled: false + druid_indexing_queue_startDelay: PT30S + druid_middlemanager_replicas: 1 + #Druid Router configurations + druid_router_heap_size: 512M + druid_router_http_numConnections: 50 + druid_router_http_readTimeout: PT5M + druid_router_http_numMaxThreads: 100 + druid_server_http_numThreads: 100 + druid_router_managementProxy_enabled: true + druid_router_replicas: 1 + rollup: + #Druid Postgres Details + druid_postgres_db: "druid-{{ cluster_type }}" + druid_postgres_host: "{{ postgres.db_url }}" + druid_postgres_port: "{{ postgres.db_port }}" + druid_postgres_user: "druid@{{ postgres.db_url }}" + #Druid Azure Details + druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}" + azure_account_name: "{{ sunbird_druid_storage_account_name }}" + azure_storage_secret: "{{ sunbird_druid_storage_account_key }}" + azure_container: "{{ druid_azure_container_name }}" + #Logging the indexing logs to azure + druid_log_azure_container: "{{ druid_azure_container_name }}" + druid_log_azure_folder: "druidlogs" + druid_coordinator_heap_size: 128m + druid_coordinator_period: PT30S + druid_coordinator_startDelay: PT30S + druid_coordinator_balance_strategy: diskNormalized + druid_coordinator_monitoring_monitor_enabled: false + druid_coordinator_replicas: 1 + #Druid overlord node configuration + druid_overlord_heap_size: 256M + druid_overlord_monitoring_monitor_enabled: false + druid_overlord_replicas: 1 + #Druid broker node configuration + druid_broker_min_heap_size: 128m + druid_broker_max_heap_size: 128m + druid_broker_max_direct_size: 700m + druid_broker_http_numConnections: 5 + druid_broker_server_http_numThread: 25 + druid_broker_processing_bufferBytes: 134217728 + druid_broker_processing_threads: 2 + druid_broker_monitor_enabled: false + druid_broker_replicas: 1 + #Druid historical node configuration + druid_historical_min_heap_size: 1048m + druid_historical_max_heap_size: 1048m + druid_historical_max_direct_size: 800m + druid_historical_http_numConnections: 5 + druid_historical_server_http_numThread: 25 + druid_historical_processing_bufferBytes: 134217728 + druid_historical_processing_threads: 2 + druid_historical_enable_cache: false + druid_historical_segmentcache_size: 2000000000 + druid_historical_server_maxsize: 10000000000 + druid_historical_processing_num_merge_buffers: 2 + druid_query_ondiskstorage_enabled: false + druid_historical_segmentcache_numloadingthreads: 4 + druid_historical_segmentcache_path: "/var/segments/store" + druid_historical_replicas: 1 + #Druid middlemanager configuration + druid_middlemanager_heap_size: 128m + druid_middlemanager_worker_cap: 4 + druid_middlemanager_peon_heap_size: 512m + druid_mm_java_opts_array: "-server -Xms256m -Xmx256m -XX:+UseG1GC -XX:MaxGCPauseMillis=100" + druid_middlemanager_peon_server_http_numThread: 25 + druid_middlemanager_peon_processing_bufferBytes: 25000000 + druid_middlemanager_peon_processing_threads: 2 + druid_middlemanager_peon_server_maxsize: 0 + druid_indexing_queue_startDelay: PT30S + druid_middlemanager_replicas: 1 + #Druid router configuration + druid_router_heap_size: 1g + druid_router_http_numConnections: 50 + druid_router_http_readTimeout: PT5M + druid_router_http_numMaxThreads: 100 + druid_server_http_numThreads: 100 + druid_router_managementProxy_enabled: true + druid_router_replicas: 1 \ No newline at end of file diff --git a/kubernetes/ansible/roles/druid/tasks/main.yml b/kubernetes/ansible/roles/druid/tasks/main.yml new file mode 100644 index 0000000000..2ff7190516 --- /dev/null +++ b/kubernetes/ansible/roles/druid/tasks/main.yml @@ -0,0 +1,37 @@ +- name: template values.yaml file + template: + src: "{{ chart_base_path }}/druid-cluster/values.j2" + dest: "{{ chart_base_path }}/druid-cluster/values.yaml" + +- name: Create druid database if not exists + postgresql_db: name="{{ postgres.druid.db_name }}" \ + login_host="{{ postgres.db_url }}" \ + port="{{ postgres.db_port }}" \ + login_user="{{ postgres.db_admin_user }}" \ + login_password="{{ postgres.db_admin_password }}" \ + encoding='UTF-8' \ + state=present + +- name: Create druid user if not exists + postgresql_user: name="druid" \ + password="{{ dp_vault_druid_postgress_pass }}" \ + no_password_changes=true \ + priv=ALL \ + state=present \ + login_host="{{ postgres.db_url }}" \ + port="{{ postgres.db_port }}" \ + login_user="{{ postgres.db_admin_user }}" \ + login_password="{{ postgres.db_admin_password }}" \ + db="{{ postgres.druid.db_name }}" + +- name: Install druid operator + shell: helm upgrade --install -n {{ druid_namespace }} --set env.WATCH_NAMESPACE="{{ druid_namespace }}" druid-operator {{ chart_base_path }}/druid-operator + +- name: Install nginx-controller + shell: helm upgrade --install -n {{ druid_namespace }} druid-{{ cluster_type }} {{ chart_base_path }}/nginx-ingress --skip-crds + +- name: Install dependencies + shell: helm dependency update {{ chart_base_path }}/druid-cluster + +- name: Install druid cluster + shell: helm upgrade --install -n {{ druid_namespace }} druid-cluster {{ chart_base_path }}/druid-cluster \ No newline at end of file diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index c2e52f22dd..848847425a 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -1,6 +1,6 @@ -flink_namespace: flink-dev -imagepullsecrets: sunbird-registry-secret - +flink_namespace: "flink-{{ env }}" +imagepullsecrets: "{{ env }}-registry-secret" +service_monitor_enabled: true ### Job manager related vars jobmanager_rpc_port: 6123 jobmanager_blob_port: 6124 @@ -30,65 +30,108 @@ taskmanager_heap_memory: 1024 ### base-config related vars postgres_max_connections: 2 -azure_account: "{{ sunbird_private_storage_account_name }}" -azure_secret: "{{ sunbird_private_storage_account_key }}" -flink_container_name: dev-data-store checkpoint_interval: 60000 -restart_attempts: 3 +checkpoint_pause_between_seconds: 5000 +checkpoint_compression_enabled: true +restart_attempts: 5 restart_delay: 30000 # in milli-seconds producer_max_request_size: 1572864 +producer_batch_size: 98304 +producer_linger_ms: 10 +producer_compression: none + redis_timeout: 30000 device_profile_table: "{{ env }}_device_profile" ### Extractor job related vars extractor_event_max_size: 1048576 # Max is only 1MB extractor_max_request_size: 5242880 -extractor_consumer_parallelism: 1 -dedup_parallelism: 1 -extraction_parallelism: 1 -redactor_parallelism: 1 +extractor_consumer_parallelism: 2 +extractor_operators_parallelism: 2 telemetry_extractor_key_expiry_seconds: 3600 +### Ingest Raw router job related vars +ingest_router_consumer_parallelism: 2 +ingest_router_operators_parallelism: 2 +raw_router_consumer_parallelism: 2 +raw_router_downstream_parallelism: 2 + ### Pipeline-preprocessor related vars -pipeline_preprocessor_consumer_parallelism: 1 -telemetry_validation_parallelism: 1 -telemetry_router_parallelism: 1 -share_events_flattener_parallelism: 1 -portal_id: dev.sunbird.portal -desktop_id: dev.sunbird.desktop +pipeline_preprocessor_consumer_parallelism: 2 +pipeline_preprocessor_operators_parallelism: 2 +portal_id: "{{ env }}.sunbird.portal" +desktop_id: "{{ env }}.sunbird.desktop" +frac_id: "{{ env }}.FRac.portal" +mdo_id: "{{ env }}.mdo.portal" + pipeline_preprocessor_key_expiry_seconds: 3600 +### cb-preprocessor related vars +cb_preprocessor_consumer_parallelism: 1 +cb_preprocessor_operators_parallelism: 1 +cb_preprocessor_key_expiry_seconds: 3600 + ### De-normalization related vars -denorm_consumer_parallelism: 1 -device_denorm_parallelism: 1 -user_denorm_parallelism: 1 -content_denorm_parallelism: 1 -loc_denorm_parallelism: 1 -dialcode_denorm_parallelism: 1 -denorm_parallelism: 1 -denorm_sink_parallelism: 1 -denorm_summary_dedup_parallelism: 1 -summary_sink_parallelism: 1 +denorm_consumer_parallelism: 2 +telemetry_denorm_operators_parallelism: 2 de_normalization_duplicationstore_key_expiry_seconds: 3600 de_normalization_key_expiry_seconds: 3600 +denorm_window_count: 1 +denorm_window_shards: 1400 + +denorm_secondary_consumer_parallelism: 2 +telemetry_denorm_secondary_operators_parallelism: 2 +denorm_primary_consumer_parallelism: 2 +telemetry_denorm_primary_operators_parallelism: 2 + +redis_meta_content_port: "{{ content_port }}" +redis_meta_device_port: "{{ device_port }}" +redis_meta_user_port: "{{ user_port }}" +redis_meta_dialcode_port: "{{ dialcode_port }}" + +redis_meta_content_host: "{% if metadata2_redis_host is defined %}{{ metadata2_redis_host }}{% else %}{{ redis_host }}{% endif %}" +redis_meta_device_host: "{% if metadata2_redis_host is defined %}{{ metadata2_redis_host }}{% else %}{{ redis_host }}{% endif %}" +redis_meta_user_host: "{% if metadata2_redis_host is defined %}{{ metadata2_redis_host }}{% else %}{{ redis_host }}{% endif %}" +redis_meta_dialcode_host: "{% if metadata2_redis_host is defined %}{{ metadata2_redis_host }}{% else %}{{ redis_host }}{% endif %}" + +denorm_secondary_window_count: 1 +denorm_secondary_window_shards: 1400 +denorm_primary_window_count: 1 +denorm_primary_window_shards: 1400 + +denorm_summary_window_count: 1 +denorm_summary_window_shards: 1400 ### summary-denormalization related vars summary_denorm_consumer_parallelism: 1 -summary_denorm_parallelism: 1 -summary_denorm_sink_parallelism: 1 -summary_denorm_dedup_parallelism: 1 -summary_denorm_summary_sink_parallelism: 1 +summary_denorm_operators_parallelism: 1 summary_denorm_duplication_key_expiry_seconds: 3600 summary_denorm_key_expiry_seconds: 3600 +### rating related vars +rating_consumer_parallelism: 1 +rating_operators_parallelism: 1 +rating_key_expiry_seconds: 3600 +rating_parallelism: 1 + +### cb_course_aggregator related vars +cb_course_aggregator_consumer_parallelism: 1 +cb_course_aggregator_operators_parallelism: 1 +cb_course_aggregator_key_expiry_seconds: 3600 +cb_course_aggregator_parallelism: 1 +cb_course_aggregator_scoresubmit_parallelism: 1 + ### Druid-validator related vars -druid_validator_consumer_parallelism: 1 -validator_parallelism: 1 -router_parallelism: 1 +druid_validator_consumer_parallelism: 2 +druid_validator_operators_parallelism: 2 druid_validator_key_expiry_seconds: 3600 druid_validation_enabled: true druid_deduplication_enabled: true +### error-denormalization related vars +error_denorm_consumer_parallelism: 1 +error_denorm_operators_parallelism: 1 + ### Device-profile-updater related vars deviceprofile_parallelism: 1 device_profile_updater_key_expiry_seconds: 3600 @@ -96,7 +139,7 @@ device_profile_updater_key_expiry_seconds: 3600 ### content-cache-updater dialcode_host: "{{ proto }}://{{ domain_name }}" dialcode_api_url: "{{ dialcode_host }}/{{ dialcode_endpoint }}" -dialcode_api_auth_key: "{{ dp_vault_dialcode_api_auth_key }}" +dialcode_api_auth_key: "{{ sunbird_api_auth_token }}" ### User-cache-updater related vars usercache_updater_parallelism: 1 @@ -105,25 +148,24 @@ middleware_cassandra_keyspace: sunbird middleware_cassandra_user_table: user middleware_cassandra_location_table: location +#(user read api details) +user_read_api_endpoint: "/learner/private/user/v1/read/" +user_read_api_url : "http://{{private_ingressgateway_ip}}{{ user_read_api_endpoint }}" + + ### assessment-aggregator related vars assessaggregator_parallelism: 1 +assessaggregator_consumer_parallelism: 1 +assessaggregator_downstream_parallelism: 1 +assessaggregator_scoreaggregator_parallelism: 1 middleware_cassandra_courses_keyspace: sunbird_courses middleware_cassandra_assessment_aggregator_table: assessment_aggregator middleware_cassandra_assessment_question_type : question +middleware_cassandra_user_enrolments_table: user_enrolments +middleware_cassandra_user_activity_agg_table: user_activity_agg +content_read_api_host: "http://dev.sunbirded.org" +content_read_api_endpoint: "/api/content/v1/read/" -### taskmanager related vars -healthcheck: true -taskmanager_liveness: - livenessProbe: - tcpSocket: - port: 6122 - initialDelaySeconds: 30 - periodSeconds: 60 - -taskmanager_prom_port: 9251 -taskmanager_rpc_port: 6122 -taskmanager_heap_memory: 1024 -producer_max_request_size: 1572864 ### to be removed job_classname: "" @@ -133,27 +175,233 @@ flink_job_names: telemetry-extractor: job_class_name: 'org.sunbird.dp.extractor.task.TelemetryExtractorStreamTask' replica: 2 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 2 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: true + scale_target_value: 500000 + min_replica: 1 + max_replica: 4 pipeline-preprocessor: job_class_name: 'org.sunbird.dp.preprocessor.task.PipelinePreprocessorStreamTask' + replica: 2 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 2 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: true + scale_target_value: 1000000 + min_replica: 1 + max_replica: 4 + cb-preprocessor: + job_class_name: 'org.sunbird.dp.cbpreprocessor.task.CBPreprocessorStreamTask' + replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: true + scale_target_value: 500000 + min_replica: 1 + max_replica: 2 + rating: + job_class_name: 'org.sunbird.dp.rating.task.RatingTask' replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: true + scale_target_value: 500000 + min_replica: 1 + max_replica: 2 de-normalization: job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask' replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 0.3 + mem_requests: 2Gi + scale_enabled: false + de-normalization-v2: + job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask' + replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 0.3 + mem_requests: 2Gi + scale_enabled: false + de-normalization-secondary: + job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask' + replica: 2 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 2 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: true + scale_target_value: 1000000 + min_replica: 1 + max_replica: 4 + de-normalization-primary: + job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask' + replica: 2 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 2 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: true + scale_target_value: 1000000 + min_replica: 1 + max_replica: 4 druid-validator: job_class_name: 'org.sunbird.dp.validator.task.DruidValidatorStreamTask' - replica: 1 + replica: 2 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 2 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: true + scale_target_value: 1000000 + min_replica: 1 + max_replica: 4 assessment-aggregator: job_class_name: 'org.sunbird.dp.assessment.task.AssessmentAggregatorStreamTask' replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: false content-cache-updater: job_class_name: 'org.sunbird.dp.contentupdater.task.ContentCacheUpdaterStreamTask' replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: false user-cache-updater: job_class_name: 'org.sunbird.dp.usercache.task.UserCacheUpdaterStreamTask' replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: false + user-cache-updater-v2: + job_class_name: 'org.sunbird.dp.usercache.task.UserCacheUpdaterStreamTaskV2' + replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: false summary-denormalization: job_class_name: 'org.sunbird.dp.denorm.task.SummaryDenormalizationStreamTask' replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: true + scale_target_value: 100 + min_replica: 1 + max_replica: 2 device-profile-updater: job_class_name: 'org.sunbird.dp.deviceprofile.task.DeviceProfileUpdaterStreamTask' replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: false + ingest-router: + job_class_name: 'org.sunbird.dp.ingestrouter.task.IngestRouterStreamTask' + replica: 2 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 2 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: true + scale_target_value: 500000 + min_replica: 1 + max_replica: 4 + error-denormalization: + job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask' + replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskmanager_process_memory: 1700m + jobmanager_process_memory: 1600m + taskslots: 1 + cpu_requests: 1 + cpu_limits: 2 + mem_requests: 2Gi + scale_enabled: false + +### controlling the flink jobs log level +flink_jobs_console_log_level: INFO +flink_libraries_log_level: ERROR diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/tasks/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/tasks/main.yml index 852620d12f..058c478a00 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/tasks/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/tasks/main.yml @@ -10,6 +10,6 @@ ignore_errors: yes - name: helm upgrade - shell: helm upgrade --install {{ item }} {{ chart_path }} --set job_classname={{ flink_job_names['%s'|format(item)].job_class_name }} --set taskmanager.replicas={{ flink_job_names['%s'|format(item)].replica }} -n {{ flink_namespace }} + shell: helm upgrade --install {{ item }} {{ chart_path }} --set job_classname={{ flink_job_names['%s'|format(item)].job_class_name }} --set taskmanager.replicas={{ flink_job_names['%s'|format(item)].replica }} --set taskmanager.cpu_requests={{ flink_job_names['%s'|format(item)].cpu_requests }} --set taskmanager.cpu_limits={{ flink_job_names['%s'|format(item)].cpu_limits }} --set taskmanager.mem_requests={{ flink_job_names['%s'|format(item)].mem_requests }} -n {{ flink_namespace }} with_items: - "{{ job_names_to_deploy.split(',')|list }}" diff --git a/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml b/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml new file mode 100644 index 0000000000..85d6f0f625 --- /dev/null +++ b/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml @@ -0,0 +1,571 @@ +secor_namespace: "secor" +image_repository: "secor" +image_tag: "0.29" +image_pullPolicy: IfNotPresent +imagepullsecrets: "{{ env }}-registry-secret" +secor_storage_class: managed-premium + +# this varible is to control the monitoring deployment. override this value by adding into +# private repo common.yml to not to create the secor lag alert rules. The scenorio is , if adopters does not have +# monitoring stack then the secor deployment job will fail. To avoid that we introduced this var. +secor_alertrule_enabled: true +# image: +# prefix: {} +# repository: "secor" +# tag: "secor-0.29_cf79409_7018" +# pullPolicy: IfNotPresent + +secor_jobs: + raw-telemetry-backup: + replicas: 1 + consumer_group: "{{env_name}}.telemetry.raw.backup" + service_name: "raw_telemetry_backup" + base_path: "raw" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{env_name}}.telemetry.raw" + kafka_broker_host: "{{ groups['processing-cluster-kafka']|join(',') }}" + zookeeper_quorum: "{{ groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + message_channel_identifier: "" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + failed-telemetry-backup: + replicas: 1 + service_name: "failed_telemetry_backup" + consumer_group: "{{ env_name }}.telemetry.failed.backup" + base_path: "failed" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.failed" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + unique-telemetry-backup: + replicas: 1 + service_name: "unique_telemetry_backup" + consumer_group: "{{ env_name }}.telemetry.unique.backup" + base_path: "unique/raw" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.unique" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + denorm-events-backup: + replicas: 1 + service_name: "denorm_events_backup" + consumer_group: "{{ env_name }}.telemetry.denorm.backup" + base_path: "telemetry-denormalized" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.denorm" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "true" + partition_prefix_key: "eid" + partition_prefix_mapping: '{"ME_WORKFLOW_SUMMARY":"summary","DEFAULT":"raw"}' + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + derived-denorm-events-backup: + replicas: 1 + service_name: "derived_denorm_events_backup" + consumer_group: "{{ env_name }}.summary.backup" + base_path: "telemetry-denormalized/summary" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.druid.events.summary" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + channel-telemetry-backup: + replicas: 1 + service_name: "channel_telemetry_backup" + consumer_group: "{{ env_name }}.telemetry.channel.backup" + base_path: "data-exhaust/raw" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.denorm" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "eid" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" + message_channel_identifier: "derivedlocationdata.state" + message_parser: "com.pinterest.secor.parser.ChannelDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + channel-summary-backup: + replicas: 1 + service_name: "channel_summary_backup" + consumer_group: "{{ env_name }}.summary.channel.backup" + base_path: "data-exhaust/summary" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.druid.events.summary" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json" + message_channel_identifier: "derivedlocationdata.state" + message_parser: "com.pinterest.secor.parser.ChannelDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + extractor-failed-backup: + replicas: 1 + service_name: "extractor_failed_backup" + consumer_group: "{{ env_name }}.extractor.failed.backup" + base_path: "extractor-failed" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.extractor.failed" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 21600 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + assess-raw-events-backup: + replicas: 1 + service_name: "assess_raw_events_backup" + consumer_group: "{{ env_name }}.telemetry.assess.raw" + base_path: "telemetry-raw-assess" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.assess.raw" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 21600 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + telemetry-duplicate-backup: + replicas: 1 + service_name: "telemetry_duplicate_backup" + consumer_group: "{{ env_name }}.telemetry.duplicate.backup" + base_path: "telemetry-duplicate" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.duplicate" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + extractor-duplicate-backup: + replicas: 1 + service_name: "extractor_duplicate_backup" + consumer_group: "{{ env_name }}.extractor.duplicate.backup" + base_path: "extractor-duplicate" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.extractor.duplicate" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + derived-telemetry-backup: + replicas: 1 + service_name: "derived_telemetry_backup" + consumer_group: "{{ env_name }}.telemetry.derived.unique.backup" + base_path: "unique/summary" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.derived.unique" + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 600 + partition_prefix_enabled: "true" + partition_prefix_key: "eid" + partition_prefix_mapping: '{"ME_WORKFLOW_SUMMARY":"workflow_summary","DEFAULT":"me"}' + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + device-profile-backup: + replicas: 1 + service_name: "device_profile_backup" + consumer_group: "{{ env_name }}.events.device.profile.backup" + base_path: "device-profile-events" + timestamp_key: "updated_date" + fallback_timestamp_key: "updated_date" + topic: "{{ env_name }}.events.deviceprofile" + kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}" + zookeeper_quorum: "{{ ingestion_zookeepers }}" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + learning-events-backup: + replicas: 1 + service_name: "learning_events_backup" + consumer_group: "{{ env_name }}.learning.graph.events.backup" + base_path: "learning-events" + timestamp_key: "createdOn" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.learning.graph.events" + kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}" + zookeeper_quorum: "{{ ingestion_zookeepers }}" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + learning-failed-backup: + replicas: 1 + service_name: "learning_failed_backup" + consumer_group: "{{ env_name }}.failed.learning.events.backup" + base_path: "learning-failed-events" + timestamp_key: "ets" + fallback_timestamp_key: "@timestamp" + azure_account_name: "{{ secor.azure.account_name }}" + container_name: "{{ secor.azure.container_name }}" + azure_account_key: "{{ secor.azure.account_key }}" + topic: "{{ env_name }}.learning.events.failed" + kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}" + zookeeper_quorum: "{{ ingestion_zookeepers }}" + max_file_size: 100000000 + max_file_age: 3600 + partition_prefix_enabled: "true" + partition_prefix_key: "jobName" + partition_prefix_mapping: '{"publish-pipeline":"publish_pipeline","composite-search-indexer":"cs_index","DEFAULT":"failed_events"}' + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + assess-events-backup: + replicas: 1 + service_name: "assess_events_backup" + consumer_group: "{{ env_name }}.telemetry.assess.events.backup" + base_path: "telemetry-batch-assess" + timestamp_key: "assessmentTs" + fallback_timestamp_key: "assessmentTs" + topic: "{{ env_name }}.telemetry.assess" + kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}" + zookeeper_quorum: "{{ ingestion_zookeepers }}" + max_file_size: 100000000 + max_file_age: 21600 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + batch-assess-failed-events-backup: + replicas: 1 + service_name: "batch_assess_failed_events_backup" + consumer_group: "{{ env_name }}.telemetry.assess.failed.events.backup" + base_path: "telemetry-batch-assess-failed" + timestamp_key: "assessmentTs" + fallback_timestamp_key: "assessmentTs" + topic: "{{ env_name }}.telemetry.assess.failed" + kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}" + zookeeper_quorum: "{{ ingestion_zookeepers }}" + max_file_size: 100000000 + max_file_age: 21600 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + ingestion-telemetry-backup: + replicas: 1 + service_name: "ingestion_telemetry_backup" + consumer_group: "{{ env_name }}.telemetry.ingestion.events.backup" + base_path: "ingestion-telemetry" + timestamp_key: "syncts" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.telemetry.ingestion" + kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}" + zookeeper_quorum: "{{ ingestion_zookeepers }}" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + content-consumption-events-backup: + replicas: 1 + service_name: "content_consumption_events_backup" + consumer_group: "{{ env_name }}.coursebatch.job.request.backup" + base_path: "content-consumption-events" + timestamp_key: "ets" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.coursebatch.job.request" + kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}" + zookeeper_quorum: "{{ ingestion_zookeepers }}" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + issue-certificate-events-backup: + replicas: 1 + service_name: "issue_certificate_events_backup" + consumer_group: "{{ env_name }}.issue.certificate.request.backup" + base_path: "issue-certificate-events" + timestamp_key: "ets" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.issue.certificate.request" + kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}" + zookeeper_quorum: "{{ ingestion_zookeepers }}" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + content-auto-creation-events-backup: + replicas: 1 + service_name: "content_auto_creation_events_backup" + consumer_group: "{{ env_name }}.auto.creation.job.request.backup" + base_path: "content-auto-creation-events" + timestamp_key: "ets" + fallback_timestamp_key: "@timestamp" + topic: "{{ env_name }}.auto.creation.job.request" + kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}" + zookeeper_quorum: "{{ ingestion_zookeepers }}" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 + error-telemetry-backup: + replicas: 1 + service_name: "error_events_backup" + consumer_group: '{{ env_name }}.druid.events.error.backup' + base_path: "error_events" + timestamp_key: "syncts" + fallback_timestamp_key: '@timestamp' + topic: '{{ env_name }}.druid.events.error' + kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}" + zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181" + max_file_size: 100000000 + max_file_age: 14400 + partition_prefix_enabled: "false" + partition_prefix_key: "" + partition_prefix_mapping: "{}" + output_file_pattern: "{partition}-{currentTimestamp}.json" + message_channel_identifier: "" + message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" + storage: + size: 10Gi + requests: + cpu: 500m + memory: 500Mi + lag_threshold_warning: 50000 + lag_threshold_critical: 100000 \ No newline at end of file diff --git a/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml b/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml new file mode 100644 index 0000000000..68778084ee --- /dev/null +++ b/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml @@ -0,0 +1,18 @@ +- name: template values.yaml file + vars: + secor_job_list: "{{ job_names_to_deploy.split(',')|list }}" + template: + src: "{{ chart_path }}/values.j2" + dest: "{{ chart_path }}/values.yaml" + +- name: helm upgrade + shell: helm --debug upgrade --install {{ item }} {{ chart_path }} -n {{ secor_namespace }} + with_items: + - "{{ job_names_to_deploy.split(',')|list }}" + tags: secor_deploy + +- name: create alert rules + shell: helm upgrade --install {{ item }} {{ chart_path }}/charts/alertrules -f {{ chart_path }}/values.yaml + with_items: + - "{{ job_names_to_deploy.split(',')|list }}" + tags: alertrule diff --git a/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml b/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml index ddf8c0362c..5958977ac7 100644 --- a/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml +++ b/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml @@ -4,6 +4,8 @@ fullnameOverride: sunbird-monitoring monitoring_stack: - prometheus-operator - alertrules + - prometheus-adapter + - processing-kafka-exporter namespace: monitoring flink_namespace: flink-dev @@ -15,29 +17,64 @@ monitor_alerts_mail_server_host: "{{ sendgrid_hostname }}" monitor_alerts_mail_server_port: 587 monitor_alerts_mail_server_username: "{{ SGUSER }}" monitor_alerts_mail_server_password: "{{ SGPASS }}" -monitor_alerts_slack_url: -monitor_alerts_slack_channel: +dp_monitor_alerts_slack_url: +dp_monitor_alerts_slack_channel: "{{ env }}_flink_alerts" +dp_monitor_alerts_warning_slack_url: +dp_monitor_alerts_warning_slack_channel: "{{ env }}_pipeline_warning_alerts" +dp_monitor_alerts_critical_slack_url: +dp_monitor_alerts_critical_slack_channel: "{{ env }}_pipeline_critical_alerts" default_critical_mailing_list: "{{ default_mailing_list }}" default_fatal_mailing_list: "{{ default_mailing_list }}" default_mailing_list: kaliraja.ramasamy@tarento.com, anandp@ilimi.in, manjunathd@ilimi.in -# flink Jobs lag vars +# flink Jobs lag vars critical telemetry_extractor_threshold_critical: 50000 pipeline_preprocessor_threshold_critical: 50000 -de_normalization_threshold_critical: 50000 +cb_preprocessor_threshold_critical: 50000 +de_normalization_threshold_critical: 50000 ### this var being used for de-normalization-primary and secondary jobs druid_validator_threshold_critical: 50000 assessment_aggregator_threshold_critical: 50000 content_cache_updater_threshold_critical: 50000 user_cache_updater_threshold_critical: 50000 summary_denormalization_threshold_critical: 50000 device_profile_updater_threshold_critical: 50000 -telemetry_extractor_failed_events_percentage_threshold_fatal: 1 -telemetry_extractor_failed_events_percentage_threshold_critical: 0.5 +ingest_router_threshold_critical: 50000 +error_denormalization_threshold_critical: 50000 +de_normalization_primary_threshold_critical: 50000 +de_normalization_secondary_threshold_critical: 50000 +telemetry_extractor_failed_events_percentage_threshold_critical: 1 +telemetry_extractor_failed_events_percentage_threshold_warning: 0.5 + +# Flink jobs lag warning +telemetry_extractor_threshold_warning: 30000 +pipeline_preprocessor_threshold_warning: 30000 +cb_preprocessor_threshold_warning: 30000 +de_normalization_threshold_warning: 30000 ### this var being used for de-normalization-primary and secondary jobs +druid_validator_threshold_warning: 30000 +assessment_aggregator_threshold_warning: 30000 +content_cache_updater_threshold_warning: 30000 +user_cache_updater_threshold_warning: 30000 +summary_denormalization_threshold_warning: 30000 +device_profile_updater_threshold_warning: 30000 +ingest_router_threshold_warning: 30000 +error_denormalization_threshold_warning: 30000 +de_normalization_primary_threshold_warning: 30000 +de_normalization_secondary_threshold_warning: 30000 +# Flink KP Jobs lag vars critical +activity_aggregater_updater_threshold_critical: 50 +relation_cache_updater_threshold_critical: 50 +post_publish_processor_threshold_critical: 50 + +# Flink KP Jobs lag vars warning +activity_aggregater_updater_threshold_warning: 40 +relation_cache_updater_threshold_warning: 40 +post_publish_processor_threshold_warning: 40 # Flink jobmanager Checkpoint failure vars telemetry_extractor_checkpointfailure_threshold_critical: 5 pipeline_preprocessor_checkpointfailure_threshold_critical: 5 +cb_preprocessor_checkpointfailure_threshold_critical: 5 de_normalization_checkpointfailure_threshold_critical: 5 druid_validator_checkpointfailure_threshold_critical: 5 assessment_aggregator_checkpointfailure_threshold_critical: 5 @@ -45,3 +82,24 @@ content_cache_updater_checkpointfailure_threshold_critical: 5 user_cache_updater_checkpointfailure_threshold_critical: 5 summary_denormalization_checkpointfailure_threshold_critical: 5 device_profile_updater_checkpointfailure_threshold_critical: 5 +user_cache_v2_updater_checkpointfailure_threshold_critical: 5 +relation_cache_updater_checkpointfailure_threshold_critical: 5 +activity_aggregate_updater_checkpointfailure_threshold_critical: 5 +ingest_router_checkpointfailure_threshold_critical: 5 +error_denormalization_checkpointfailure_threshold_critical: 5 +post_publish_processor_checkpontfailure_threshold_critical: 5 + +nginx_tps_threshold: 50 # being used in flink jobs percentage lag alert rules +flink_lag_percentage_threshold: 10 # being used in flink jobs percentage lag alert rule +flink_lag_offset_duration: 30 # values in minutes and being used in alert rule + +pv_usage_threshold_critical: 90 # If the persistant volume used percentage is greater than 90% then will get an alert. +kafka_exporter_image_tag: v1.3.0 +# kafka exporters vars +### processing kafka exporters related vars +processing_cluster_zookeeper: "{{ groups['processing-cluster-zookeepers'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:2181') | list}}" +processing_cluster_kafka: "{{ groups['processing-cluster-kafka'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:9092') | list}}" + +### ingestion kafka exporters related vars. +ingestion_cluster_zookeeper: "{{ groups['ingestion-cluster-zookeepers'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:2181') | list}}" +ingestion_cluster_kafka: "{{ groups['ingestion-cluster-kafka'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:9092') | list}}" diff --git a/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml b/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml index 6ce13d49d2..37f74a264a 100644 --- a/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml +++ b/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml @@ -2,10 +2,10 @@ # tasks file for sunbird-monitoring - name: tempating variables template: - src: "{{ item }}.yaml" - dest: "/tmp/{{item}}.yaml" + src: "dp_{{ item }}.yaml" + dest: "/tmp/dp_{{item}}.yaml" with_items: "{{ monitoring_stack }}" - name: Creating sunbird data infra monitoring stack - shell: "helm upgrade --install {{ item }} {{chart_path}}/{{ item }} --namespace monitoring -f /tmp/{{ item }}.yaml" + shell: "helm upgrade --install {{ item }} {{chart_path}}/{{ item }} --namespace monitoring -f /tmp/dp_{{ item }}.yaml" with_items: "{{ monitoring_stack }}" diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml deleted file mode 100644 index ee7499d061..0000000000 --- a/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml +++ /dev/null @@ -1,43 +0,0 @@ -## This file contains alert rules related vars -prometheus_rule_selector_app: prometheus-operator -prometheus_rule_selector_release: prometheus-operator -fullnameOverride: sunbird-monitoring -namespace: monitoring - -# Node Exporter vars -node_cpu_usage_percentage_threshold_Warning: 75 -node_cpu_usage_percentage_threshold_Critical: 85 -node_cpu_usage_percentage_threshold_Fatal: 95 -node_memory_usage_percentage_threshold_Critical: 85 -node_memory_usage_percentage_threshold_Warning: 75 -node_memory_usage_percentage_threshold_Fatal: 95 -node_load_avg_threshold_Critical: 95 -node_load_avg_threshold_Warning: 85 -node_load_avg_threshold_Fatal: 120 -node_disk_usage_percentage_threshold_Warning: 75 -node_disk_usage_percentage_threshold_Critical: 85 -node_disk_usage_percentage_threshold_Fatal: 95 - -# Jobs lag vars -telemetry_extractor_threshold_critical: {{ telemetry_extractor_threshold_critical }} -pipeline_preprocessor_threshold_critical: {{ pipeline_preprocessor_threshold_critical }} -de_normalization_threshold_critical: {{ de_normalization_threshold_critical }} -druid_validator_threshold_critical: {{ druid_validator_threshold_critical }} -assessment_aggregator_threshold_critical: {{ assessment_aggregator_threshold_critical }} -content_cache_updater_threshold_critical: {{ content_cache_updater_threshold_critical }} -user_cache_updater_threshold_critical: {{ user_cache_updater_threshold_critical }} -summary_denormalization_threshold_critical: {{ summary_denormalization_threshold_critical }} -device_profile_updater_threshold_critical: {{ device_profile_updater_threshold_critical }} -telemetry_extractor_failed_events_percentage_threshold_fatal: {{ telemetry_extractor_failed_events_percentage_threshold_fatal }} -telemetry_extractor_failed_events_percentage_threshold_critical: {{ telemetry_extractor_failed_events_percentage_threshold_critical }} - -# Checkpoint failure vars -telemetry_extractor_checkpointfailure_threshold_critical: {{ telemetry_extractor_checkpointfailure_threshold_critical }} -pipeline_preprocessor_checkpointfailure_threshold_critical: {{ pipeline_preprocessor_checkpointfailure_threshold_critical }} -de_normalization_checkpointfailure_threshold_critical: {{ de_normalization_checkpointfailure_threshold_critical }} -druid_validator_checkpointfailure_threshold_critical: {{ druid_validator_checkpointfailure_threshold_critical }} -assessment_aggregator_checkpointfailure_threshold_critical: {{ assessment_aggregator_checkpointfailure_threshold_critical }} -content_cache_updater_checkpointfailure_threshold_critical: {{ content_cache_updater_checkpointfailure_threshold_critical }} -user_cache_updater_checkpointfailure_threshold_critical: {{ user_cache_updater_checkpointfailure_threshold_critical }} -summary_denormalization_checkpointfailure_threshold_critical: {{ summary_denormalization_checkpointfailure_threshold_critical }} -device_profile_updater_checkpointfailure_threshold_critical: {{ device_profile_updater_checkpointfailure_threshold_critical }} diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_additional-scrape-configs.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_additional-scrape-configs.yaml new file mode 100644 index 0000000000..a3cf54dd1f --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_additional-scrape-configs.yaml @@ -0,0 +1,26 @@ +# This name is used to define the +# additionalScrapeConfigs name +# {{ fullnameOverride }}-prometheus-scrape-confg +# If you change this, make sure to update the value in +# additionalScrapeConfigs/defautls/main.yaml +#} + +fullnameOverride: "sunbird-monitoring" + +scrapeconfig: + # This scrape config will enable us to annotate any pod and that pod will get automonitored + # + # annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "2020" + # prometheus.io/path: /api/v1/metrics/prometheus + # + - job_name: 'federate' + metrics_path: /prometheus/federate + honor_labels: true + params: + match[]: + - '{__name__=~"^job:.*"}' + static_configs: + # This empty line make sure the indentation is correct + - targets: [ '{{private_ingressgateway_ip}}' ] diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_alertrules.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_alertrules.yaml new file mode 100644 index 0000000000..eea73ac4ad --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_alertrules.yaml @@ -0,0 +1,99 @@ +## This file contains alert rules related vars +prometheus_rule_selector_app: prometheus-operator +prometheus_rule_selector_release: prometheus-operator +fullnameOverride: sunbird-monitoring +namespace: monitoring + +# Node Exporter vars +node_cpu_usage_percentage_threshold_Warning: 80 +node_cpu_usage_percentage_threshold_Critical: 90 +node_cpu_usage_percentage_threshold_Fatal: 95 +node_memory_usage_percentage_threshold_Critical: 90 +node_memory_usage_percentage_threshold_Warning: 80 +node_memory_usage_percentage_threshold_Fatal: 95 +node_load_avg_threshold_Critical: 90 +node_load_avg_threshold_Warning: 80 +node_load_avg_threshold_Fatal: 95 +node_disk_usage_percentage_threshold_Warning: 80 +node_disk_usage_percentage_threshold_Critical: 90 +node_disk_usage_percentage_threshold_Fatal: 95 + +# DP Jobs lag vars critical +telemetry_extractor_threshold_critical: {{ telemetry_extractor_threshold_critical }} +pipeline_preprocessor_threshold_critical: {{ pipeline_preprocessor_threshold_critical }} +cb_preprocessor_threshold_critical: {{ cb_preprocessor_threshold_critical }} +de_normalization_threshold_critical: {{ de_normalization_threshold_critical }} +druid_validator_threshold_critical: {{ druid_validator_threshold_critical }} +assessment_aggregator_threshold_critical: {{ assessment_aggregator_threshold_critical }} +content_cache_updater_threshold_critical: {{ content_cache_updater_threshold_critical }} +user_cache_updater_threshold_critical: {{ user_cache_updater_threshold_critical }} +summary_denormalization_threshold_critical: {{ summary_denormalization_threshold_critical }} +device_profile_updater_threshold_critical: {{ device_profile_updater_threshold_critical }} +telemetry_extractor_failed_events_percentage_threshold_warning: {{ telemetry_extractor_failed_events_percentage_threshold_warning }} +telemetry_extractor_failed_events_percentage_threshold_critical: {{ telemetry_extractor_failed_events_percentage_threshold_critical }} +ingest_router_threshold_critical: {{ ingest_router_threshold_critical }} + +# DP Jobs lag vars warning +telemetry_extractor_threshold_warning: {{ telemetry_extractor_threshold_warning }} +pipeline_preprocessor_threshold_warning: {{ pipeline_preprocessor_threshold_warning }} +cb_preprocessor_threshold_warning: {{ cb_preprocessor_threshold_warning }} +de_normalization_threshold_warning: {{ de_normalization_threshold_warning }} +druid_validator_threshold_warning: {{ druid_validator_threshold_warning }} +assessment_aggregator_threshold_warning: {{ assessment_aggregator_threshold_warning }} +content_cache_updater_threshold_warning: {{ content_cache_updater_threshold_warning }} +user_cache_updater_threshold_warning: {{ user_cache_updater_threshold_warning }} +summary_denormalization_threshold_warning: {{ summary_denormalization_threshold_warning }} +device_profile_updater_threshold_warning: {{ device_profile_updater_threshold_warning }} +ingest_router_threshold_warning: {{ ingest_router_threshold_warning }} + + +# KP flink jobs lag vars critical +activity_aggregater_updater_threshold_critical: {{ activity_aggregater_updater_threshold_critical }} +relation_cache_updater_threshold_critical: {{ relation_cache_updater_threshold_critical }} +post_publish_processor_threshold_critical: {{ post_publish_processor_threshold_critical }} + +# KP flink jobs lag vars warning +activity_aggregater_updater_threshold_warning: {{ activity_aggregater_updater_threshold_warning }} +relation_cache_updater_threshold_warning: {{ relation_cache_updater_threshold_warning }} +post_publish_processor_threshold_warning: {{ post_publish_processor_threshold_warning }} + +# Checkpoint failure vars +telemetry_extractor_checkpointfailure_threshold_critical: {{ telemetry_extractor_checkpointfailure_threshold_critical }} +pipeline_preprocessor_checkpointfailure_threshold_critical: {{ pipeline_preprocessor_checkpointfailure_threshold_critical }} +cb_preprocessor_checkpointfailure_threshold_critical: {{ cb_preprocessor_checkpointfailure_threshold_critical }} +de_normalization_checkpointfailure_threshold_critical: {{ de_normalization_checkpointfailure_threshold_critical }} +druid_validator_checkpointfailure_threshold_critical: {{ druid_validator_checkpointfailure_threshold_critical }} +assessment_aggregator_checkpointfailure_threshold_critical: {{ assessment_aggregator_checkpointfailure_threshold_critical }} +content_cache_updater_checkpointfailure_threshold_critical: {{ content_cache_updater_checkpointfailure_threshold_critical }} +user_cache_updater_checkpointfailure_threshold_critical: {{ user_cache_updater_checkpointfailure_threshold_critical }} +summary_denormalization_checkpointfailure_threshold_critical: {{ summary_denormalization_checkpointfailure_threshold_critical }} +device_profile_updater_checkpointfailure_threshold_critical: {{ device_profile_updater_checkpointfailure_threshold_critical }} +user_cache_v2_updater_checkpointfailure_threshold_critical: {{ user_cache_v2_updater_checkpointfailure_threshold_critical }} +relation_cache_updater_checkpointfailure_threshold_critical: {{ relation_cache_updater_checkpointfailure_threshold_critical }} +activity_aggregate_updater_checkpointfailure_threshold_critical: {{ activity_aggregate_updater_checkpointfailure_threshold_critical }} + + +flink_job_names: + assessment-aggregator: {{ assessment_aggregator_threshold_critical }} + content-cache-updater: {{ content_cache_updater_threshold_critical }} + de-normalization-primary: {{ de_normalization_primary_threshold_critical }} + de-normalization-secondary: {{ de_normalization_secondary_threshold_critical }} + device-profile-updater: {{ device_profile_updater_threshold_critical }} + druid-validator: {{ druid_validator_threshold_critical }} + error-denormalization: {{ error_denormalization_threshold_critical }} + ingest-router: {{ ingest_router_threshold_critical }} + pipeline-preprocessor: {{ pipeline_preprocessor_threshold_critical }} + summary-denormalization: {{ summary_denormalization_threshold_critical }} + telemetry-extractor: {{ telemetry_extractor_threshold_critical }} + user-cache-updater-v2: {{ user_cache_updater_threshold_critical }} + cb-preprocessor: {{ cb_preprocessor_threshold_critical }} +## KP flink jobs + activity-aggregate-updater: {{ activity_aggregater_updater_threshold_critical }} + relation-cache-updater: {{ relation_cache_updater_threshold_critical }} + post-publish-processor: {{ post_publish_processor_threshold_critical }} + +nginx_tps_threshold: {{ nginx_tps_threshold }} # being used in flink jobs percentage lag alertrules +flink_lag_percentage_threshold: {{ flink_lag_percentage_threshold }} # being used in flink jobs percentage lag alertrules +flink_lag_offset_duration: {{ flink_lag_offset_duration }} # values in mins. offset time window to compare the metrics between current metrics and last 30(example) minutes. + +pv_usage_threshold_critical: {{ pv_usage_threshold_critical }} diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_ingestion-kafka-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_ingestion-kafka-exporter.yaml new file mode 100644 index 0000000000..2956b3976f --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_ingestion-kafka-exporter.yaml @@ -0,0 +1,22 @@ +kafkaExporter: + zookeeper: + servers: ["{{ ingestion_cluster_zookeeper | join('","') }}"] + kafka: + servers: ["{{ ingestion_cluster_kafka | join('","') }}"] + additionalFlags: + - --use.consumelag.zookeeper + +image: + repository: danielqsj/kafka-exporter + tag: "{{ kafka_exporter_image_tag }}" + pullPolicy: IfNotPresent + +prometheus: + serviceMonitor: + enabled: true + namespace: monitoring + interval: "120s" + scrapeTimeout: "90s" + additionalLabels: + app: prometheus-operator + release: prometheus-operator diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_processing-kafka-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_processing-kafka-exporter.yaml new file mode 100644 index 0000000000..1f03467f34 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_processing-kafka-exporter.yaml @@ -0,0 +1,22 @@ +kafkaExporter: + zookeeper: + servers: ["{{ processing_cluster_zookeeper | join('","') }}"] + kafka: + servers: ["{{ processing_cluster_kafka | join('","') }}"] + additionalFlags: + - --use.consumelag.zookeeper + +image: + repository: danielqsj/kafka-exporter + tag: "{{ kafka_exporter_image_tag }}" + pullPolicy: IfNotPresent + +prometheus: + serviceMonitor: + enabled: true + namespace: monitoring + interval: "120s" + scrapeTimeout: "90s" + additionalLabels: + app: prometheus-operator + release: prometheus-operator diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-adapter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-adapter.yaml new file mode 100644 index 0000000000..74a8add5ea --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-adapter.yaml @@ -0,0 +1,146 @@ +# Default values for k8s-prometheus-adapter.. + +# Url to access prometheus +prometheus: + url: http://sunbird-monitoring-prometheus.monitoring.svc.cluster.local + port: 9090 + path: "" + +rules: + default: true + custom: {} + existing: + external: + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "telemetry-extractor_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-telemetry-extractor-group",topic="{{ env_name }}.telemetry.ingest"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "ingest-router_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-ingest-router-group"}) by (consumergroup) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "pipeline-preprocessor_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-pipeline-preprocessor-group",topic="{{ env_name }}.telemetry.raw"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "de-normalization-primary_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-telemetry-denorm-primary-group",topic="{{ env_name }}.telemetry.unique.primary"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "de-normalization-secondary_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-telemetry-denorm-secondary-group",topic="{{ env_name }}.telemetry.unique.secondary"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "druid-validator_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-druid-validator-group",topic="{{ env_name }}.telemetry.denorm"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "error-denormalization_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-error-denorm-group",topic="{{ env_name }}.telemetry.error"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "summary-denormalization_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-summmary-denorm-group",topic="{{ env_name }}.telemetry.derived"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "content-cache-updater_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-content-cache-updater-group",topic="{{ env_name }}.learning.graph.events"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "device-profile-updater_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-device-profile-updater-group",topic="{{ env_name }}.events.deviceprofile"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "user-cache-updater-v2_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-user-cache-updater-consumer-group",topic="{{ env_name }}.telemetry.audit"}) by (<<.GroupBy>>) + - seriesQuery: 'kafka_consumergroup_lag_sum' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)" + as: "assessment-aggregator_kafka_consumergroup_lag_sum" + metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-assessment-aggregator-group",topic="{{ env_name }}.telemetry.assess"}) by (<<.GroupBy>>) + diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-operator.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-operator.yaml new file mode 100644 index 0000000000..2fb4f89192 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-operator.yaml @@ -0,0 +1,242 @@ +#jinja2:lstrip_blocks: True +{# +# This name is used to define the +# additionalScrapeConfigs name +# {{ fullnameOverride }}-prometheus-scrape-confg +# If you change this, make sure to update the value in +# additionalScrapeConfigs/defautls/main.yaml +#} +fullnameOverride: sunbird-monitoring + +# Enabling external prometheus scrape config +prometheus: + prometheusSpec: + additionalScrapeConfigsExternal: {{ additional_scrape_configs_enabled | d(false) }} + retention: "{{ prometheus_retention_time | d('60d') }}" + externalLabels: + sb_cluster: "{{ kubernetes_cluster_name | default('kubernetes-1')}}" +{% if prometheus_storage_spec is defined and prometheus_storage_spec %} + storageSpec: {{ prometheus_storage_spec|to_json }} +{% endif %} +{% if prometheus_service is defined and prometheus_service %} + service: {{ (prometheus_service | to_json) }} +{% endif %} +kubeControllerManager: + enabled: false +kubeScheduler: + enabled: false + + +alertmanager: + config: + global: + smtp_from: "{{ monitor_alerts_mail_from_email }}" + smtp_smarthost: "{{ monitor_alerts_mail_server_host }}:{{ monitor_alerts_mail_server_port}}" + smtp_auth_username: "{{ monitor_alerts_mail_server_username }}" + smtp_auth_password: "{{ monitor_alerts_mail_server_password }}" + route: + receiver: '{{env}}_devops_team' + group_by: [sb_cluster, alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + routes: + - match_re: + alertname: ^(node_exporter_down_critical|Watchdog|PrometheusRuleFailures) + receiver: "null" + - match: + alerttype: nodemetrics + severity: warning + receiver: nodemetrics_slack_warning + - match: + alerttype: nodemetrics + severity: critical + receiver: nodemetrics_slack_critical + - match: + alerttype: lag + severity: warning + receiver: lag_slack_warning + - match: + alerttype: lag + severity: critical + receiver: lag_slack_critical + - match: + alerttype: job + severity: critical + receiver: jobs_slack_critical + - receiver: slack + continue: true + - match: + owner: + receiver: {{env}}_devops_team + receivers: + - name: {{env}}_devops_team + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + - name: 'jobs_slack_critical' + slack_configs: + - send_resolved: true + api_url: "{{ dp_monitor_alerts_critical_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ dp_monitor_alerts_critical_slack_channel }}" + title_link: "" + title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }} + *Job:* {{ "{{" }} .Annotations.job_id {{ "}}" }} + *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }} + *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + + - name: 'lag_slack_warning' + slack_configs: + - send_resolved: true + api_url: "{{ dp_monitor_alerts_warning_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ dp_monitor_alerts_warning_slack_channel }}" + title_link: "" + title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }} + *Job:* {{ "{{" }} .Annotations.job_id {{ "}}" }} + *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }} + *AlertMetric:* {{ "{{" }} .Annotations.lag {{ "}}" }} + *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + + - name: 'lag_slack_critical' + slack_configs: + - send_resolved: true + api_url: "{{ dp_monitor_alerts_critical_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ dp_monitor_alerts_critical_slack_channel }}" + title_link: "" + title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }} + *Job:* {{ "{{" }} .Annotations.job_id {{ "}}" }} + *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }} + *AlertMetric:* {{ "{{" }} .Annotations.lag {{ "}}" }} + *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + + - name: 'nodemetrics_slack_warning' + slack_configs: + - send_resolved: true + api_url: "{{ dp_monitor_alerts_warning_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ dp_monitor_alerts_warning_slack_channel }}" + title_link: "" + title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }} + *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }} + *AlertMetric:* {{ "{{" }} .Annotations.metrics {{ "}}" }} + *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + + - name: 'nodemetrics_slack_critical' + slack_configs: + - send_resolved: true + api_url: "{{ dp_monitor_alerts_critical_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ dp_monitor_alerts_critical_slack_channel }}" + title_link: "" + title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }} + *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }} + *AlertMetric:* {{ "{{" }} .Annotations.metrics {{ "}}" }} + *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + + + - name: 'slack' + slack_configs: + - send_resolved: true + api_url: "{{ dp_monitor_alerts_warning_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ dp_monitor_alerts_warning_slack_channel }}" + title_link: "" + title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.message {{ "}}" }} - `{{ "{{" }} .Labels.severity {{ "}}" }}` + *Description:* {{ "{{" }} .Annotations.message {{ "}}" }} + *Details:* + {{ "{{" }} range .Labels.SortedPairs {{ "}}" }} • *{{ "{{" }} .Name {{ "}}" }}:* `{{ "{{" }} .Value {{ "}}" }}` + {{ "{{" }} end {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + - name: 'null' + +grafana: + env: + # GF_SERVER_ROOT_URL: http://grafana.local.com/grafana + adminPassword: {{ (grafana_admin_password| default('prom-operator'))}} +{% if grafana_data_sources is defined and grafana_data_sources %} + additionalDataSources: {{ (grafana_data_sources) | to_json}} +{% endif %} +{% if grafana_persistence is defined and grafana_persistence %} + persistence: {{ grafana_persistence|to_json}} +{% endif %} +{% if grafana_service is defined and grafana_service %} + service: {{ grafana_service|to_json}} +{% endif %} + +## Overriding the kubernetes-apps alert rules by disabling default alert rule creation. +# Why: We were getting few unwanted alerts and we wanted to disable those alerts. Hence created the custom alert rules. +defaultRules: + create: true + rules: + kubernetesApps: false diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml deleted file mode 100644 index f624aad38f..0000000000 --- a/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml +++ /dev/null @@ -1,71 +0,0 @@ -#jinja2:lstrip_blocks: True -{# -# This name is used to define the -# additionalScrapeConfigs name -# {{ fullnameOverride }}-prometheus-scrape-confg -# If you change this, make sure to update the value in -# additionalScrapeConfigs/defautls/main.yaml -#} -fullnameOverride: sunbird-monitoring - -# Enabling external prometheus scrape config -prometheus: - prometheusSpec: - additionalScrapeConfigsExternal: {{ additional_scrape_configs_enabled | d(false) }} - retention: "{{ prometheus_retention_time | d('90d') }}" - externalLabels: - sb_cluster: "{{ kubernetes_cluster_name | default('kubernetes-1')}}" -{% if prometheus_storage_spec is defined and prometheus_storage_spec %} - storageSpec: {{ prometheus_storage_spec|to_json }} -{% endif %} -{% if prometheus_service is defined and prometheus_service %} - service: {{ (prometheus_service | to_json) }} -{% endif %} - - -alertmanager: - config: - global: - smtp_from: "{{ monitor_alerts_mail_from_email }}" - smtp_smarthost: "{{ monitor_alerts_mail_server_host }}:{{ monitor_alerts_mail_server_port}}" - smtp_auth_username: "{{ monitor_alerts_mail_server_username }}" - smtp_auth_password: "{{ monitor_alerts_mail_server_password }}" - route: - receiver: '{{env}}_devops_team' - group_by: [sb_cluster, alertname] - group_wait: 30s - group_interval: 5m - repeat_interval: 4h - routes: - - match: - owner: - receiver: {{env}}_devops_team - receivers: - - name: {{env}}_devops_team - email_configs: - - send_resolved: true - to: '{{ default_mailing_list }}' - html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' - headers: - subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' - -grafana: - env: - # GF_SERVER_ROOT_URL: http://grafana.local.com/grafana - adminPassword: {{ (grafana_admin_password| default('prom-operator'))}} -{% if grafana_data_sources is defined and grafana_data_sources %} - additionalDataSources: {{ (grafana_data_sources) | to_json}} -{% endif %} -{% if grafana_persistence is defined and grafana_persistence %} - persistence: {{ grafana_persistence|to_json}} -{% endif %} -{% if grafana_service is defined and grafana_service %} - service: {{ grafana_service|to_json}} -{% endif %} - -## Overriding the kubernetes-apps alert rules by disabling default alert rule creation. -# Why: We were getting few unwanted alerts and we wanted to disable those alerts. Hence created the custom alert rules. -defaultRules: - create: true - rules: - kubernetesApps: false diff --git a/kubernetes/ansible/secor_deploy.yaml b/kubernetes/ansible/secor_deploy.yaml new file mode 100644 index 0000000000..15de8ddd4b --- /dev/null +++ b/kubernetes/ansible/secor_deploy.yaml @@ -0,0 +1,20 @@ +--- +- hosts: local + gather_facts: no + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + roles: + - secor-deploy + tags: secor_deploy + +- hosts: local + gather_facts: no + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + KUBECONFIG: "{{ core_kubeconfig_path }}" + roles: + - secor-deploy + tags: alertrule diff --git a/kubernetes/helm_charts/bootstrap/reloader/.helmignore b/kubernetes/helm_charts/bootstrap/reloader/.helmignore new file mode 100755 index 0000000000..9e1690881f --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/.helmignore @@ -0,0 +1,2 @@ +# OWNERS file for Kubernetes +OWNERS diff --git a/kubernetes/helm_charts/bootstrap/reloader/Chart.yaml b/kubernetes/helm_charts/bootstrap/reloader/Chart.yaml new file mode 100755 index 0000000000..b4995a9a82 --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/Chart.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +appVersion: v0.0.49 +description: Reloader chart that runs on kubernetes +home: https://github.com/stakater/Reloader +icon: https://raw.githubusercontent.com/stakater/Reloader/master/assets/web/reloader-round-100px.png +keywords: +- Reloader +- kubernetes +maintainers: +- email: hello@stakater.com + name: Stakater +- email: rasheed@aurorasolutions.io + name: rasheedamir +- email: waseemhassan@stakater.com + name: waseem-h +- email: faizan.ahmad55@outlook.com + name: faizanahmad055 +- email: ali.kahoot@aurorasolutions.io + name: kahootali +- email: ahmad@aurorasolutions.io + name: ahmadiq +- email: ahsanmuhammad1@outlook.com + name: ahsan-storm +name: reloader +sources: +- https://github.com/stakater/IngressMonitorController +version: v0.0.49 diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/NOTES.txt b/kubernetes/helm_charts/bootstrap/reloader/templates/NOTES.txt new file mode 100755 index 0000000000..f2a38752a4 --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/templates/NOTES.txt @@ -0,0 +1,7 @@ +- For a `Deployment` called `foo` have a `ConfigMap` called `foo-configmap`. Then add this annotation to main metadata of your `Deployment` + configmap.reloader.stakater.com/reload: "foo-configmap" + +- For a `Deployment` called `foo` have a `Secret` called `foo-secret`. Then add this annotation to main metadata of your `Deployment` + secret.reloader.stakater.com/reload: "foo-secret" + +- After successful installation, your pods will get rolling updates when a change in data of configmap or secret will happen. diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/_helpers.tpl b/kubernetes/helm_charts/bootstrap/reloader/templates/_helpers.tpl new file mode 100755 index 0000000000..15ca27ef7d --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/templates/_helpers.tpl @@ -0,0 +1,35 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} + +{{- define "reloader-name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" | lower -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "reloader-fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "reloader-labels.chart" -}} +app: {{ template "reloader-fullname" . }} +chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" +release: {{ .Release.Name | quote }} +heritage: {{ .Release.Service | quote }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "reloader-serviceAccountName" -}} +{{- if .Values.reloader.serviceAccount.create -}} + {{ default (include "reloader-fullname" .) .Values.reloader.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.reloader.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml new file mode 100755 index 0000000000..8d51ef406b --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml @@ -0,0 +1,62 @@ +{{- if and .Values.reloader.watchGlobally (.Values.reloader.rbac.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.rbac.labels }} +{{ toYaml .Values.reloader.rbac.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }}-role + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + resources: +{{- if .Values.reloader.ignoreSecrets }}{{- else }} + - secrets +{{- end }} +{{- if .Values.reloader.ignoreConfigMaps }}{{- else }} + - configmaps +{{- end }} + verbs: + - list + - get + - watch +{{- if or (.Capabilities.APIVersions.Has "apps.openshift.io/v1") (.Values.isOpenshift) }} + - apiGroups: + - "apps.openshift.io" + - "" + resources: + - deploymentconfigs + verbs: + - list + - get + - update + - patch +{{- end }} + - apiGroups: + - "apps" + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - list + - get + - update + - patch + - apiGroups: + - "extensions" + resources: + - deployments + - daemonsets + verbs: + - list + - get + - update + - patch +{{- end }} diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml new file mode 100755 index 0000000000..28c9d4b916 --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.reloader.watchGlobally (.Values.reloader.rbac.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.rbac.labels }} +{{ toYaml .Values.reloader.rbac.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }}-role-binding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "reloader-fullname" . }}-role +subjects: + - kind: ServiceAccount + name: {{ template "reloader-serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/deployment.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/deployment.yaml new file mode 100755 index 0000000000..c03a86f50e --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/templates/deployment.yaml @@ -0,0 +1,129 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: +{{- if .Values.reloader.deployment.annotations }} + annotations: +{{ toYaml .Values.reloader.deployment.annotations | indent 4 }} +{{- end }} + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.deployment.labels }} +{{ toYaml .Values.reloader.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }} +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: {{ template "reloader-fullname" . }} + release: {{ .Release.Name | quote }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 6 }} +{{- end }} + template: + metadata: +{{- if .Values.reloader.deployment.pod.annotations }} + annotations: +{{ toYaml .Values.reloader.deployment.pod.annotations | indent 8 }} +{{- end }} + labels: +{{ include "reloader-labels.chart" . | indent 8 }} +{{- if .Values.reloader.deployment.labels }} +{{ toYaml .Values.reloader.deployment.labels | indent 8 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 8 }} +{{- end }} + spec: + {{- if .Values.reloader.deployment.nodeSelector }} + nodeSelector: +{{ toYaml .Values.reloader.deployment.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.reloader.deployment.affinity }} + affinity: +{{ toYaml .Values.reloader.deployment.affinity | indent 8 }} + {{- end }} + {{- if .Values.reloader.deployment.tolerations }} + tolerations: +{{ toYaml .Values.reloader.deployment.tolerations | indent 8 }} + {{- end }} + containers: + - env: + {{- range $name, $value := .Values.reloader.deployment.env.open }} + {{- if not (empty $value) }} + - name: {{ $name | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- $secret_name := include "reloader-fullname" . }} + {{- range $name, $value := .Values.reloader.deployment.env.secret }} + {{- if not ( empty $value) }} + - name: {{ $name | quote }} + valueFrom: + secretKeyRef: + name: {{ $secret_name }} + key: {{ $name | quote }} + {{- end }} + {{- end }} + {{- range $name, $value := .Values.reloader.deployment.env.field }} + {{- if not ( empty $value) }} + - name: {{ $name | quote }} + valueFrom: + fieldRef: + fieldPath: {{ $value | quote}} + {{- end }} + {{- end }} + {{- if eq .Values.reloader.watchGlobally false }} + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + image: "{{ .Values.reloader.deployment.image.name }}:{{ .Values.reloader.deployment.image.tag }}" + imagePullPolicy: {{ .Values.reloader.deployment.image.pullPolicy }} + name: {{ template "reloader-fullname" . }} + {{- if eq .Values.reloader.readOnlyRootFileSystem true }} + volumeMounts: + - mountPath: /tmp/ + name: tmp-volume + {{- end }} + args: + {{- if .Values.reloader.ignoreSecrets }} + - "--resources-to-ignore=secrets" + {{- end }} + {{- if eq .Values.reloader.ignoreConfigMaps true }} + - "--resources-to-ignore=configMaps" + {{- end }} + + {{- if .Values.reloader.custom_annotations }} + {{- if .Values.reloader.custom_annotations.configmap }} + - "--configmap-annotation" + - "{{ .Values.reloader.custom_annotations.configmap }}" + {{- end }} + {{- if .Values.reloader.custom_annotations.secret }} + - "--secret-annotation" + - "{{ .Values.reloader.custom_annotations.secret }}" + {{- end }} + {{- if .Values.reloader.custom_annotations.auto }} + - "--auto-annotation" + - "{{ .Values.reloader.custom_annotations.auto }}" + {{- end }} + {{- end }} + + {{- if .Values.reloader.deployment.resources }} + resources: +{{ toYaml .Values.reloader.deployment.resources | indent 10 }} + {{- end }} +{{- if .Values.reloader.deployment.securityContext }} + securityContext: {{ toYaml .Values.reloader.deployment.securityContext | nindent 8 }} +{{- end }} + serviceAccountName: {{ template "reloader-serviceAccountName" . }} + {{- if eq .Values.reloader.readOnlyRootFileSystem true }} + volumes: + - emptyDir: {} + name: tmp-volume + {{- end }} diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml new file mode 100755 index 0000000000..5827f5cdcb --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml @@ -0,0 +1,62 @@ +{{- if and (not (.Values.reloader.watchGlobally)) (.Values.reloader.rbac.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.rbac.labels }} +{{ toYaml .Values.reloader.rbac.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }}-role + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + resources: +{{- if .Values.reloader.ignoreSecrets }}{{- else }} + - secrets +{{- end }} +{{- if .Values.reloader.ignoreConfigMaps }}{{- else }} + - configmaps +{{- end }} + verbs: + - list + - get + - watch +{{- if or (.Capabilities.APIVersions.Has "apps.openshift.io/v1") (.Values.reloader.isOpenshift) }} + - apiGroups: + - "apps.openshift.io" + - "" + resources: + - deploymentconfigs + verbs: + - list + - get + - update + - patch +{{- end }} + - apiGroups: + - "apps" + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - list + - get + - update + - patch + - apiGroups: + - "extensions" + resources: + - deployments + - daemonsets + verbs: + - list + - get + - update + - patch +{{- end }} diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml new file mode 100755 index 0000000000..94fb1f838b --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml @@ -0,0 +1,23 @@ +{{- if and (not (.Values.reloader.watchGlobally)) (.Values.reloader.rbac.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.rbac.labels }} +{{ toYaml .Values.reloader.rbac.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }}-role-binding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "reloader-fullname" . }}-role +subjects: + - kind: ServiceAccount + name: {{ template "reloader-serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/serviceaccount.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/serviceaccount.yaml new file mode 100755 index 0000000000..62e5f1a0af --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.reloader.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 2 }} +{{- end }} +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.serviceAccount.labels }} +{{ toYaml .Values.reloader.serviceAccount.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-serviceAccountName" . }} +{{- end }} diff --git a/kubernetes/helm_charts/bootstrap/reloader/values.yaml b/kubernetes/helm_charts/bootstrap/reloader/values.yaml new file mode 100755 index 0000000000..0973025cbd --- /dev/null +++ b/kubernetes/helm_charts/bootstrap/reloader/values.yaml @@ -0,0 +1,88 @@ +# Generated from deployments/kubernetes/templates/chart/values.yaml.tmpl +global: + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + +kubernetes: + host: https://kubernetes.default + +reloader: + isOpenshift: false + ignoreSecrets: false + ignoreConfigMaps: false + watchGlobally: true + # Set to true if you have a pod security policy that enforces readOnlyRootFilesystem + readOnlyRootFileSystem: false + matchLabels: {} + deployment: + nodeSelector: + # cloud.google.com/gke-nodepool: default-pool + + # An affinity stanza to be applied to the Deployment. + # Example: + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "node-role.kubernetes.io/infra-worker" + # operator: "Exists" + affinity: {} + + # A list of tolerations to be applied to the Deployment. + # Example: + # tolerations: + # - key: "node-role.kubernetes.io/infra-worker" + # operator: "Exists" + # effect: "NoSchedule" + tolerations: [] + + labels: + provider: stakater + group: com.stakater.platform + version: v0.0.49 + image: + name: stakater/reloader + tag: "v0.0.49" + pullPolicy: IfNotPresent + # Support for extra environment variables. + env: + # Open supports Key value pair as environment variables. + open: + # secret supports Key value pair as environment variables. It gets the values based on keys from default reloader secret if any. + secret: + # field supports Key value pair as environment variables. It gets the values from other fields of pod. + field: + + # Specify resource requests/limits for the deployment. + # Example: + # resources: + # limits: + # cpu: "100m" + # memory: "512Mi" + # requests: + # cpu: "10m" + # memory: "128Mi" + resources: {} + pod: + annotations: {} + + rbac: + enabled: true + labels: {} + # Service account config for the agent pods + serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + labels: {} + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Optional flags to pass to the Reloader entrypoint + # Example: + # custom_annotations: + # configmap: "my.company.com/configmap" + # secret: "my.company.com/secret" + custom_annotations: {} diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_configmap.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_configmap.yaml index 4c0201904f..48a2719ab6 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_configmap.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_configmap.yaml @@ -8,6 +8,7 @@ metadata: data: base-config: |+ {{ .Values.base_config | indent 4 }} - {{ .Release.Name }}: {{- $name := .Release.Name }} -{{ index .Values $name | toYaml | indent 4 }} +{{ index .Values $name | toYaml | indent 2 }} + log4j_console_properties: |+ +{{ .Values.log4j_console_properties | indent 4 }} diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 89c40c6b15..ad540b9a78 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -84,16 +84,21 @@ spec: component: {{ .Release.Name }}-jobmanager annotations: prometheus.io/scrape: 'true' + prometheus.io/port: "{{ .Values.jobmanager.prom_port }}" spec: volumes: - name: flink-config-volume configMap: name: {{ .Release.Name }}-config items: + - key: flink-conf + path: flink-conf.yaml - key: base-config path: base-config.conf - key: {{ .Release.Name }} path: {{ .Release.Name }}.conf + - key: log4j_console_properties + path: log4j-console.properties restartPolicy: OnFailure imagePullSecrets: - name: {{ .Values.imagepullsecrets }} @@ -104,8 +109,7 @@ spec: workingDir: /opt/flink command: ["/opt/flink/bin/standalone-job.sh"] args: ["start-foreground", - "--job-classname={{ .Values.job_classname }}", - "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}", + "--job-classname={{ .Values.job_classname }}", "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}", @@ -114,7 +118,6 @@ spec: "-Dparallelism.default=1", "-Dblob.server.port={{ .Values.jobmanager.blob_port }}", "-Dqueryable-state.server.ports={{ .Values.jobmanager.query_port }}", - "-Djobmanager.heap.size={{ .Values.jobmanager.heap_memory }}m", "--config.file.path", "/data/flink/conf/{{ .Release.Name }}.conf"] ports: @@ -128,7 +131,17 @@ spec: name: ui volumeMounts: - name: flink-config-volume - mountPath: /data/flink/conf + mountPath: /opt/flink/conf/flink-conf.yaml + subPath: flink-conf.yaml + - name: flink-config-volume + mountPath: /data/flink/conf/base-config.conf + subPath: base-config.conf + - name: flink-config-volume + mountPath: /data/flink/conf/{{ .Release.Name }}.conf + subPath: {{ .Release.Name }}.conf + - name: flink-config-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties --- apiVersion: apps/v1 @@ -148,23 +161,36 @@ spec: app: flink component: {{ .Release.Name }}-taskmanager spec: + volumes: + - name: flink-config-volume + configMap: + name: {{ .Release.Name }}-config + items: + - key: flink-conf + path: flink-conf.yaml + - key: log4j_console_properties + path: log4j-console.properties imagePullSecrets: - name: {{ .Values.imagepullsecrets }} containers: - name: {{ .Release.Name }}-taskmanager image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" imagePullPolicy: Always + resources: + requests: + cpu: "{{ .Values.taskmanager.cpu_requests }}" + memory: "{{ .Values.taskmanager.mem_requests }}" + limits: + cpu: "{{ .Values.taskmanager.cpu_limits }}" workingDir: {{ .Values.taskmanager.flink_work_dir }} command: ["/opt/flink/bin/taskmanager.sh"] args: ["start-foreground", - "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}", "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager", "-Dmetrics.reporter.prom.port=9251-9260", "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager", - "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}", - "-Dtaskmanager.heap.size={{ .Values.taskmanager.heap_memory }}m"] + "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"] ports: - containerPort: {{ .Values.taskmanager.rpc_port }} name: rpc @@ -172,3 +198,33 @@ spec: livenessProbe: {{ toYaml .Values.livenessProbe | indent 10 }} {{- end }} + volumeMounts: + - name: flink-config-volume + mountPath: /opt/flink/conf/flink-conf.yaml + subPath: flink-conf.yaml + - name: flink-config-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties + +{{- $name := .Release.Name }} +{{- $prop := (index .Values.scale_properties $name)}} +{{- if $prop.enabled}} +#--- +#apiVersion: autoscaling/v2beta1 +#kind: HorizontalPodAutoscaler +#metadata: +# name: {{ .Release.Name }}-taskmanager-hpa +# namespace: {{ .Values.namespace }} +#spec: +# scaleTargetRef: +# apiVersion: apps/v1 +# kind: Deployment +# name: {{ .Release.Name }}-taskmanager +# minReplicas: {{ $prop.min_replica }} +# maxReplicas: {{ $prop.max_replica }} +# metrics: +# - type: External +# external: +# metricName: {{ .Release.Name }}_kafka_consumergroup_lag_sum +# targetValue: "{{ $prop.scale_target_value }}" +#{{- end }} diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/serviceMonitor.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/serviceMonitor.yaml index 76cbf60215..6b6d0c2dda 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/serviceMonitor.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/serviceMonitor.yaml @@ -1,3 +1,4 @@ +{{- if .Values.serviceMonitor.enabled }} --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -36,3 +37,4 @@ spec: namespaceSelector: matchNames: - {{ .Values.namespace }} +{{- end }} diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index f7867d2ef1..1db1fccfcb 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -3,11 +3,82 @@ imagepullsecrets: {{ imagepullsecrets }} dockerhub: {{ dockerhub }} repository: {{flink_repository|default('sunbird-datapipeline')}} image_tag: {{ image_tag }} -registry: {{ docker_registry }} -azure_account: {{ azure_account }} -azure_secret: {{ azure_secret }} -replicaCount: {{taskmana_replicacount|default(1)}} +serviceMonitor: + enabled: {{ service_monitor_enabled | lower}} +replicaCount: {{taskmanager_replicacount|default(1)}} + +scale_properties: + telemetry-extractor: + enabled: {{ flink_job_names['telemetry-extractor'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['telemetry-extractor'].scale_target_value }} + min_replica: {{ flink_job_names['telemetry-extractor'].min_replica | default(1) }} + max_replica: {{ flink_job_names['telemetry-extractor'].max_replica | default(2) }} + pipeline-preprocessor: + enabled: {{ flink_job_names['pipeline-preprocessor'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['pipeline-preprocessor'].scale_target_value }} + min_replica: {{ flink_job_names['pipeline-preprocessor'].min_replica | default(1) }} + max_replica: {{ flink_job_names['pipeline-preprocessor'].max_replica | default(2) }} + cb-preprocessor: + enabled: {{ flink_job_names['cb-preprocessor'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['cb-preprocessor'].scale_target_value }} + min_replica: {{ flink_job_names['cb-preprocessor'].min_replica | default(1) }} + max_replica: {{ flink_job_names['cb-preprocessor'].max_replica | default(2) }} + de-normalization-secondary: + enabled: {{ flink_job_names['de-normalization-secondary'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['de-normalization-secondary'].scale_target_value }} + min_replica: {{ flink_job_names['de-normalization-secondary'].min_replica | default(1) }} + max_replica: {{ flink_job_names['de-normalization-secondary'].max_replica | default(2) }} + de-normalization-primary: + enabled: {{ flink_job_names['de-normalization-primary'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['de-normalization-primary'].scale_target_value }} + min_replica: {{ flink_job_names['de-normalization-primary'].min_replica | default(1) }} + max_replica: {{ flink_job_names['de-normalization-primary'].max_replica | default(2) }} + druid-validator: + enabled: {{ flink_job_names['druid-validator'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['druid-validator'].scale_target_value }} + min_replica: {{ flink_job_names['druid-validator'].min_replica | default(1) }} + max_replica: {{ flink_job_names['druid-validator'].max_replica | default(2) }} + assessment-aggregator: + enabled: {{ flink_job_names['assessment-aggregator'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['assessment-aggregator'].scale_target_value | default(0) }} + min_replica: {{ flink_job_names['assessment-aggregator'].min_replica | default(1) }} + max_replica: {{ flink_job_names['assessment-aggregator'].max_replica | default(2) }} + content-cache-updater: + enabled: {{ flink_job_names['content-cache-updater'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['content-cache-updater'].scale_target_value | default(0) }} + min_replica: {{ flink_job_names['content-cache-updater'].min_replica | default(1) }} + max_replica: {{ flink_job_names['content-cache-updater'].max_replica | default(2) }} + user-cache-updater-v2: + enabled: {{ flink_job_names['user-cache-updater-v2'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['user-cache-updater-v2'].scale_target_value | default(0) }} + min_replica: {{ flink_job_names['user-cache-updater-v2'].min_replica | default(1) }} + max_replica: {{ flink_job_names['user-cache-updater-v2'].max_replica | default(2) }} + summary-denormalization: + enabled: {{ flink_job_names['summary-denormalization'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['summary-denormalization'].scale_target_value | default(0) }} + min_replica: {{ flink_job_names['summary-denormalization'].min_replica | default(1) }} + max_replica: {{ flink_job_names['summary-denormalization'].max_replica | default(2) }} + device-profile-updater: + enabled: {{ flink_job_names['device-profile-updater'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['device-profile-updater'].scale_target_value | default(0) }} + min_replica: {{ flink_job_names['device-profile-updater'].min_replica | default(1) }} + max_replica: {{ flink_job_names['device-profile-updater'].max_replica | default(2) }} + rating: + enabled: {{ flink_job_names['rating'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['rating'].scale_target_value | default(0) }} + min_replica: {{ flink_job_names['rating'].min_replica | default(1) }} + max_replica: {{ flink_job_names['rating'].max_replica | default(2) }} + ingest-router: + enabled: {{ flink_job_names['ingest-router'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['ingest-router'].scale_target_value }} + min_replica: {{ flink_job_names['ingest-router'].min_replica | default(1) }} + max_replica: {{ flink_job_names['ingest-router'].max_replica | default(2) }} + error-denormalization: + enabled: {{ flink_job_names['error-denormalization'].scale_enabled | lower}} + scale_target_value: {{ flink_job_names['error-denormalization'].scale_target_value | default(0) }} + min_replica: {{ flink_job_names['error-denormalization'].min_replica | default(1) }} + max_replica: {{ flink_job_names['error-denormalization'].max_replica | default(2) }} jobmanager: rpc_port: {{ jobmanager_rpc_port }} @@ -27,39 +98,77 @@ taskmanager: rpc_port: {{ taskmanager_rpc_port }} heap_memory: {{ taskmanager_heap_memory }} replicas: {{taskmanager_replicacount|default(1)}} + cpu_requests: {{ cpu_requests|default(1)}} + mem_requests: {{ mem_requests|default(2)}} + cpu_limits: {{ cpu_limits|default(2)}} job_classname: {{ job_classname }} {{ taskmanager_liveness | to_nice_yaml }} +log4j_console_properties: | + # This affects logging for both user code and Flink + rootLogger.level = {{ flink_jobs_console_log_level | default(INFO) }} + rootLogger.appenderRef.console.ref = ConsoleAppender + + # Uncomment this if you want to _only_ change Flink's logging + #logger.flink.name = org.apache.flink + #logger.flink.level = {{ flink_jobs_console_log_level | default(INFO) }} + + # The following lines keep the log level of common libraries/connectors on + # log level INFO. The root logger does not override this. You have to manually + # change the log levels here. + logger.akka.name = akka + logger.akka.level = {{ flink_libraries_log_level | default(INFO) }} + logger.kafka.name= org.apache.kafka + logger.kafka.level = {{ flink_libraries_log_level | default(INFO) }} + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = {{ flink_libraries_log_level | default(INFO) }} + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = {{ flink_libraries_log_level | default(INFO) }} + + # Log all infos to the console + appender.console.name = ConsoleAppender + appender.console.type = CONSOLE + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + + # Suppress the irrelevant (wrong) warnings from the Netty channel handler + logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline + logger.netty.level = OFF base_config: | kafka { broker-servers = "{{ kafka_brokers }}" + producer.broker-servers = "{{ kafka_brokers }}" + consumer.broker-servers = "{{ kafka_brokers }}" zookeeper = "{{ zookeepers }}" producer { max-request-size = {{ producer_max_request_size }} + batch.size = {{ producer_batch_size }} + linger.ms = {{ producer_linger_ms }} + compression = {{ producer_compression }} } } job { env = "{{ env_name }}" - enable.distributed.checkpointing = true + enable.distributed.checkpointing = false statebackend { blob { storage { - account = "{{ azure_account }}.blob.core.windows.net" - container = "{{ flink_container_name }}" + account = "" + container = "" checkpointing.dir = "checkpoint" } } - base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir} + base.url = "" } } task { parallelism = 1 - consumer { - parallelism = 1 - } + consumer.parallelism = 1 + checkpointing.compressed = {{ checkpoint_compression_enabled|lower }} checkpointing.interval = {{ checkpoint_interval }} + checkpointing.pause.between.seconds = {{ checkpoint_pause_between_seconds }} restart-strategy.attempts = {{ restart_attempts }} restart-strategy.delay = {{ restart_delay }} # in milli-seconds } @@ -69,8 +178,8 @@ base_config: | port = 6379 } redis-meta { -{% if metadata_redis_host is defined %} - host = {{ metadata_redis_host }} +{% if metadata2_redis_host is defined %} + host = {{ metadata2_redis_host }} {% else %} host = {{ redis_host }} {% endif %} @@ -88,269 +197,696 @@ base_config: | port = "9042" } -telemetry-extractor: | - include file("/data/flink/conf/base-config.conf") - kafka { - input.topic = {{ env_name }}.telemetry.ingest - output.success.topic = {{ env_name }}.telemetry.raw - output.duplicate.topic = {{ env_name }}.telemetry.extractor.duplicate - output.failed.topic = {{ env_name }}.telemetry.failed - output.batch.failed.topic = {{ env_name }}.telemetry.extractor.failed - output.assess.raw.topic = {{ env_name }}.telemetry.assess.raw - event.max.size = "{{ extractor_event_max_size }}" # Max is only 1MB - groupId = {{ env_name }}-telemetry-extractor-group - producer { - max-request-size = {{ extractor_max_request_size }} - } - } - task { - consumer.parallelism = {{ extractor_consumer_parallelism }} - dedup.parallelism = {{ dedup_parallelism }} - extraction.parallelism = {{ extraction_parallelism }} - redactor.parallelism = {{ redactor_parallelism }} - } - redis { - database { - duplicationstore.id = 1 - key.expiry.seconds = {{ telemetry_extractor_key_expiry_seconds }} - contentstore.id = 5 - } - } - redis-meta { +ingest-router: + ingest-router: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + producer.broker-servers = "{{ kafka_brokers }}" + consumer.broker-servers = "{{ ingestion_kafka_brokers }}" + ingest { + input.topic = {{ env_name }}.telemetry.ingestion + output.success.topic = {{ env_name }}.telemetry.ingest + } + raw { + input.topic = "{{ env_name }}.telemetry.raw" + output.success.topic = "{{ env_name }}.telemetry.raw" + } + groupId = {{ env_name }}-ingest-router-group + producer { + max-request-size = {{ extractor_max_request_size }} + } + } + task { + consumer.parallelism = {{ ingest_router_consumer_parallelism }} + downstream.operators.parallelism = {{ ingest_router_operators_parallelism }} + raw { + consumer.parallelism = {{ raw_router_consumer_parallelism }} + downstream.operators.parallelism = {{ raw_router_downstream_parallelism }} + } + } + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['ingest-router'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['ingest-router'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['ingest-router'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + taskmanager.memory.process.size: {{ flink_job_names['ingest-router'].taskmanager_process_memory }} + jobmanager.memory.process.size: {{ flink_job_names['ingest-router'].jobmanager_process_memory }} + + +telemetry-extractor: + telemetry-extractor: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.topic = {{ env_name }}.telemetry.ingest + output.success.topic = {{ env_name }}.telemetry.raw + output.log.route.topic = {{ env_name }}.druid.events.log + output.duplicate.topic = {{ env_name }}.telemetry.extractor.duplicate + output.failed.topic = {{ env_name }}.telemetry.failed + output.batch.failed.topic = {{ env_name }}.telemetry.extractor.failed + output.assess.raw.topic = {{ env_name }}.telemetry.assess.raw + event.max.size = "{{ extractor_event_max_size }}" # Max is only 1MB + groupId = {{ env_name }}-telemetry-extractor-group + producer { + max-request-size = {{ extractor_max_request_size }} + } + } + task { + consumer.parallelism = {{ extractor_consumer_parallelism }} + downstream.operators.parallelism = {{ extractor_operators_parallelism }} + } + redis { database { + duplicationstore.id = 1 + key.expiry.seconds = {{ telemetry_extractor_key_expiry_seconds }} contentstore.id = 5 } } - redact.events.list = ["ASSESS","RESPONSE"] + redis-meta { + database { + contentstore.id = 5 + } + host = {{ redis_meta_content_host }} + port = {{ redis_meta_content_port }} + } + redact.events.list = ["ASSESS","RESPONSE"] -pipeline-preprocessor: | - include file("/data/flink/conf/base-config.conf") - kafka { - input.topic = {{ env_name }}.telemetry.raw - output.failed.topic = {{ env_name }}.telemetry.failed - output.primary.route.topic = {{ env_name }}.telemetry.unique - output.log.route.topic = {{ env_name }}.druid.events.log - output.error.route.topic = {{ env_name }}.druid.events.error - output.audit.route.topic = {{ env_name }}.telemetry.audit - output.duplicate.topic = {{ env_name }}.telemetry.duplicate - groupId = {{ env_name }}-pipeline-preprocessor-group - } - task { - consumer.parallelism = {{ pipeline_preprocessor_consumer_parallelism }} - telemetry.validation.parallelism = {{ telemetry_validation_parallelism }} - telemetry.router.parallelism = {{ telemetry_router_parallelism }} - share.events.flattener.parallelism = {{ share_events_flattener_parallelism }} - } - telemetry.schema.path="schemas/telemetry/3.0" - default.channel="b00bc992ef25f1a9a8d63291e20efc8d" - dedup.producer.included.ids = ["{{ portal_id }}", "{{ desktop_id }}"] - redis { - database { - duplicationstore.id = 7 - key.expiry.seconds = {{ pipeline_preprocessor_key_expiry_seconds }} - } - } - -de-normalization: | - include file("/data/flink/conf/base-config.conf") - kafka { - input.telemetry.topic = {{ env_name }}.telemetry.unique - input.summary.topic = {{ env_name }}.telemetry.derived - output.success.topic = {{ env_name }}.telemetry.denorm - output.failed.topic = {{ env_name }}.telemetry.failed - output.duplicate.topic = {{ env_name }}.telemetry.duplicate - output.summary.topic = {{ env_name }}.telemetry.derived.unique - groupId = {{ env_name }}-telemetry-denorm-group - } - task { - consumer.parallelism = {{ denorm_consumer_parallelism }} - denorm.parallelism = {{ denorm_parallelism }} - denorm.sink.parallelism = {{ denorm_sink_parallelism }} - denorm.summary-dedup.parallelism = {{ denorm_summary_dedup_parallelism }} - summary.sink.parallelism = {{ summary_sink_parallelism }} - } - redis { - database { - duplicationstore.id = 9 - key.expiry.seconds = {{ de_normalization_duplicationstore_key_expiry_seconds }} - } - } - # redis-metadata - redis-meta { - database { - devicestore.id = 2 - userstore.id = 4 - contentstore.id = 5 - dialcodestore.id = 6 - key.expiry.seconds = {{ de_normalization_key_expiry_seconds }} - } - } - -summary-denormalization: |+ - include file("/data/flink/conf/base-config.conf") - kafka { - input.telemetry.topic = {{ env_name }}.telemetry.unique - input.summary.topic = {{ env_name }}.telemetry.derived - output.success.topic = {{ env_name }}.telemetry.denorm - output.failed.topic = {{ env_name }}.telemetry.failed - output.duplicate.topic = {{ env_name }}.telemetry.duplicate - output.summary.topic = {{ env_name }}.telemetry.derived.unique - groupId = {{ env_name }}-summmary-denorm-group - } - task { - consumer.parallelism = {{ summary_denorm_consumer_parallelism }} - denorm.parallelism = {{ summary_denorm_parallelism }} - denorm.sink.parallelism = {{ summary_denorm_sink_parallelism }} - denorm.summary-dedup.parallelism = {{ summary_denorm_dedup_parallelism }} - summary.sink.parallelism = {{ summary_denorm_summary_sink_parallelism }} - } - redis { - database { - duplicationstore.id = 9 - key.expiry.seconds = {{ summary_denorm_duplication_key_expiry_seconds }} - } - } - # redis-metadata - redis-meta { - database { - devicestore.id = 2 - userstore.id = 4 - contentstore.id = 5 - dialcodestore.id = 6 - key.expiry.seconds = {{ summary_denorm_key_expiry_seconds }} - } - } - -druid-validator: | - include file("/data/flink/conf/base-config.conf") - kafka { - input.topic = {{ env_name }}.telemetry.denorm - output.telemetry.route.topic = {{ env_name }}.druid.events.telemetry - output.summary.route.topic = {{ env_name }}.druid.events.summary - output.failed.topic = {{ env_name }}.telemetry.failed - output.duplicate.topic = {{ env_name }}.telemetry.duplicate - groupId = {{ env_name }}-druid-validator-group - } - task { - consumer.parallelism = {{ druid_validator_consumer_parallelism }} - validator.parallelism = {{ validator_parallelism }} - router.parallelism = {{ router_parallelism }} - druid.validation.enabled = {{ druid_validation_enabled|lower }} - druid.deduplication.enabled = {{ druid_deduplication_enabled|lower }} - } - schema { - path { - telemetry = "schemas/telemetry" - summary = "schemas/summary" - } - file { - default = envelope.json - summary = me_workflow_summary.json - search = search.json - } - } - redis { - database { - duplicationstore.id = 8 - key.expiry.seconds = {{ druid_validator_key_expiry_seconds }} - } - } - -device-profile-updater: | - include file("/data/flink/conf/base-config.conf") - kafka { - broker-servers = "{{ ingestion_kafka_brokers }}" - zookeeper = "{{ ingestion_zookeepers }}" - input.topic = {{ env_name }}.events.deviceprofile - groupId = {{ env_name }}-device-profile-updater-group - } - task { - deviceprofile { - parallelism = {{ deviceprofile_parallelism }} - } - } - # redis-metadata - redis-meta { - host = {{ metadata_redis_host }} - port = 6379 - database { - devicestore.id = 2 - } - } - postgres { - database = "{{ postgres.db_name }}", - table = "{{ device_profile_table }}" - } - -content-cache-updater: | - include file("/data/flink/conf/base-config.conf") - kafka { - broker-servers = "{{ ingestion_kafka_brokers }}" - zookeeper = "{{ ingestion_zookeepers }}" - input.topic = {{ env_name }}.learning.graph.events - groupId = {{ env_name }}-content-cache-updater-group - } - # redis-metadata - redis-meta { - host = {{ metadata_redis_host }} - port = 6379 - database { - contentstore.id = 5 - dialcodestore.id = 6 - } - } - dialcode { - api { - url = "{{ dialcode_api_url }}" - token = "{{ dialcode_api_auth_key }} " - } - } - -user-cache-updater: | - include file("/data/flink/conf/base-config.conf") - kafka { - input.topic = {{ env_name }}.telemetry.audit - groupId = {{ env_name }}-user-cache-updater-group - } - task { - usercache.updater.parallelism = {{ usercache_updater_parallelism }} - } - # redis-metadata - redis-meta { - host = {{ metadata_redis_host }} - port = 6379 - database { - userstore.id = 4 - } - } - lms-cassandra { - keyspace = "{{ middleware_cassandra_keyspace }}" - table { - user = "{{ middleware_cassandra_user_table }}" - location = "{{ middleware_cassandra_location_table }}" - } - } - user.self.signin.types = ["google","self"] - user.validated.types = ["sso"] - user.self.signin.key = "Self-Signed-In" - user.valid.key = "Validated" - -assessment-aggregator: |+ - include file("/data/flink/conf/base-config.conf") - kafka { - broker-servers = "{{ ingestion_kafka_brokers }}" - zookeeper = "{{ ingestion_zookeepers }}" - input.topic = {{ env_name }}.telemetry.assess - failed.topic= {{ env_name }}.telemetry.assess.failed - groupId = {{ env_name }}-assessment-aggregator-group - } - task { - assessaggregator { - parallelism = {{ assessaggregator_parallelism }} - } - } - lms-cassandra { - keyspace = "{{ middleware_cassandra_courses_keyspace }}" - table = "{{ middleware_cassandra_assessment_aggregator_table }}" - questionudttype= "{{ middleware_cassandra_assessment_question_type }}" - } + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['telemetry-extractor'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['telemetry-extractor'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['telemetry-extractor'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + taskmanager.memory.process.size: {{ flink_job_names['telemetry-extractor'].taskmanager_process_memory }} + jobmanager.memory.process.size: {{ flink_job_names['telemetry-extractor'].jobmanager_process_memory }} + +pipeline-preprocessor: + pipeline-preprocessor: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.topic = {{ env_name }}.telemetry.raw + output.failed.topic = {{ env_name }}.telemetry.failed + output.primary.route.topic = {{ env_name }}.telemetry.unique + output.log.route.topic = {{ env_name }}.druid.events.log + output.error.route.topic = {{ env_name }}.telemetry.error + output.audit.route.topic = {{ env_name }}.telemetry.audit + output.duplicate.topic = {{ env_name }}.telemetry.duplicate + output.denorm.secondary.route.topic = {{ env_name }}.telemetry.unique.secondary + output.denorm.primary.route.topic = {{ env_name }}.telemetry.unique.primary + output.cb.audit.route.topic = {{ env_name }}.telemetry.cb.audit + groupId = {{ env_name }}-pipeline-preprocessor-group + } + task { + consumer.parallelism = {{ pipeline_preprocessor_consumer_parallelism }} + downstream.operators.parallelism = {{ pipeline_preprocessor_operators_parallelism }} + } + telemetry.schema.path="schemas/telemetry/3.0" + default.channel="{{ telemetry_default_org_id }}" + dedup.producer.included.ids = ["{{ portal_id }}", "{{ desktop_id }}", "{{ frac_id }}", "{{ mdo_id }}"] + secondary.events = ["INTERACT", "IMPRESSION", "SHARE_ITEM"] + redis { + database { + duplicationstore.id = 7 + key.expiry.seconds = {{ pipeline_preprocessor_key_expiry_seconds }} + } + } + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['pipeline-preprocessor'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['pipeline-preprocessor'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['pipeline-preprocessor'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + taskmanager.memory.process.size: {{ flink_job_names['pipeline-preprocessor'].taskmanager_process_memory }} + jobmanager.memory.process.size: {{ flink_job_names['pipeline-preprocessor'].jobmanager_process_memory }} + +cb-preprocessor: + cb-preprocessor: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.topic = {{ env_name }}.telemetry.cb.audit + output.failed.topic = {{ env_name }}.telemetry.failed + output.duplicate.topic = {{ env_name }}.telemetry.duplicate + output.cb.audit.topic = {{ env_name }}.druid.cb.audit + output.cb.work.order.row.topic = {{ env_name }}.druid.cb.work.order.row + output.cb.work.order.officer.topic = {{ env_name }}.druid.cb.work.order.officer + groupId = {{ env_name }}-cb-preprocessor-group + } + task { + consumer.parallelism = {{ cb_preprocessor_consumer_parallelism }} + downstream.operators.parallelism = {{ cb_preprocessor_operators_parallelism }} + } + + default.channel="{{ telemetry_default_org_id }}" + frac.channel="{{ telemetry_frac_org_id }}" + redis { + database { + duplicationstore.id = 7 + key.expiry.seconds = {{ cb_preprocessor_key_expiry_seconds }} + } + } + redis-meta { + database { + userstore.id = 12 + } + } + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['cb-preprocessor'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['cb-preprocessor'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['cb-preprocessor'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + taskmanager.memory.process.size: {{ flink_job_names['cb-preprocessor'].taskmanager_process_memory }} + jobmanager.memory.process.size: {{ flink_job_names['cb-preprocessor'].jobmanager_process_memory }} + +de-normalization-secondary: + de-normalization-secondary: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.telemetry.topic = {{ env_name }}.telemetry.unique.secondary + input.summary.topic = {{ env_name }}.telemetry.derived + telemetry.denorm.output.topic = {{ env_name }}.telemetry.denorm + summary.denorm.output.topic = {{ env_name }}.druid.events.summary + summary.unique.events.topic = {{ env_name }}.telemetry.derived.unique + output.failed.topic = {{ env_name }}.telemetry.failed + output.duplicate.topic = {{ env_name }}.telemetry.duplicate + groupId = {{ env_name }}-telemetry-denorm-secondary-group + } + skip.events = ["INTERRUPT"] + permit.eid=["AUDIT"] + task { + window.count = {{ denorm_secondary_window_count }} + window.shards = {{ denorm_secondary_window_shards }} + consumer.parallelism = {{ denorm_secondary_consumer_parallelism }} + telemetry.downstream.operators.parallelism = {{ telemetry_denorm_secondary_operators_parallelism }} + summary.downstream.operators.parallelism = {{ summary_denorm_operators_parallelism }} + } + redis { + database { + duplicationstore.id = 9 + key.expiry.seconds = {{ de_normalization_duplicationstore_key_expiry_seconds }} + } + } + # redis-metadata + redis-meta { + database { + devicestore.id = 2 + userstore.id = 12 + contentstore.id = 5 + dialcodestore.id = 6 + key.expiry.seconds = {{ de_normalization_key_expiry_seconds }} + } + content.host = {{ redis_meta_content_host }} + device.host = {{ redis_meta_device_host }} + user.host = {{ redis_meta_user_host }} + dialcode.host = {{ redis_meta_dialcode_host }} + content.port = {{ redis_meta_content_port }} + device.port = {{ redis_meta_device_port }} + user.port = {{ redis_meta_user_port }} + dialcode.port = {{ redis_meta_dialcode_port }} + } + # config version + user_denorm_version = v2 + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['de-normalization-secondary'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['de-normalization-secondary'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['de-normalization-secondary'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + taskmanager.memory.process.size: {{ flink_job_names['de-normalization-secondary'].taskmanager_process_memory }} + jobmanager.memory.process.size: {{ flink_job_names['de-normalization-secondary'].jobmanager_process_memory }} + +de-normalization-primary: + de-normalization-primary: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.telemetry.topic = {{ env_name }}.telemetry.unique.primary + input.summary.topic = {{ env_name }}.telemetry.derived + telemetry.denorm.output.topic = {{ env_name }}.telemetry.denorm + summary.denorm.output.topic = {{ env_name }}.druid.events.summary + summary.unique.events.topic = {{ env_name }}.telemetry.derived.unique + output.failed.topic = {{ env_name }}.telemetry.failed + output.duplicate.topic = {{ env_name }}.telemetry.duplicate + groupId = {{ env_name }}-telemetry-denorm-primary-group + } + skip.events = ["INTERRUPT"] + permit.eid=["AUDIT"] + task { + window.count = {{ denorm_primary_window_count }} + window.shards = {{ denorm_primary_window_shards }} + consumer.parallelism = {{ denorm_primary_consumer_parallelism }} + telemetry.downstream.operators.parallelism = {{ telemetry_denorm_primary_operators_parallelism }} + summary.downstream.operators.parallelism = {{ summary_denorm_operators_parallelism }} + } + redis { + database { + duplicationstore.id = 9 + key.expiry.seconds = {{ de_normalization_duplicationstore_key_expiry_seconds }} + } + } + # redis-metadata + redis-meta { + database { + devicestore.id = 2 + userstore.id = 12 + contentstore.id = 5 + dialcodestore.id = 6 + key.expiry.seconds = {{ de_normalization_key_expiry_seconds }} + } + content.host = {{ redis_meta_content_host }} + device.host = {{ redis_meta_device_host }} + user.host = {{ redis_meta_user_host }} + dialcode.host = {{ redis_meta_dialcode_host }} + content.port = {{ redis_meta_content_port }} + device.port = {{ redis_meta_device_port }} + user.port = {{ redis_meta_user_port }} + dialcode.port = {{ redis_meta_dialcode_port }} + } + # config version + user_denorm_version = v2 + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['de-normalization-primary'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['de-normalization-primary'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['de-normalization-primary'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + taskmanager.memory.process.size: {{ flink_job_names['de-normalization-primary'].taskmanager_process_memory }} + jobmanager.memory.process.size: {{ flink_job_names['de-normalization-primary'].jobmanager_process_memory }} + +summary-denormalization: + summary-denormalization: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.telemetry.topic = {{ env_name }}.telemetry.unique + input.summary.topic = {{ env_name }}.telemetry.derived + telemetry.denorm.output.topic = {{ env_name }}.telemetry.denorm + summary.denorm.output.topic = {{ env_name }}.druid.events.summary + summary.unique.events.topic = {{ env_name }}.telemetry.derived.unique + output.failed.topic = {{ env_name }}.telemetry.failed + output.duplicate.topic = {{ env_name }}.telemetry.duplicate + groupId = {{ env_name }}-summmary-denorm-group + } + skip.events = ["INTERRUPT"] + permit.eid=["AUDIT"] + task { + window.count = {{ denorm_summary_window_count }} + window.shards = {{ denorm_summary_window_shards }} + consumer.parallelism = {{ summary_denorm_consumer_parallelism }} + telemetry.downstream.operators.parallelism = {{ telemetry_denorm_operators_parallelism }} + summary.downstream.operators.parallelism = {{ summary_denorm_operators_parallelism }} + } + redis { + database { + duplicationstore.id = 9 + key.expiry.seconds = {{ summary_denorm_duplication_key_expiry_seconds }} + } + } + # redis-metadata + redis-meta { + database { + devicestore.id = 2 + userstore.id = 12 + contentstore.id = 5 + dialcodestore.id = 6 + key.expiry.seconds = {{ summary_denorm_key_expiry_seconds }} + } + content.host = {{ redis_meta_content_host }} + device.host = {{ redis_meta_device_host }} + user.host = {{ redis_meta_user_host }} + dialcode.host = {{ redis_meta_dialcode_host }} + content.port = {{ redis_meta_content_port }} + device.port = {{ redis_meta_device_port }} + user.port = {{ redis_meta_user_port }} + dialcode.port = {{ redis_meta_dialcode_port }} + } + # config version + user_denorm_version = v2 + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['summary-denormalization'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['summary-denormalization'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['summary-denormalization'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + taskmanager.memory.process.size: {{ flink_job_names['summary-denormalization'].taskmanager_process_memory }} + jobmanager.memory.process.size: {{ flink_job_names['summary-denormalization'].jobmanager_process_memory }} + +rating: + rating: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + producer.broker-servers = "{{ ingestion_kafka_brokers }}" + consumer.broker-servers = "{{ ingestion_kafka_brokers }}" + input.topic = {{ env_name }}.rating.event + output.topic = {{ env_name }}.rating.failedRating + groupId = {{ env_name }}-rating-group + } + ext-cassandra { + course_table = "user_enrolments" + ratings_table = "ratings" + ratings_summary_table = "ratings_summary" + ratings_lookup_table = "ratings_lookup" + keyspace = "sunbird" + courses_keyspace = "sunbird_courses" + host = "{{ core_cassandra_host }}" + port = "9042" + } + skip.events = ["INTERRUPT"] + permit.eid = ["AUDIT"] + task { + consumer.parallelism = {{ rating_consumer_parallelism }} + downstream.operators.parallelism = {{ rating_operators_parallelism }} + rating { + parallelism = {{ rating_parallelism }} + } + } + redis { + database { + duplicationstore.id = 9 + key.expiry.seconds = {{ rating_key_expiry_seconds }} + } + } + # redis-metadata + redis-meta { + database { + devicestore.id = 2 + userstore.id = 12 + contentstore.id = 5 + dialcodestore.id = 6 + key.expiry.seconds = {{ rating_key_expiry_seconds }} + } + content.host = {{ redis_meta_content_host }} + device.host = {{ redis_meta_device_host }} + user.host = {{ redis_meta_user_host }} + dialcode.host = {{ redis_meta_dialcode_host }} + content.port = {{ redis_meta_content_port }} + device.port = {{ redis_meta_device_port }} + user.port = {{ redis_meta_user_port }} + dialcode.port = {{ redis_meta_dialcode_port }} + } + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['rating'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['rating'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['rating'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + taskmanager.memory.process.size: {{ flink_job_names['rating'].taskmanager_process_memory }} + jobmanager.memory.process.size: {{ flink_job_names['rating'].jobmanager_process_memory }} + +druid-validator: + druid-validator: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.topic = {{ env_name }}.telemetry.denorm + output.telemetry.route.topic = {{ env_name }}.druid.events.telemetry + output.summary.route.topic = {{ env_name }}.druid.events.summary + output.failed.topic = {{ env_name }}.telemetry.failed + output.duplicate.topic = {{ env_name }}.telemetry.duplicate + groupId = {{ env_name }}-druid-validator-group + } + task { + consumer.parallelism = {{ druid_validator_consumer_parallelism }} + downstream.operators.parallelism = {{ druid_validator_operators_parallelism }} + druid.validation.enabled = {{ druid_validation_enabled|lower }} + druid.deduplication.enabled = {{ druid_deduplication_enabled|lower }} + } + schema { + path { + telemetry = "schemas/telemetry" + summary = "schemas/summary" + } + file { + default = envelope.json + summary = me_workflow_summary.json + search = search.json + } + } + redis { + database { + duplicationstore.id = 8 + key.expiry.seconds = {{ druid_validator_key_expiry_seconds }} + } + } + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['druid-validator'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['druid-validator'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['druid-validator'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + taskmanager.memory.process.size: {{ flink_job_names['druid-validator'].taskmanager_process_memory }} + jobmanager.memory.process.size: {{ flink_job_names['druid-validator'].jobmanager_process_memory }} + +device-profile-updater: + device-profile-updater: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + producer.broker-servers = "{{ ingestion_kafka_brokers }}" + consumer.broker-servers = "{{ ingestion_kafka_brokers }}" + zookeeper = "{{ ingestion_zookeepers }}" + input.topic = {{ env_name }}.events.deviceprofile + groupId = {{ env_name }}-device-profile-updater-group + } + task { + deviceprofile { + parallelism = {{ deviceprofile_parallelism }} + } + } + # redis-metadata + redis-meta { + database { + devicestore.id = 2 + } + host = {{ redis_meta_device_host }} + port = {{ redis_meta_device_port }} + } + postgres { + database = "{{ postgres.db_name }}", + table = "{{ device_profile_table }}" + } + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['device-profile-updater'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['device-profile-updater'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['device-profile-updater'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + +content-cache-updater: + content-cache-updater: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + producer.broker-servers = "{{ ingestion_kafka_brokers }}" + consumer.broker-servers = "{{ ingestion_kafka_brokers }}" + zookeeper = "{{ ingestion_zookeepers }}" + input.topic = {{ env_name }}.learning.graph.events + groupId = {{ env_name }}-content-cache-updater-group + } + # redis-metadata + redis-meta { + database { + contentstore.id = 5 + dialcodestore.id = 6 + } + content.host = {{ redis_meta_content_host }} + dialcode.host = {{ redis_meta_dialcode_host }} + content.port = {{ redis_meta_content_port }} + dialcode.port = {{ redis_meta_dialcode_port }} + } + dialcode { + api { + url = "{{ dialcode_api_url }}" + token = "{{ sunbird_api_auth_token }} " + } + } + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['content-cache-updater'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['content-cache-updater'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['content-cache-updater'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + +user-cache-updater-v2: + user-cache-updater-v2: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.topic = {{ env_name }}.telemetry.audit + groupId = {{ env_name }}-user-cache-updater-consumer-group + } + task { + usercache.updater.parallelism = {{ usercache_updater_parallelism }} + } + # redis-metadata + redis-meta { + database { + userstore.id = 12 + } + host = {{ redis_meta_user_host }} + port = {{ redis_meta_user_port }} + } + user-read { + api { + url = "{{ user_read_api_url }}" + } + } + regd.user.producer.pid = "learner-service" + user.self.signin.types = ["google","self"] + user.validated.types = ["sso"] + user.self.signin.key = "Self-Signed-In" + user.valid.key = "Validated" + user.read.url.fields = "locations,organisations" + user.read.api.error = ["CLIENT_ERROR"] + user.redis.registration-count.path = "mdo_registered_officer_count" + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['user-cache-updater-v2'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['user-cache-updater-v2'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['user-cache-updater-v2'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + +assessment-aggregator: + assessment-aggregator: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + producer.broker-servers = "{{ ingestion_kafka_brokers }}" + consumer.broker-servers = "{{ ingestion_kafka_brokers }}" + zookeeper = "{{ ingestion_zookeepers }}" + input.topic = {{ env_name }}.telemetry.assess + failed.topic= {{ env_name }}.telemetry.assess.failed + groupId = {{ env_name }}-assessment-aggregator-group + output.certissue.topic = {{ env_name }}.issue.certificate.request + } + task { + consumer.parallelism = {{ assessaggregator_consumer_parallelism }} + downstream.parallelism = {{ assessaggregator_downstream_parallelism }} + assessaggregator { + parallelism = {{ assessaggregator_parallelism }} + } + scoreaggregator.parallelism = {{ assessaggregator_scoreaggregator_parallelism }} + } + lms-cassandra { + keyspace = "{{ middleware_cassandra_courses_keyspace }}" + table = "{{ middleware_cassandra_assessment_aggregator_table }}" + questionudttype= "{{ middleware_cassandra_assessment_question_type }}" + enrolmentstable = "{{ middleware_cassandra_user_enrolments_table }}" + activitytable = "{{ middleware_cassandra_user_activity_agg_table }}" + } + redis { + database { + relationCache.id = 10 + contentCache.id = 5 + } + } + assessment.skip.missingRecords = true + content.read.api = "{{ content_read_api_host }}/{{ content_read_api_endpoint }}" + user.activity.agg.type="attempt_metrics" + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['assessment-aggregator'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['assessment-aggregator'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['assessment-aggregator'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + +error-denormalization: + error-denormalization: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.telemetry.topic = {{ env_name }}.telemetry.error + input.summary.topic = {{ env_name }}.telemetry.derived + telemetry.denorm.output.topic = {{ env_name }}.druid.events.error + summary.denorm.output.topic = {{ env_name }}.druid.events.summary + summary.unique.events.topic = {{ env_name }}.telemetry.derived.unique + output.failed.topic = {{ env_name }}.telemetry.failed + output.duplicate.topic = {{ env_name }}.telemetry.duplicate + groupId = {{ env_name }}-error-denorm-group + } + skip.events = ["INTERRUPT"] + permit.eid=["AUDIT"] + task { + window.count = {{ denorm_primary_window_count }} + window.shards = {{ denorm_primary_window_shards }} + consumer.parallelism = {{ error_denorm_consumer_parallelism }} + telemetry.downstream.operators.parallelism = {{ error_denorm_operators_parallelism }} + summary.downstream.operators.parallelism = {{ summary_denorm_operators_parallelism }} + } + redis { + database { + duplicationstore.id = 9 + key.expiry.seconds = {{ de_normalization_duplicationstore_key_expiry_seconds }} + } + } + # redis-metadata + redis-meta { + database { + devicestore.id = 2 + userstore.id = 12 + contentstore.id = 5 + dialcodestore.id = 6 + key.expiry.seconds = {{ de_normalization_key_expiry_seconds }} + } + content.host = {{ redis_meta_content_host }} + device.host = {{ redis_meta_device_host }} + user.host = {{ redis_meta_user_host }} + dialcode.host = {{ redis_meta_dialcode_host }} + content.port = {{ redis_meta_content_port }} + device.port = {{ redis_meta_device_port }} + user.port = {{ redis_meta_user_port }} + dialcode.port = {{ redis_meta_dialcode_port }} + } + # config version + user_denorm_version = v2 + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['error-denormalization'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['error-denormalization'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['error-denormalization'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 diff --git a/kubernetes/helm_charts/druid-cluster/.helmignore b/kubernetes/helm_charts/druid-cluster/.helmignore new file mode 100644 index 0000000000..fbe01f88f2 --- /dev/null +++ b/kubernetes/helm_charts/druid-cluster/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/kubernetes/helm_charts/druid-cluster/Chart.yaml b/kubernetes/helm_charts/druid-cluster/Chart.yaml new file mode 100644 index 0000000000..9588f92cc4 --- /dev/null +++ b/kubernetes/helm_charts/druid-cluster/Chart.yaml @@ -0,0 +1,18 @@ +name: druid-cluster +version: 0.1.0 +appVersion: 0.21.1 +description: Helm chart to deploy a druid cluster on Kubernetes +keywords: +- druid +- zookeeper +maintainers: +- name: druid-operator-maintainers +sources: +- https://github.com/druid-io/druid-operator +engine: gotpl +home: https://github.com/druid-io/druid-operator +dependencies: +- name: zookeeper + version: 7.4.0 + repository: https://charts.bitnami.com/bitnami + condition: zookeeper.enabled diff --git a/kubernetes/helm_charts/druid-cluster/requirements.lock b/kubernetes/helm_charts/druid-cluster/requirements.lock new file mode 100644 index 0000000000..3ce72f9405 --- /dev/null +++ b/kubernetes/helm_charts/druid-cluster/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: zookeeper + repository: https://charts.bitnami.com/bitnami + version: 7.4.0 +digest: sha256:ac1eacfdf8dca15edf525810420baab03ce1e0150c9d70305df0c56cdac19dce +generated: "2021-08-25T22:12:41.072958472+05:30" diff --git a/kubernetes/helm_charts/druid-cluster/templates/_helpers.tpl b/kubernetes/helm_charts/druid-cluster/templates/_helpers.tpl new file mode 100644 index 0000000000..f05bab6764 --- /dev/null +++ b/kubernetes/helm_charts/druid-cluster/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "druid-cluster.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "name" -}} +{{- default .Release.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "druid-cluster.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "druid-cluster.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +App config names +*/}} +{{- define "druid-cluster.appConfigName" -}} +{{- $.Release.Name -}} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "druid-cluster.selectorLabels" -}} +app.kubernetes.io/name: {{ include "druid-cluster.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/kubernetes/helm_charts/druid-cluster/templates/druid_statefulset.yaml b/kubernetes/helm_charts/druid-cluster/templates/druid_statefulset.yaml new file mode 100644 index 0000000000..bd28c18413 --- /dev/null +++ b/kubernetes/helm_charts/druid-cluster/templates/druid_statefulset.yaml @@ -0,0 +1,459 @@ +# This spec only works on a single node kubernetes cluster(e.g. typical k8s cluster setup for dev using kind/minikube or single node AWS EKS cluster etc) +# as it uses local disk as "deep storage". +# +apiVersion: "druid.apache.org/v1alpha1" +kind: "Druid" +metadata: + name: "{{ .Values.druid_cluster_type }}-{{ .Values.druid_env }}" +spec: + image: "{{ .Values.druid_image }}" + # Optionally specify image for all nodes. Can be specify on nodes also + # imagePullSecrets: + # - name: tutu + startScript: /druid.sh + podLabels: + environment: {{ .Values.druid_env }} + release: alpha + podAnnotations: + dummykey: dummyval + readinessProbe: + httpGet: + path: /status/health + port: 8088 + securityContext: + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + services: + - spec: + type: ClusterIP + clusterIP: None + commonConfigMountPath: "/opt/druid/conf/druid/cluster/_common" + # volumeMounts: + # - mountPath: /druid/data + # name: data-volume + # volumes: + # - name: data-volume + # emptyDir: {} + jvm.options: |- + -server + -XX:+UseG1GC + -XX:+ExitOnOutOfMemoryError + -XX:MaxDirectMemorySize=10240g + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + -Djava.io.tmpdir=/druid/data + log4j.config: |- + + + + + + + + + + + + + + common.runtime.properties: | + druid.extensions.loadList=[{{ .Values.druid_extensions_loadList }}] + # druid.extensions.directory=/opt/druid/extensions + + # Logging + # Log all runtime properties on startup. Disable to avoid logging properties on startup: + druid.startup.logging.logProperties=true + + # Zookeeper + # druid.zk.service.host=druid-cluster-zookeeper-headless.{{ .Values.druid_namespace }}.svc.cluster.local + druid.zk.service.host=druid-cluster-zookeeper-0.druid-cluster-zookeeper-headless.druid-raw.svc.cluster.local:2181,druid-cluster-zookeeper-1.druid-cluster-zookeeper-headless.druid-raw.svc.cluster.local:2181,druid-cluster-zookeeper-2.druid-cluster-zookeeper-headless.druid-raw.svc.cluster.local:2181 + + druid.zk.paths.base=/druid + + # Metadata storage + # For PostgreSQL: + druid.metadata.storage.type={{ .Values.druid_metadata_storage_type }} + druid.metadata.storage.connector.connectURI={{ .Values.druid_metadata_storage_connector_connectURI }} + druid.metadata.storage.connector.user={{ .Values.druid_metadata_storage_connector_user }} + druid.metadata.storage.connector.password={{ .Values.druid_metadata_storage_connector_password }} + + # Deep storage + #Using azure + druid.storage.type={{ .Values.druid_deepstorage_type }} + druid.azure.account={{ .Values.druid_azure_storage_account }} + druid.azure.key={{ .Values.druid_azure_storage_account_key }} + druid.azure.container={{ .Values.druid_azure_container }} + + # Indexing service logs + # For local disk (only viable in a cluster if this is a network mount): + druid.indexer.logs.type={{ .Values.druid_indexer_logs_type }} + druid.indexer.logs.container={{ .Values.druid_indexer_logs_container }} + druid.indexer.logs.prefix= {{ .Values.druid_indexer_logs_prefix }} + + # Service discovery + druid.selectors.indexing.serviceName={{ .Values.druid_selectors_indexing_serviceName }} + druid.selectors.coordinator.serviceName={{ .Values.druid_selectors_coordinator_serviceName }} + + # Monitoring + # druid.monitoring.monitors=[{{ .Values.druid_monitoring_monitors }}] + # druid.emitter=composing + # druid.emitter.composing.emitters=[{{ .Values.druid_emitter_composing_emitters }}] + # druid.emitter.logging.logLevel={{ .Values.druid_emitter_logging_logLevel }} + + {{- if .Values.druid_monitoring -}} + druid.emitter.graphite.port={{ .Values.druid_emitter_graphite_port }} + druid.emitter.graphite.hostname={{ .Values.druid_emitter_graphite_hostname }} + druid.emitter.graphite.protocol={{ .Values.druid_emitter_graphite_protocol }} + druid.emitter.graphite.eventConverter={{ .Values.druid_emitter_graphite_eventConverter }} + {{- end -}} + + # Storage type of double columns + # ommiting this will lead to index double as float at the storage layer + + druid.indexing.doubleStorage={{ .Values.druid_indexing_doubleStorage }} + + #Writing query logs into file + druid.request.logging.type={{ .Values.druid_request_logging_type }} + druid.request.logging.dir={{ .Values.druid_request_logging_dir }} + + druid.javascript.enabled=true + druid.sql.enable={{ .Values.druid_sql_enable }} + + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + + nodes: + brokers: + # Optionally specify for running broker as Deployment + kind: Deployment + nodeType: "broker" + # Optionally specify for broker nodes + # imagePullSecrets: + # - name: tutu + druid.port: {{ .Values.druid_broker_port }} + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/broker" + replicas: {{ .Values.druid_broker_replicas }} + runtime.properties: | + druid.service={{ .Values.druid_broker_service }} + + # HTTP server threads + druid.broker.http.numConnections={{ .Values.druid_broker_http_numConnections }} + druid.server.http.numThreads={{ .Values.druid_server_http_numThreads }} + + # Processing threads and buffers + druid.processing.buffer.sizeBytes={{ .Values.druid_broker_processing_buffer_sizeBytes }} + druid.processing.numThreads={{ .Values.druid_broker_processing_numThreads }} + + druid.javascript.enabled=true + druid.sql.enable={{ .Values.druid_sql_enable }} + + extra.jvm.options: |- + -Xms{{ .Values.druid_broker_min_heap_size }} + -Xmx{{ .Values.druid_broker_max_heap_size }} + -XX:MaxDirectMemorySize={{ .Values.druid_broker_max_direct_size }} + + # volumeMounts: + # - mountPath: /druid/data + # name: data-volume + # volumes: + # - name: data-volume + # emptyDir: {} + + readinessProbe: + httpGet: + path: /status/health + port: {{ .Values.druid_broker_port }} + + hpAutoscaler: + maxReplicas: 10 + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: druid-tiny-cluster-brokers + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 + + coordinators: + # Optionally specify for running coordinator as Deployment + kind: Deployment + nodeType: "coordinator" + druid.port: {{ .Values.druid_coordinator_port }} + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/master/coordinator-overlord" + replicas: {{ .Values.druid_coordinator_replicas }} + runtime.properties: | + druid.service={{ .Values.druid_coordinator_service }} + druid.coordinator.startDelay={{ .Values.druid_coordinator_startDelay }} + druid.coordinator.period={{ .Values.druid_coordinator_period }} + druid.coordinator.balancer.strategy={{ .Values.druid_coordinator_balancer_strategy }} + druid.coordinator.asOverlord.enabled=false + + extra.jvm.options: |- + -Xms{{ .Values.druid_coordinator_heap_size }} + -Xmx{{ .Values.druid_coordinator_heap_size }} + + # volumeMounts: + # - mountPath: /druid/data + # name: data-volume + # volumes: + # - name: data-volume + # emptyDir: {} + + readinessProbe: + httpGet: + path: /status/health + port: {{ .Values.druid_coordinator_port }} + + overlords: + # Optionally specify for running coordinator as Deployment + kind: Deployment + nodeType: "overlord" + druid.port: {{ .Values.druid_overlord_port }} + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/master/coordinator-overlord" + replicas: {{ .Values.druid_overlord_replicas }} + runtime.properties: | + druid.service={{ .Values.druid_overlord_service }} + + druid.indexer.queue.startDelay={{ .Values.druid_indexer_queue_startDelay }} + + druid.indexer.runner.type={{ .Values.druid_indexer_runner_type }} + druid.indexer.storage.type={{ .Values.druid_indexer_storage_type }} + + # Additional parameters for minor compaction + druid.indexer.tasklock.forceTimeChunkLock={{ .Values.druid_indexer_tasklock_forceTimeChunkLock }} + + extra.jvm.options: |- + -Xms{{ .Values.druid_overlord_heap_size }} + -Xmx{{ .Values.druid_overlord_heap_size }} + + # volumeMounts: + # - mountPath: /druid/data + # name: data-volume + # volumes: + # - name: data-volume + # emptyDir: {} + + readinessProbe: + httpGet: + path: /status/health + port: {{ .Values.druid_overlord_port }} + + historicals: + nodeType: "historical" + druid.port: {{ .Values.druid_historical_port }} + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/historical" + replicas: {{ .Values.druid_historical_replicas }} + readinessProbe: + httpGet: + path: /status/health + port: {{ .Values.druid_historical_port }} + runtime.properties: | + druid.service={{ .Values.druid_historical_service }} + + # HTTP server threads + druid.server.http.numThreads={{ .Values.druid_server_http_numThreads }} + + # Processing threads and buffers + druid.processing.buffer.sizeBytes={{ .Values.druid_historical_processing_buffer_sizeBytes }} + druid.processing.numThreads={{ .Values.druid_historical_processing_numThreads }} + druid.processing.numMergeBuffers={{ .Values.druid_historical_processing_numMergeBuffers }} + + # Segmentstorage + druid.segmentCache.locations=[{{ .Values.druid_segmentCache_locations }}] + + druid.segmentCache.numLoadingThreads={{ .Values.druid_segmentCache_numLoadingThreads }} + + # Caching + druid.historical.cache.useCache={{ .Values.druid_historical_cache_useCache }} + druid.historical.cache.populateCache={{ .Values.druid_historical_cache_populateCache }} + druid.historical.cache.unCacheable=[{{ .Values.druid_historical_cache_unCacheable }}] + druid.cache.type={{ .Values.druid_cache_type }} + druid.cache.sizeInBytes={{ .Values.druid_historical_cache_size }} + # druid.cache.expireAfter={{ .Values.druid_historical_cache_expiry }} + + extra.jvm.options: |- + -Xms{{ .Values.druid_historical_min_heap_size }} + -Xmx{{ .Values.druid_historical_max_heap_size }} + -XX:MaxDirectMemorySize={{ .Values.druid_historical_max_direct_size }} + + securityContext: + fsGroup: 0 + runAsUser: 0 + runAsGroup: 0 + + volumeMounts: + - mountPath: /druid/data + name: historical-volume + volumeClaimTemplates: + - metadata: + name: historical-volume + spec: + storageClassName: "{{- .Values.storageClass }}" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.druid_historical_persistent_volume_size }} + + # volumes: + # - name: historical-volume + # persistentVolumeClaim: + # claimName: historical-pv-claim + # # - name: data-volume + # # emptyDir: {} + # volumeMounts: + # - mountPath: "/var/segments" + # name: historical-volume + # # - mountPath: /druid/data + # # name: data-volume + + middlemanagers: + nodeType: "middleManager" + druid.port: {{ .Values.druid_middlemanager_port }} + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/middleManager" + replicas: {{ .Values.druid_middlemanager_replicas }} + runtime.properties: | + druid.service={{ .Values.druid_middlemanager_service }} + + # Number of tasks per middleManager + druid.worker.capacity={{ .Values.druid_worker_capacity }} + + # Task launch parameters + druid.indexer.runner.javaOpts={{ .Values.druid_indexer_runner_javaOpts }} + druid.indexer.task.baseTaskDir={{ .Values.druid_indexer_task_baseTaskDir }} + + # Peon properties + druid.indexer.fork.property.druid.processing.buffer.sizeBytes={{ .Values.druid_indexer_fork_property_druid_processing_buffer_sizeBytes }} + druid.indexer.fork.property.druid.processing.numThreads={{ .Values.druid_indexer_fork_property_druid_processing_numThreads }} + druid.indexer.fork.property.druid.server.http.numThreads={{ .Values.druid_indexer_fork_property_druid_server_http_numThreads }} + + #Additional Parameters + druid.indexer.task.restoreTasksOnRestart={{ .Values.druid_indexer_task_restoreTasksOnRestart }} + druid.indexer.task.defaultHadoopCoordinates=[\"org.apache.hadoop:hadoop-client:2.8.3\"] + + extra.jvm.options: |- + -Xmx{{ .Values.druid_middlemanager_heap_size }} + -Xms{{ .Values.druid_middlemanager_heap_size }} + + # services: + # - spec: + # clusterIP: None + # ports: + # - name: middlemanager-port + # port: {{ .Values.druid_middlemanager_port }} + # targetPort: {{ .Values.druid_middlemanager_port }} + # type: ClusterIP + + readinessProbe: + initialDelaySeconds: 30 + httpGet: + path: /status/health + port: {{ .Values.druid_middlemanager_port }} + + securityContext: + fsGroup: 0 + runAsUser: 0 + runAsGroup: 0 + + volumeMounts: + - mountPath: /druid/data + name: middlemanager-volume + volumeClaimTemplates: + - metadata: + name: middlemanager-volume + spec: + storageClassName: "{{- .Values.storageClass }}" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.druid_middlemanager_persistent_volume_size }} + + # volumeMounts: + # - mountPath: /var/task + # name: middlemanager-volume + # # - mountPath: /druid/data + # # name: data-volume + # volumes: + # - name: middlemanager-volume + # persistentVolumeClaim: + # claimName: middlemanager-pv-claim + # # - name: data-volume + # # emptyDir: {} + + routers: + nodeType: "router" + druid.port: {{ .Values.druid_router_plaintextPort }} + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/router" + replicas: {{ .Values.druid_router_replicas }} + services: + - spec: + type: ClusterIP + ports: + - name: druidrouterport + port: 80 + targetPort: 8888 + protocol: TCP + ingressAnnotations: + name: router-ingress + nginx.ingress.kubernetes.io/rewrite-target: /$1 + ingress: + ingressClassName: nginx + rules: + - host: "*.nip.io" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: druid-{{ .Values.druid_cluster_type }}-{{ .Values.druid_env }}-routers + port: + name: druidrouterport + + runtime.properties: | + druid.service={{ .Values.druid_router_service }} + # druid.plaintextPort={{ .Values.druid_router_plaintextPort }} + + # HTTP proxy + druid.router.http.numConnections={{ .Values.druid_router_http_numConnections }} + druid.router.http.readTimeout={{ .Values.druid_router_http_readTimeout }} + druid.router.http.numMaxThreads={{ .Values.druid_router_http_numMaxThreads }} + druid.server.http.numThreads={{ .Values.druid_server_http_numThreads }} + + # Service discovery + druid.router.defaultBrokerServiceName={{ .Values.druid_broker_service }} + druid.router.coordinatorServiceName={{ .Values.druid_coordinator_service }} + + # Management proxy to coordinator / overlord: required for unified web console. + druid.router.managementProxy.enabled={{ .Values.druid_router_managementProxy_enabled }} + + extra.jvm.options: |- + -Xms{{ .Values.druid_router_heap_size }} + -Xmx{{ .Values.druid_router_heap_size }} + + # volumeMounts: + # - mountPath: /druid/data + # name: data-volume + # volumes: + # - name: data-volume + # emptyDir: {} + + readinessProbe: + httpGet: + path: /status/health + port: {{ .Values.druid_router_plaintextPort }} diff --git a/kubernetes/helm_charts/druid-cluster/values.j2 b/kubernetes/helm_charts/druid-cluster/values.j2 new file mode 100644 index 0000000000..6527a71bd8 --- /dev/null +++ b/kubernetes/helm_charts/druid-cluster/values.j2 @@ -0,0 +1,272 @@ +druid_env: "{{ env }}" +druid_cluster_type: "{{ cluster_type }}" +druid_image: "{{ druid_image }}" +druid_namespace: "{{ druid_namespace }}" +dp_vault_pgdb_admin_password: postgres +druid_monitoring: {{ druid_monitoring }} + +storageClass: {{ persistent_storage_class }} + +######################### Druid common variables ######################## + +druid_directory: "/opt/druid" +druid_extensions_loadList: '{{ druid_extensions_loadList }}' + +# Logging +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties: true + +# Druid Metadata Store +druid_metadata_storage_type: "postgresql" +druid_metadata_storage_connector_connectURI: "jdbc:postgresql://{{ druid_configs[cluster_type].druid_postgres_host }}:{{ druid_configs[cluster_type].druid_postgres_port }}/{{ druid_configs[cluster_type].druid_postgres_db }}" +druid_metadata_storage_connector_user: {{ druid_configs[cluster_type].druid_postgres_user }} +druid_metadata_storage_connector_password: {{ druid_configs[cluster_type].druid_postgres_pass }} + +# Druid Storage Type +druid_deepstorage_type: azure +druid_azure_storage_account: "{{ sunbird_private_storage_account_name }}" +druid_azure_storage_account_key: "{{ sunbird_private_storage_account_key }}" +druid_azure_container: "{{ druid_configs[cluster_type].azure_container }}" + +# Indexing service logs +# For local disk (only viable in a cluster_type if this is a network mount): +druid_indexer_logs_type: "{{ druid_indexing_logs_type }}" +druid_indexer_logs_container: "{{ druid_configs[cluster_type].druid_log_azure_container }}" +druid_indexer_logs_prefix: "{{ druid_configs[cluster_type].druid_log_azure_folder }}" + +# Service discovery +druid_selectors_indexing_serviceName: {{ druid_overlord_service }} +druid_selectors_coordinator_serviceName: {{ druid_coordinator_service }} + +# Monitoring +druid_monitoring_monitors: '{{ druid_common_monitors }}' +druid_emitter: composing +druid_emitter_composing_emitters: '{{ druid_common_emitters }}' +druid_emitter_logging_logLevel: {{ druid_emitter_logging_logLevel }} + +{% if druid_monitoring %} +druid_emitter_graphite_port: {{ druid_graphite_port | d(9109)}} +druid_emitter_graphite_hostname: {{ druid_graphite_host | d("localhost") }} +druid_emitter_graphite_protocol: plaintext +druid_emitter_graphite_eventConverter: '{"type":"whiteList", "namespacePrefix": "{{ druid_graphite_prefix }}", "ignoreHostname":false, "ignoreServiceName":false ,"mapPath":"{{ druid_whitelist_filepath }}"}' +{% endif %} + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer +druid_indexing_doubleStorage: double + +#Writing query logs into file +druid_request_logging_type: {{ druid_request_logging_type }} +druid_request_logging_dir: "{{ druid_log_dir }}" + +druid_javascript_enabled: true +druid_sql_enable: {{ enable_druid_sql | lower }} + +####################### Druid Broker Variables ########################## + +druid_broker_service: {{ druid_broker_service }} +druid_broker_port: {{ druid_broker_port }} +druid_broker_min_heap_size: {{ druid_configs[cluster_type].druid_broker_min_heap_size }} +druid_broker_max_heap_size: {{ druid_configs[cluster_type].druid_broker_max_heap_size }} +druid_broker_max_direct_size: {{ druid_configs[cluster_type].druid_broker_max_direct_size }} + +# HTTP server threads +druid_broker_http_numConnections: {{ druid_configs[cluster_type].druid_broker_http_numConnections }} +druid_server_http_numThreads: {{ druid_configs[cluster_type].druid_broker_server_http_numThread }} + +# Processing threads and buffers +druid_broker_processing_buffer_sizeBytes: {{ druid_configs[cluster_type].druid_broker_processing_bufferBytes }} +druid_broker_processing_numThreads: {{ druid_configs[cluster_type].druid_broker_processing_threads }} + +druid_javascript.enabled: true +druid_sql_enable: {{ enable_druid_sql | lower }} + +druid_broker_replicas: {{ druid_configs[cluster_type].druid_broker_replicas }} + +##################### Druid Coordinator Variables ####################### + +druid_coordinator_service: {{ druid_coordinator_service }} +druid_coordinator_port: {{ druid_coordinator_port }} +druid_coordinator_heap_size: {{ druid_configs[cluster_type].druid_coordinator_heap_size }} + +druid_coordinator_startDelay: {{ druid_configs[cluster_type].druid_coordinator_startDelay }} +druid_coordinator_period: {{ druid_configs[cluster_type].druid_coordinator_period }} +druid_coordinator_balancer_strategy: {{ druid_configs[cluster_type].druid_coordinator_balance_strategy }} + +druid_coordinator_replicas: {{ druid_configs[cluster_type].druid_coordinator_replicas }} + +####################### Druid Overlord Variables ######################## + +druid_overlord_service: {{ druid_overlord_service }} +druid_overlord_port: {{ druid_overlord_port }} +druid_overlord_heap_size: {{ druid_configs[cluster_type].druid_overlord_heap_size }} + +druid_indexer_queue_startDelay: {{ druid_configs[cluster_type].druid_indexing_queue_startDelay }} + +druid_indexer_runner_type: remote +druid_indexer_storage_type: {{ druid_indexing_storage_type }} + +# Additional parameters for minor compaction +druid_indexer_tasklock_forceTimeChunkLock: false + +druid_overlord_replicas: {{ druid_configs[cluster_type].druid_overlord_replicas }} + +###################### Druid Historical Variables ####################### + +druid_historical_service: {{ druid_historical_service }} +druid_historical_port: {{ druid_historical_port }} +druid_historical_min_heap_size: {{ druid_configs[cluster_type].druid_historical_min_heap_size }} +druid_historical_max_heap_size: {{ druid_configs[cluster_type].druid_historical_max_heap_size }} +druid_historical_max_direct_size: {{ druid_configs[cluster_type].druid_historical_max_direct_size }} + +# HTTP server threads +druid_server_http_numThreads: {{ druid_configs[cluster_type].druid_historical_server_http_numThread }} + +# Processing threads and buffers +druid_historical_processing_buffer_sizeBytes: {{ druid_configs[cluster_type].druid_historical_processing_bufferBytes }} +druid_historical_processing_numThreads: {{ druid_configs[cluster_type].druid_historical_processing_threads }} +druid_historical_processing_numMergeBuffers: {{ druid_configs[cluster_type].druid_historical_processing_num_merge_buffers }} + +druid_query_ondiskstorage_enabled: {{ druid_configs[cluster_type].druid_query_ondiskstorage_enabled }} +druid_query_groupBy_maxMergingDictionarySize: {{ druid_configs[cluster_type].druid_historical_maxMergingDictionarySize }} +druid_query_groupBy_maxOnDiskStorage: {{ druid_configs[cluster_type].druid_query_groupBy_maxOnDiskStorage }} + +# Segmentstorage +druid_segmentCache_locations: '{% for index in range(5) %}{"path": "{{ druid_configs[cluster_type].druid_historical_segmentcache_path }}{{index}}", "maxSize": {{ druid_configs[cluster_type].druid_historical_segmentcache_size }}}{% if not loop.last %}, {% endif %} +{% endfor %}' + +druid_historical_persistent_volume_size: {{ druid_configs[cluster_type].druid_historical_persistent_volume_size }} + +druid_segmentCache_numLoadingThreads: {{ druid_configs[cluster_type].druid_historical_segmentcache_numloadingthreads }} + +# Caching +druid_historical_cache_useCache: {{ druid_configs[cluster_type].druid_historical_enable_cache }} +druid_historical_cache_populateCache: true +druid_historical_cache_unCacheable: '{{ druid_historical_cache_unCacheable }}' +druid_cache_type: caffeine +druid_historical_cache_size: {{ druid_historical_cache_size }} +druid_historical_cache_expiry: {{ druid_historical_cache_expiry }} + +druid_historical_replicas: {{ druid_configs[cluster_type].druid_historical_replicas }} + +#################### Druid Middlemanager Variables ###################### + +druid_middlemanager_service: {{ druid_middlemanager_service }} +druid_middlemanager_port: {{ druid_middlemanager_port }} +druid_middlemanager_heap_size: {{ druid_configs[cluster_type].druid_middlemanager_heap_size }} + +druid_middlemanager_persistent_volume_size: {{ druid_configs[cluster_type].druid_middlemanager_persistent_volume_size }} + +# Number of tasks per middleManager +druid_worker_capacity: {{ druid_configs[cluster_type].druid_middlemanager_worker_cap }} +druid_middlemanager_heap_size: {{ druid_configs[cluster_type].druid_middlemanager_heap_size }} +druid_middlemanager_peon_heap_size: {{ druid_configs[cluster_type].druid_middlemanager_peon_heap_size}} + +# Task launch parameters +druid_indexer_runner_javaOpts: "{{ druid_configs[cluster_type].druid_mm_java_opts_array }}" +druid_indexer_task_baseTaskDir: "{{ druid_indexing_task_basedir }}" + +# Peon properties +druid_indexer_fork_property_druid_processing_buffer_sizeBytes: {{ druid_configs[cluster_type].druid_middlemanager_peon_processing_bufferBytes }} +druid_indexer_fork_property_druid_processing_numThreads: {{ druid_configs[cluster_type].druid_middlemanager_peon_processing_threads }} +druid_indexer_fork_property_druid_server_http_numThreads: {{ druid_configs[cluster_type].druid_middlemanager_peon_server_http_numThread }} + +#Additional Parameters +druid_indexer_task_restoreTasksOnRestart: true + +druid_middlemanager_replicas: {{ druid_configs[cluster_type].druid_middlemanager_replicas }} + +######################## Druid Router Variables ######################### + +druid_router_service: {{ druid_router_service }} +druid_router_plaintextPort: {{ druid_router_plaintextPort }} +druid_router_heap_size: {{ druid_configs[cluster_type].druid_router_heap_size }} + +# HTTP proxy +druid_router_http_numConnections: {{ druid_configs[cluster_type].druid_router_http_numConnections }} +druid_router_http_readTimeout: {{ druid_configs[cluster_type].druid_router_http_readTimeout }} +druid_router_http_numMaxThreads: {{ druid_configs[cluster_type].druid_router_http_numMaxThreads }} +druid_server_http_numThreads: {{ druid_configs[cluster_type].druid_server_http_numThreads }} + +# Service discovery +druid_router_defaultBrokerServiceName: {{ druid_broker_service }} +druid_router_coordinatorServiceName: {{ druid_coordinator_service }} + +# Management proxy to coordinator / overlord: required for unified web console. +druid_router_managementProxy_enabled: {{ druid_configs[cluster_type].druid_router_managementProxy_enabled }} + +druid_router_replicas: {{ druid_configs[cluster_type].druid_router_replicas }} + +#################### Zookeeper Variables ###################### + +zookeeper: + image: + registry: docker.io + repository: bitnami/zookeeper + tag: 3.6-debian-10 + pullPolicy: Always + + heapSize: 256 + replicaCount: 3 + + resources: + requests: + memory: 256Mi + cpu: 250m + + tickTime: 2000 + initLimit: 10 + syncLimit: 5 + preAllocSize: 65536 + snapCount: 100000 + maxClientCnxns: 60 + fourlwCommandsWhitelist: srvr, mntr, ruok + listenOnAllIPs: false + allowAnonymousLogin: true + autopurge: + snapRetainCount: 3 + purgeInterval: 0 + maxSessionTimeout: 40000 + + allowAnonymousLogin: true + + minServerId: 1 + + securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + + service: + type: ClusterIP + # loadBalancerIP: "" + port: 2181 + followerPort: 2888 + electionPort: 3888 + nodePorts: + client: "" + clientTls: "" + publishNotReadyAddresses: true + tlsClientPort: 3181 + disableBaseClientPort: false + annotations: {} + headless: + annotations: {} \ No newline at end of file diff --git a/kubernetes/helm_charts/druid-operator/Chart.yaml b/kubernetes/helm_charts/druid-operator/Chart.yaml new file mode 100644 index 0000000000..6588f1e73d --- /dev/null +++ b/kubernetes/helm_charts/druid-operator/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: druid-operator +description: Druid Kubernetes Operator + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 0.0.7 diff --git a/kubernetes/helm_charts/druid-operator/templates/NOTES.txt b/kubernetes/helm_charts/druid-operator/templates/NOTES.txt new file mode 100644 index 0000000000..88a89946c8 --- /dev/null +++ b/kubernetes/helm_charts/druid-operator/templates/NOTES.txt @@ -0,0 +1 @@ +Refer to https://github.com/druid-io/druid-operator/blob/master/docs/README.md to get started. diff --git a/kubernetes/helm_charts/druid-operator/templates/_helpers.tpl b/kubernetes/helm_charts/druid-operator/templates/_helpers.tpl new file mode 100644 index 0000000000..628079bfe6 --- /dev/null +++ b/kubernetes/helm_charts/druid-operator/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "druid-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "druid-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "druid-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "druid-operator.labels" -}} +helm.sh/chart: {{ include "druid-operator.chart" . }} +{{ include "druid-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "druid-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "druid-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "druid-operator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "druid-operator.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/druid-operator/templates/crds/druid.apache.org_druids.yaml b/kubernetes/helm_charts/druid-operator/templates/crds/druid.apache.org_druids.yaml new file mode 100644 index 0000000000..ed176311c0 --- /dev/null +++ b/kubernetes/helm_charts/druid-operator/templates/crds/druid.apache.org_druids.yaml @@ -0,0 +1,4467 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.0-beta.0 + creationTimestamp: null + name: druids.druid.apache.org +spec: + group: druid.apache.org + names: + kind: Druid + listKind: DruidList + plural: druids + singular: druid + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + common.runtime.properties: + type: string + commonConfigMountPath: + type: string + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + deepStorage: + properties: + spec: + format: byte + type: string + type: + type: string + required: + - spec + - type + type: object + deleteOrphanPvc: + type: boolean + disablePVCDeletionFinalizer: + type: boolean + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + type: object + type: array + forceDeleteStsPodOnError: + type: boolean + ignored: + type: boolean + image: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + jvm.options: + type: string + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + log4j.config: + type: string + metadataStore: + properties: + spec: + format: byte + type: string + type: + type: string + required: + - spec + - type + type: object + nodeSelector: + additionalProperties: + type: string + type: object + nodes: + additionalProperties: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + druid.port: + format: int32 + type: integer + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + type: object + type: array + extra.jvm.options: + type: string + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + containerResource: + properties: + container: + type: string + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object + image: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + ingress: + properties: + defaultBackend: + properties: + resource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + service: + properties: + name: + type: string + port: + properties: + name: + type: string + number: + format: int32 + type: integer + type: object + required: + - name + type: object + type: object + ingressClassName: + type: string + rules: + items: + properties: + host: + type: string + http: + properties: + paths: + items: + properties: + backend: + properties: + resource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + service: + properties: + name: + type: string + port: + properties: + name: + type: string + number: + format: int32 + type: integer + type: object + required: + - name + type: object + type: object + path: + type: string + pathType: + type: string + required: + - backend + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + items: + properties: + hosts: + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + ingressAnnotations: + additionalProperties: + type: string + type: object + jvm.options: + type: string + kind: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + log4j.config: + type: string + maxSurge: + format: int32 + type: integer + maxUnavailable: + format: int32 + type: integer + nodeConfigMountPath: + type: string + nodeType: + type: string + persistentVolumeClaim: + items: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + type: array + podAnnotations: + additionalProperties: + type: string + type: object + podDisruptionBudgetSpec: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + minAvailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: object + podLabels: + additionalProperties: + type: string + type: object + podManagementPolicy: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + replicas: + format: int32 + minimum: 0 + type: integer + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + runtime.properties: + type: string + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + services: + items: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + properties: + ingress: + items: + properties: + hostname: + type: string + ip: + type: string + ports: + items: + properties: + error: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + type: array + startUpProbes: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + maxSkew: + format: int32 + type: integer + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + updateStrategy: + properties: + rollingUpdate: + properties: + partition: + format: int32 + type: integer + type: object + type: + type: string + type: object + volumeClaimTemplates: + items: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - druid.port + - nodeConfigMountPath + - nodeType + - replicas + - runtime.properties + type: object + type: object + podAnnotations: + additionalProperties: + type: string + type: object + podLabels: + additionalProperties: + type: string + type: object + podManagementPolicy: + type: string + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + rollingDeploy: + type: boolean + securityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + services: + items: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + properties: + ingress: + items: + properties: + hostname: + type: string + ip: + type: string + ports: + items: + properties: + error: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + type: array + startScript: + type: string + startUpProbes: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + updateStrategy: + properties: + rollingUpdate: + properties: + partition: + format: int32 + type: integer + type: object + type: + type: string + type: object + volumeClaimTemplates: + items: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + readOnly: + type: boolean + volumeClaimTemplate: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + zookeeper: + properties: + spec: + format: byte + type: string + type: + type: string + required: + - spec + - type + type: object + required: + - common.runtime.properties + - commonConfigMountPath + - nodes + - startScript + type: object + status: + properties: + configMaps: + items: + type: string + type: array + deployments: + items: + type: string + type: array + druidNodeStatus: + properties: + druidNode: + type: string + druidNodeConditionStatus: + type: string + druidNodeConditionType: + type: string + reason: + type: string + type: object + hpAutoscalers: + items: + type: string + type: array + ingress: + items: + type: string + type: array + persistentVolumeClaims: + items: + type: string + type: array + podDisruptionBudgets: + items: + type: string + type: array + pods: + items: + type: string + type: array + services: + items: + type: string + type: array + statefulSets: + items: + type: string + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubernetes/helm_charts/druid-operator/templates/deployment.yaml b/kubernetes/helm_charts/druid-operator/templates/deployment.yaml new file mode 100644 index 0000000000..f4bdc5ae8d --- /dev/null +++ b/kubernetes/helm_charts/druid-operator/templates/deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: {{ .Release.Namespace }} + name: {{ include "druid-operator.fullname" . }} + labels: + {{- include "druid-operator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "druid-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "druid-operator.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "druid-operator.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + command: + - /manager + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ tpl $value $ | quote }} + {{- end }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/kubernetes/helm_charts/druid-operator/templates/role.yaml b/kubernetes/helm_charts/druid-operator/templates/role.yaml new file mode 100644 index 0000000000..b12ed0d85e --- /dev/null +++ b/kubernetes/helm_charts/druid-operator/templates/role.yaml @@ -0,0 +1,95 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.env.WATCH_NAMESPACE }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: +{{- if .Values.env.WATCH_NAMESPACE }} + namespace: {{ .Values.env.WATCH_NAMESPACE }} +{{- end }} + name: {{ template "druid-operator.fullname" . }} + labels: + {{- include "druid-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - pods + - configmaps + - services + - persistentvolumeclaims + verbs: + - list + - watch + - create + - update + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - list + - watch + - create + - update + - delete + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - list + - watch + - create + - update + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - list + - watch + - create + - update + - apiGroups: + - druid.apache.org + resources: + - druids + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - druid.apache.org + resources: + - druids/status + verbs: + - get + - update + - patch +{{- end }} diff --git a/kubernetes/helm_charts/druid-operator/templates/role_binding.yaml b/kubernetes/helm_charts/druid-operator/templates/role_binding.yaml new file mode 100644 index 0000000000..2587fe6047 --- /dev/null +++ b/kubernetes/helm_charts/druid-operator/templates/role_binding.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.env.WATCH_NAMESPACE }} +kind: RoleBinding +{{- else }} +kind: ClusterRoleBinding +{{- end }} +metadata: +{{- if .Values.env.WATCH_NAMESPACE }} + namespace: {{ .Values.env.WATCH_NAMESPACE }} +{{- end }} + name: {{ template "druid-operator.fullname" . }} + labels: + {{- include "druid-operator.labels" . | nindent 4 }} +subjects: +- kind: ServiceAccount + name: {{ include "druid-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: {{ if .Values.env.WATCH_NAMESPACE }} Role {{ else }} ClusterRole {{ end }} + name: {{ template "druid-operator.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/kubernetes/helm_charts/druid-operator/templates/serviceaccount.yaml b/kubernetes/helm_charts/druid-operator/templates/serviceaccount.yaml new file mode 100644 index 0000000000..d1af85fe91 --- /dev/null +++ b/kubernetes/helm_charts/druid-operator/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: {{ .Release.Namespace }} + name: {{ include "druid-operator.serviceAccountName" . }} + labels: + {{- include "druid-operator.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/druid-operator/values.yaml b/kubernetes/helm_charts/druid-operator/values.yaml new file mode 100644 index 0000000000..75ab618eca --- /dev/null +++ b/kubernetes/helm_charts/druid-operator/values.yaml @@ -0,0 +1,63 @@ +# Default values for druid-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +env: + DENY_LIST: "default,kube-system" # Comma-separated list of namespaces to ignore + RECONCILE_WAIT: "10s" # Reconciliation delay + WATCH_NAMESPACE: "" # Namespace to watch or empty string to watch all namespaces + +replicaCount: 1 + +image: + repository: druidio/druid-operator + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +rbac: + create: true + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/.helmignore b/kubernetes/helm_charts/monitoring/additional-scrape-configs/.helmignore new file mode 100644 index 0000000000..50af031725 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/Chart.yaml b/kubernetes/helm_charts/monitoring/additional-scrape-configs/Chart.yaml new file mode 100644 index 0000000000..2ac719df1c --- /dev/null +++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: additional-scrape-configs +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 1.16.0 diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/_helpers.tpl b/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/_helpers.tpl new file mode 100644 index 0000000000..c66316674f --- /dev/null +++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "additional-scrape-configs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "additional-scrape-configs.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "additional-scrape-configs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "additional-scrape-configs.labels" -}} +helm.sh/chart: {{ include "additional-scrape-configs.chart" . }} +{{ include "additional-scrape-configs.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "additional-scrape-configs.selectorLabels" -}} +app.kubernetes.io/name: {{ include "additional-scrape-configs.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "additional-scrape-configs.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "additional-scrape-configs.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/secrets.yaml b/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/secrets.yaml new file mode 100644 index 0000000000..cbc1ca2180 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/secrets.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.fullnameOverride }}-prometheus-scrape-confg + namespace: {{ default .Values.namespace .Release.Namespace }} + labels: + {{- include "additional-scrape-configs.labels" . | nindent 4 }} + app: {{ .Values.fullnameOverride }}-scrape-confg +data: + additional-scrape-configs.yaml: {{ toYaml .Values.scrapeconfig | b64enc | quote }} diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/values.yaml b/kubernetes/helm_charts/monitoring/additional-scrape-configs/values.yaml new file mode 100644 index 0000000000..9d81e6af57 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/values.yaml @@ -0,0 +1,10 @@ +# This value will help to scrape external configs, +# which your prometheus have to watch +scrapeconfig: [] + # - job_name: 'Druid' + # metrics_path: /metrics/druid + # static_configs: + # - targets: ["11.2.4.31:9000"] + # labels: + # "service_name": "druid" +fullnameOverride: "sunbird-monitoring" diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs.yaml b/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs.yaml index 7374424f24..56beb618ef 100644 --- a/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs.yaml +++ b/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs.yaml @@ -17,82 +17,422 @@ spec: for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "telemetry-extractor-taskmanager-prometheus" }) > {{ .Values.telemetry_extractor_threshold_critical }} + message: The lag for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_threshold_critical | int }} summary: telemetry-extractor lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: telemetry-extractor + alertname: FlinkJobsLag - alert: pipeline-preprocessor lag critical expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "pipeline-preprocessor-taskmanager-prometheus" }) > {{ .Values.pipeline_preprocessor_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "pipeline-preprocessor-taskmanager-prometheus" }) > {{ .Values.pipeline_preprocessor_threshold_critical }} + message: The lag for pipeline-preprocessor job is more than threshold {{ .Values.pipeline_preprocessor_threshold_critical | int }} summary: pipeline-preprocessor lag is critical - - - alert: de-normalization lag critical - expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_critical }} + lag: {{`({{ humanize $value }})`}} + job_id: pipeline-preprocessor + alertname: FlinkJobsLag + + - alert: cb-preprocessor lag critical + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "cb-preprocessor-taskmanager-prometheus" }) > {{ .Values.cb_preprocessor_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_critical }} - summary: de-normalization lag is critical + message: The lag for cb-preprocessor job is more than threshold {{ .Values.cb_preprocessor_threshold_critical | int }} + summary: cb-preprocessor lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: cb-preprocessor + alertname: FlinkJobsLag + + - alert: de-normalization-primary lag critical + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-primary-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The lag for de-normalization-primary job is more than threshold {{ .Values.de_normalization_threshold_critical | int }} + summary: de-normalization-primary lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: de-normalization-primary + alertname: FlinkJobsLag + + - alert: de-normalization-secondary lag critical + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-secondary-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The lag for de-normalization-secondary job is more than threshold {{ .Values.de_normalization_threshold_critical | int }} + summary: de-normalization-secondary lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: de-normalization-secondary + alertname: FlinkJobsLag - alert: druid-validator lag critical expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "druid-validator-taskmanager-prometheus" }) > {{ .Values.druid_validator_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "drud-validator-taskmanager-prometheus" }) > {{ .Values.druid_validator_threshold_critical }} + message: The lag for druid-validator job is more than threshold {{ .Values.druid_validator_threshold_critical | int }} summary: druid-validator lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: druid-validator + alertname: FlinkJobsLag - alert: assessment-aggregator lag critical expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "assessment-aggregator-taskmanager-prometheus" }) > {{ .Values.assessment_aggregator_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "assessment-aggregator-taskmanager-prometheus" }) > {{ .Values.assessment_aggregator_threshold_critical }} + message: The lag for assessment-aggregator job is more than threshold {{ .Values.assessment_aggregator_threshold_critical | int }} summary: assessment-aggregator lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: assessment-aggregator + alertname: FlinkJobsLag - alert: content-cache-updater lag critical expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "content-cache-updater-taskmanager-prometheus" }) > {{ .Values.content_cache_updater_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "content-cache-updater-taskmanager-prometheus" }) > {{ .Values.content_cache_updater_threshold_critical }} + message: The lag for content-cache-updater job is more than threshold {{ .Values.content_cache_updater_threshold_critical | int }} summary: content-cache-updater lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: content-cache-updater + alertname: FlinkJobsLag - alert: user-cache-updater lag critical expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_critical }} + message: The lag for user-cache-updater job is more than threshold {{ .Values.user_cache_updater_threshold_critical | int }} summary: user-cache-updater lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: user-cache-updater + alertname: FlinkJobsLag - alert: summary-denormalization lag critical expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "summary-denormalization-taskmanager-prometheus" }) > {{ .Values.summary_denormalization_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "summary-denormalization-taskmanager-prometheus" }) > {{ .Values.summary_denormalization_threshold_critical }} + message: The lag for summary-denormalization job is more than threshold {{ .Values.summary_denormalization_threshold_critical | int }} summary: summary-denormalization lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: summary-denormalization + alertname: FlinkJobsLag - alert: device-profile-updater lag critical expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "device-profile-updater-taskmanager-prometheus" }) > {{ .Values.device_profile_updater_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "device-profile-updater-taskmanager-prometheus" }) > {{ .Values._device_profile_updater_threshold_critical }} + message: The lag for device-profile-updater job is more than threshold {{ .Values.device_profile_updater_threshold_critical | int }} summary: device-profile-updater lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: device-profile-updater + alertname: FlinkJobsLag + + - alert: user-cache-updater-v2 lag critical + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-v2-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_critical}} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The lag for user-cache-updater-v2 job is more than threshold {{ .Values.user_cache_updater_threshold_critical | int }} + summary: user-cache-updater-v2 lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: user-cache-updater-v2 + alertname: FlinkJobsLag + + - alert: ingest-router lag critical + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "ingest-router-taskmanager-prometheus" }) > {{ .Values.ingest_router_threshold_critical}} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The lag for ingest-router job is more than threshold {{ .Values.ingest_router_threshold_critical | int }} + summary: ingest-router lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: ingest-router + alertname: FlinkJobsLag + +### Dp Flink jobs lag alertrules warning + + - alert: telemetry-extractor lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "telemetry-extractor-taskmanager-prometheus" }) > {{ .Values.telemetry_extractor_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "telemetry-extractor-taskmanager-prometheus" }) < {{ .Values.telemetry_extractor_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_threshold_warning | int }} + summary: telemetry-extractor lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: telemetry-extractor + alertname: FlinkJobsLag + + - alert: pipeline-preprocessor lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "pipeline-preprocessor-taskmanager-prometheus" }) > {{ .Values.pipeline_preprocessor_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "pipeline-preprocessor-taskmanager-prometheus" }) < {{ .Values.pipeline_preprocessor_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for pipeline-preprocessor job is more than threshold {{ .Values.pipeline_preprocessor_threshold_warning | int }} + summary: pipeline-preprocessor lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: pipeline-preprocessor + alertname: FlinkJobsLag + + - alert: cb-preprocessor lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "cb-preprocessor-taskmanager-prometheus" }) > {{ .Values.cb_preprocessor_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "cb-preprocessor-taskmanager-prometheus" }) < {{ .Values.cb_preprocessor_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for cb-preprocessor job is more than threshold {{ .Values.cb_preprocessor_threshold_warning | int }} + summary: cb-preprocessor lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: cb-preprocessor + alertname: FlinkJobsLag + - alert: de-normalization-secondary lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-secondary-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-secondary-taskmanager-prometheus" }) < {{ .Values.de_normalization_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for de-normalization-secondary job is more than threshold {{ .Values.de_normalization_threshold_warning | int }} + summary: de-normalization-secondary lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: de-normalization-secondary + alertname: FlinkJobsLag + + - alert: de-normalization-primary lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-primary-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-primary-taskmanager-prometheus" }) < {{ .Values.de_normalization_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for de-normalization-primary job is more than threshold {{ .Values.de_normalization_threshold_warning | int }} + summary: de-normalization-primary lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: de-normalization-primary + alertname: FlinkJobsLag + + - alert: druid-validator lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "druid-validator-taskmanager-prometheus" }) > {{ .Values.druid_validator_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "druid-validator-taskmanager-prometheus" }) > {{ .Values.druid_validator_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "druid-validator-taskmanager-prometheus" }) < {{ .Values.druid_validator_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for druid-validator job is more than threshold {{ .Values.druid_validator_threshold_warning | int }} + summary: druid-validator lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: druid-validator + alertname: FlinkJobsLag + + - alert: assessment-aggregator lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "assessment-aggregator-taskmanager-prometheus" }) > {{ .Values.assessment_aggregator_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "assessment-aggregator-taskmanager-prometheus" }) < {{ .Values.assessment_aggregator_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for assessment-aggregator job is more than threshold {{ .Values.assessment_aggregator_threshold_warning | int }} + summary: assessment-aggregator lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: assessment-aggregator + alertname: FlinkJobsLag + + - alert: content-cache-updater lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "content-cache-updater-taskmanager-prometheus" }) > {{ .Values.content_cache_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "content-cache-updater-taskmanager-prometheus" }) < {{ .Values.content_cache_updater_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for content-cache-updater job is more than threshold {{ .Values.content_cache_updater_threshold_warning | int }} + summary: content-cache-updater lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: content-cache-updater + alertname: FlinkJobsLag + + - alert: user-cache-updater lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-taskmanager-prometheus" }) < {{ .Values.user_cache_updater_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for user-cache-updater job is more than threshold {{ .Values.user_cache_updater_threshold_warning | int }} + summary: user-cache-updater lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: user-cache-updater + alertname: FlinkJobsLag + + - alert: summary-denormalization lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "summary-denormalization-taskmanager-prometheus" }) > {{ .Values.summary_denormalization_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "summary-denormalization-taskmanager-prometheus" }) < {{ .Values.summary_denormalization_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for summary-denormalization job is more than threshold {{ .Values.summary_denormalization_threshold_warning | int }} + summary: summary-denormalization lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: summary-denormalization + alertname: FlinkJobsLag + + - alert: device-profile-updater lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "device-profile-updater-taskmanager-prometheus" }) > {{ .Values.device_profile_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "device-profile-updater-taskmanager-prometheus" }) < {{ .Values.device_profile_updater_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for device-profile-updater job is more than threshold {{ .Values.device_profile_updater_threshold_warning | int }} + summary: device-profile-updater lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: device-profile-updater + alertname: FlinkJobsLag + + - alert: user-cache-updater-v2 lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-v2-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-v2-taskmanager-prometheus" }) < {{ .Values.user_cache_updater_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for user-cache-updater-v2 job is more than threshold {{ .Values.user_cache_updater_threshold_warning | int }} + summary: user-cache-updater-v2 lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: user-cache-updater-v2 + alertname: FlinkJobsLag + + - alert: ingest-router lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "ingest-router-taskmanager-prometheus" }) > {{ .Values.ingest_router_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "ingest-router-taskmanager-prometheus" }) < {{ .Values.ingest_router_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for ingest-router job is more than threshold {{ .Values.ingest_router_threshold_warning | int }} + summary: ingest-router lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: ingest-router + alertname: FlinkJobsLag + + +#### KP jobs lag alert rules critical + - alert: activity-aggregate-updater lag critical + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "activity-aggregate-updater-taskmanager-prometheus" }) > {{ .Values.activity_aggregater_updater_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The lag for activity-aggregate-updater job is more than threshold {{ .Values.activity_aggregater_updater_threshold_critical | int }} + summary: activity-aggregate-updater lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: activity-aggregate-updater + alertname: FlinkJobsLag + + - alert: relation-cache-updater lag critical + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "relation-cache-updater-taskmanager-prometheus" }) > {{ .Values.relation_cache_updater_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The lag for relation-cache-updater job is more than threshold {{ .Values.relation_cache_updater_threshold_critical | int }} + summary: relation-cache-updater lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: relation-cache-updater + alertname: FlinkJobsLag + + - alert: post-publish-processor lag critical + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "post-publish-processor-taskmanager-prometheus" }) > {{ .Values.post_publish_processor_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The lag for post-publish-processor job is more than threshold {{ .Values.post_publish_processor_threshold_critical | int }} + summary: post-publish-processor lag is critical + lag: {{`({{ humanize $value }})`}} + job_id: post-publish-processor-taskmanager-prometheus + alertname: FlinkJobsLag + +## KP FLink jobs warning lag alertrules + + - alert: activity-aggregate-updater lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "activity-aggregate-updater-taskmanager-prometheus" }) > {{ .Values.activity_aggregater_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "activity-aggregate-updater-taskmanager-prometheus" }) < {{ .Values.activity_aggregater_updater_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for activity-aggregate-updater job is more than threshold {{ .Values.activity_aggregater_updater_threshold_warning | int }} + summary: activity-aggregate-updater lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: activity-aggregate-updater + alertname: FlinkJobsLag + + - alert: relation-cache-updater lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "relation-cache-updater-taskmanager-prometheus" }) > {{ .Values.relation_cache_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "relation-cache-updater-taskmanager-prometheus" }) < {{ .Values.relation_cache_updater_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for relation-cache-updater job is more than threshold {{ .Values.relation_cache_updater_threshold_warning | int }} + summary: relation-cache-updater lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: relation-cache-updater + alertname: FlinkJobsLag + + - alert: post-publish-processor lag warning + expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "post-publish-processor-taskmanager-prometheus" }) > {{ .Values.post_publish_processor_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "post-publish-processor-taskmanager-prometheus" }) < {{ .Values.post_publish_processor_threshold_critical }} + for: 5m + labels: + severity: warning + alerttype: lag + annotations: + message: The lag for post-publish-processor job is more than threshold {{ .Values.post_publish_processor_threshold_warning | int }} + summary: post-publish-processor lag is warning + lag: {{`({{ humanize $value }})`}} + job_id: post-publish-processor + alertname: FlinkJobsLag #### Checkpoint failure alert rules - alert: telemetry-extractor checkpoint failure critical @@ -100,98 +440,221 @@ spec: for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "telemetry-extractor-jobmanager" }) > {{ .Values.telemetry_extractor_checkpointfailure_threshold_critical }} + message: The checkpoint failure count for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_checkpointfailure_threshold_critical }} summary: telemetry-extractor checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: telemetry-extractor + alertname: Flink jobs checkpoint commit failure - alert: pipeline-preprocessor checkpoint failure critical expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "pipeline-preprocessor-jobmanager" }) > {{ .Values.pipeline_preprocessor_checkpointfailure_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "pipeline-preprocessor-jobmanager" }) > {{ .Values.pipeline_preprocessor_checkpointfailure_threshold_critical }} + message: The checkpoint failure count for pipeline-preprocessor job is more than threshold {{ .Values.pipeline_preprocessor_checkpointfailure_threshold_critical }} summary: pipeline-preprocessor checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: pipeline-preprocessor + alertname: Flink jobs checkpoint commit failure - - alert: de-normalization checkpoint failure critical - expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "de-normalization-jobmanager" }) > {{ .Values.de_normalization_checkpointfailure_threshold_critical }} + - alert: cb-preprocessor checkpoint failure critical + expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "cb-preprocessor-jobmanager" }) > {{ .Values.cb_preprocessor_checkpointfailure_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "de-normalization-jobmanager" }) > {{ .Values.de_normalization_checkpointfailure_threshold_critical }} - summary: de-normalization checkpoint failure critical + message: The checkpoint failure count for cb-preprocessor job is more than threshold {{ .Values.cb_preprocessor_checkpointfailure_threshold_critical }} + summary: cb-preprocessor checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: cb-preprocessor + alertname: Flink jobs checkpoint commit failure + + - alert: de-normalization-primary checkpoint failure critical + expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "de-normalization-primary-jobmanager" }) > {{ .Values.de_normalization_checkpointfailure_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The checkpoint failure count for de-normalization-primary job is more than threshold {{ .Values.de_normalization_checkpointfailure_threshold_critical }} + summary: de-normalization-primary checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: de-normalization-primary + alertname: Flink jobs checkpoint commit failure + + - alert: de-normalization-secondary checkpoint failure critical + expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "de-normalization-secondary-jobmanager" }) > {{ .Values.de_normalization_checkpointfailure_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The checkpoint failure count for de-normalization-secondary job is more than threshold {{ .Values.de_normalization_checkpointfailure_threshold_critical }} + summary: de-normalization-secondary checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: de-normalization-secondary + alertname: Flink jobs checkpoint commit failure - alert: druid-validator checkpoint failure critical expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "druid-validator-jobmanager" }) > {{ .Values.druid_validator_checkpointfailure_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "druid-validator-jobmanager" }) > {{ .Values.druid_validator_checkpointfailure_threshold_critical }} + message: The checkpoint failure count for druid-validator job is more than threshold {{ .Values.druid_validator_checkpointfailure_threshold_critical }} summary: druid-validator checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: druid-validator + alertname: Flink jobs checkpoint commit failure - alert: assessment-aggregator checkpoint failure critical expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "assessment-aggregator-jobmanager" }) > {{ .Values.assessment_aggregator_checkpointfailure_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "assessment-aggregator-jobmanager" }) > {{ .Values.assessment_aggregator_checkpointfailure_threshold_critical }} + message: The checkpoint failure count for assessment-aggregator job is more than threshold {{ .Values.assessment_aggregator_checkpointfailure_threshold_critical }} summary: assessment-aggregator checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: assessment-aggregator + alertname: Flink jobs checkpoint commit failure - alert: content-cache-updater checkpoint failure critical expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "content-cache-updater-jobmanager" }) > {{ .Values.content_cache_updater_checkpointfailure_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "content-cache-updater-jobmanager" }) > {{ .Values.content_cache_updater_checkpointfailure_threshold_critical }} + message: The checkpoint failure count for content-cache-updater job is more than threshold {{ .Values.content_cache_updater_checkpointfailure_threshold_critical }} summary: content-cache-updater checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: content-cache-updater + alertname: Flink jobs checkpoint commit failure - alert: user-cache-updater checkpoint failure critical expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "user-cache-updater-jobmanager" }) > {{ .Values.user_cache_updater_checkpointfailure_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "user-cache-updater-jobmanager" }) > {{ .Values.user_cache_updater_checkpointfailure_threshold_critical }} + message: The checkpoint failure count for user-cache-updater job is more than threshold {{ .Values.user_cache_updater_checkpointfailure_threshold_critical }} summary: user-cache-updater checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: user-cache-updater + alertname: Flink jobs checkpoint commit failure - alert: summary-denormalization checkpoint failure critical expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "summary-denormalization-jobmanager" }) > {{ .Values.summary_denormalization_checkpointfailure_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "summary-denormalization-jobmanager" }) > {{ .Values.summary_denormalization_checkpointfailure_threshold_critical }} + message: The checkpoint failure count for summary-denormalization job is more than threshold {{ .Values.summary_denormalization_checkpointfailure_threshold_critical }} summary: summary-denormalization checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: summary-denormalization + alertname: Flink jobs checkpoint commit failure - alert: device-profile-updater checkpoint failure critical expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "device-profile-updater-jobmanager" }) > {{ .Values.device_profile_updater_checkpointfailure_threshold_critical }} for: 5m labels: severity: critical + alerttype: lag annotations: - message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "device_profile_updater-jobmanager" }) > {{ .Values.device_profile_updater_checkpointfailure_threshold_critical }} + message: The checkpoint failure count for device-profile-updater job is more than threshold {{ .Values.device_profile_updater_checkpointfailure_threshold_critical }} summary: device-profile-updater checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: device-profile-updater + alertname: Flink jobs checkpoint commit failure + + - alert: activity-aggregate-updater checkpoint failure critical + expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "activity-aggregate-updater-jobmanager" }) > {{ .Values.activity_aggregate_updater_checkpointfailure_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The checkpoint failure count for device-profile-updater job is more than threshold {{ .Values.activity_aggregate_updater_checkpointfailure_threshold_critical }} + summary: activity-aggregate-updater checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: activity-aggregate-updater + alertname: Flink jobs checkpoint commit failure + + - alert: relation-cache-updater checkpoint failure critical + expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "relation-cache-updater-jobmanager" }) > {{ .Values.relation_cache_updater_checkpointfailure_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The checkpoint failure count for relation-cache-updater job is more than threshold {{ .Values.relation_cache_updater_checkpointfailure_threshold_critical }} + summary: relation-cache-updater checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: relation-cache-updater + alertname: Flink jobs checkpoint commit failure + + - alert: user-cache-updater-v2 checkpoint failure critical + expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "user-cache-updater-v2-jobmanager" }) > {{ .Values.user_cache_v2_updater_checkpointfailure_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The checkpoint failure count for user-cache-updater-v2 job is more than threshold {{ .Values.user_cache_v2_updater_checkpointfailure_threshold_critical }} + summary: user-cache-updater-v2 checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: user-cache-updater-v2 + alertname: Flink jobs checkpoint commit failure + + - alert: post-publish-processor checkpoint failure critical + expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "post-publish-processor-jobmanager" }) > {{ .Values.relation_cache_updater_checkpointfailure_threshold_critical }} + for: 5m + labels: + severity: critical + alerttype: lag + annotations: + message: The checkpoint failure count for post-publish-processor job is more than threshold {{ .Values.relation_cache_updater_checkpointfailure_threshold_critical }} + summary: post-publish-processor checkpoint failure critical + lag: {{`({{ humanize $value }})`}} + job_id: post-publish-processor-jobmanager + alertname: Flink jobs checkpoint commit failure + ### telemetry extractor failed event percentage alert rule - - alert: telemetry-extractor failed events percentage fatal - expr: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_fatal }} + - alert: telemetry-extractor failed events percentage warning + expr: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_warning }} for: 5m labels: - severity: fatal + severity: warning + alerttype: lag annotations: - message: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_fatal }} - summary: telemetry-extractor failed events percentage fatal + message: The failed events percentage for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_failed_events_percentage_threshold_warning }} + summary: telemetry-extractor failed events percentage warning + lag: {{`({{ humanize $value }})`}} + job_id: telemetry-extractor + alertname: Flink jobs failed events percentage - alert: telemetry-extractor failed events percentage critical expr: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_critical }} for: 5m labels: - severity: fatal + severity: critical + alerttype: lag annotations: - message: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_critical }} + message: The failed events percentage for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_failed_events_percentage_threshold_critical }} summary: telemetry-extractor failed events percentage critical + lag: {{`({{ humanize $value }})`}} + job_id: telemetry-extractor + alertname: Flink jobs failed events percentage diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs_lag.yaml b/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs_lag.yaml new file mode 100644 index 0000000000..c4f6e8578c --- /dev/null +++ b/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs_lag.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + role: alert-rules + app: {{ .Values.prometheus_rule_selector_app }} + release: {{ .Values.prometheus_rule_selector_release }} + name: {{ .Values.fullnameOverride }}-flinkjobs-lag + namespace: {{ .Values.namespace }} +spec: + groups: + - name: alertrules.flinkjobs-lag + rules: + {{ range $key, $threshold := .Values.flink_job_names }} + - alert: {{ $key }} lag increase critical + expr: ((sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) - sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m ) ) / sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m) ) * 100 > {{ $.Values.flink_lag_percentage_threshold }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "{{ $key}}-taskmanager-prometheus" }) > {{ $threshold }} + for: 10m + labels: + severity: critical + alerttype: lag + annotations: + message: The lag for {{ $key }} is increased by {{`{{ humanize $value }}`}} percentage + summary: {{ $key }} lag is increasing + lag: '{{`{{printf `}}`sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job = "{{ $key }}-taskmanager-prometheus" })` | query | first | value | humanize }}' + job_id: {{ $key }} + alertname: FlinkJobsLag + {{- end }} + + {{ range $key, $threshold := .Values.flink_job_names }} + - alert: {{ $key }} lag increase percentage critical + expr: sum(job:nginx_http_requests_total:rate:sum:5m) - sum(job:nginx_http_requests_total:rate:sum:5m offset {{ $.Values.flink_lag_offset_duration }}m) > {{ $.Values.nginx_tps_threshold }} and ((sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) - sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m)) / sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m)) * 100 > {{ $.Values.flink_lag_percentage_threshold }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) > {{ $threshold }} + for: 10m + labels: + severity: critical + alerttype: lag + annotations: + message: The tps is increasing and {{ $key }} lag is also increasing. Please check and increase the replica. + lag: '{{`{{printf `}}`sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job = "{{ $key }}-taskmanager-prometheus" })` | query | first | value | humanize }}' + job_id: {{ $key }} + alertname: FlinkJobsPercentageLag + {{- end }} + + {{ range $key, $threshold := .Values.flink_job_names }} + - alert: {{ $key }} lag increase percentage critical + expr: sum(job:nginx_http_requests_total:rate:sum:5m) - sum(job:nginx_http_requests_total:rate:sum:5m offset {{ $.Values.flink_lag_offset_duration }}m) < {{ $.Values.nginx_tps_threshold }} and ((sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) - sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m)) / sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m)) * 100 > {{ $.Values.flink_lag_percentage_threshold }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) > {{ $threshold }} + for: 10m + labels: + severity: critical + alerttype: lag + annotations: + message: There is no increase in tps but {{ $key }} lag is increasing. Please check and restart the job. + lag: '{{`{{printf `}}`sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job = "{{ $key }}-taskmanager-prometheus" })` | query | first | value | humanize }}' + job_id: {{ $key }} + alertname: FlinkJobsPercentageLag + {{- end }} diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/kubernetesApps.yaml b/kubernetes/helm_charts/monitoring/alertrules/templates/kubernetesApps.yaml index 42ea895d11..c9e7fe89f6 100644 --- a/kubernetes/helm_charts/monitoring/alertrules/templates/kubernetesApps.yaml +++ b/kubernetes/helm_charts/monitoring/alertrules/templates/kubernetesApps.yaml @@ -14,25 +14,30 @@ spec: rules: - alert: KubePodCrashLooping annotations: - message: {{`Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container - }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes.`}} + message: {{` The pod {{ $labels.namespace }}/{{ $labels.pod }} is restarting. `}} runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping + job_id: {{`({{ $labels.container }})`}} + alertname: KubePodCrashLooping expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~".*"}[15m]) * 60 * 5 > 0 for: 15m labels: severity: critical + alerttype: job - alert: KubePodNotReady annotations: message: {{`Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready`}} state for longer than 15 minutes. runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready + job_id: {{`({{ $labels.pod }})`}} + alertname: KubePodNotReady expr: sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", namespace=~".*", phase=~"Pending|Unknown"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})) > 0 for: 15m labels: severity: critical + alerttype: job - alert: KubeDeploymentGenerationMismatch annotations: message: {{`Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment @@ -132,11 +137,14 @@ spec: message: {{`Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.`}} runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting + job_id: {{`({{ $labels.container }})`}} + alertname: KubeContainerWaiting expr: sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", namespace=~".*"}) > 0 for: 1h labels: severity: warning + alerttype: job - alert: KubeDaemonSetNotScheduled annotations: message: {{`'{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset @@ -173,10 +181,13 @@ spec: annotations: message: {{`Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.`}} runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed + job_id: {{`({{ $labels.job_name }})`}} + alertname: KubeJobFailed expr: kube_job_failed{job="kube-state-metrics", namespace=~".*"} > 0 for: 15m labels: - severity: warning + severity: critical + alerttype: job - alert: KubeHpaReplicasMismatch annotations: message: {{`HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired @@ -203,3 +214,19 @@ spec: for: 15m labels: severity: warning + +## kubepersistentvolumeusagecritical + - alert: KubePersistentVolumeUsageCritical + annotations: + message: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} usage is {{`{{`}} printf "%0.2f" $value {{`}}`}}%. + metrics: {{`({{ humanize $value }})`}} + alertname: KubePersistentVolumeUsageCritical + expr: |- + 100 * kubelet_volume_stats_used_bytes{job="kubelet", namespace="secor"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", namespace="secor"} + > {{ .Values.pv_usage_threshold_critical }} + for: 1m + labels: + alerttype: nodemetrics + severity: critical diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yaml b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yaml index f743a8ea8e..1de7338bda 100644 --- a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yaml +++ b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yaml @@ -13,106 +13,105 @@ spec: - name: alertrules.nodes rules: - alert: high_cpu_usage_on_node_warning - expr: (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Warning }} and (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) < {{ .Values.node_cpu_usage_percentage_threshold_Critical }} + expr: (avg by (instance) (irate(node_cpu_seconds_total{job="node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Warning }} and (avg by (instance) (irate(node_cpu_seconds_total{job="node-exporter",mode!="idle"}[5m])) * 100) < {{ .Values.node_cpu_usage_percentage_threshold_Critical }} for: 1m labels: severity: warning + alerttype: nodemetrics annotations: message: {{`'{{ $labels.instance }} is using a LOT of CPU. CPU usage is {{ humanize $value}}%.'`}} summary: {{`'HIGH CPU USAGE warning ON {{ $labels.instance }}'`}} + alertname: HighCpuUsageOnK8sNode + metrics: {{`({{ humanize $value }})`}}% + - alert: high_cpu_usage_on_node_critical - expr: (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Critical }} and (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) < {{ .Values.node_cpu_usage_percentage_threshold_Fatal }} + expr: (avg by (instance) (irate(node_cpu_seconds_total{job="node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Critical }} for: 1m labels: severity: critical + alerttype: nodemetrics annotations: message: {{`'{{ $labels.instance }} is using a LOT of CPU. CPU usage is {{ humanize $value}}%.'`}} summary: {{`'HIGH CPU USAGE warning ON {{ $labels.instance }}'`}} - - alert: high_cpu_usage_on_node_fatal - expr: (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Fatal }} - for: 1m - labels: - severity: fatal - annotations: - message: {{`'{{ $labels.instance }} is using a LOT of CPU. CPU usage is {{ humanize $value}}%.'`}} - summary: {{`'HIGH CPU USAGE warning ON {{ $labels.instance }}'`}} + alertname: HighCpuUsageOnK8sNode + metrics: {{`({{ humanize $value }})`}}% + - alert: high_memory_usage_on_node_warning expr: sum by(nodename) ((((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) >= {{ .Values.node_memory_usage_percentage_threshold_Warning }} and (((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) < {{ .Values.node_memory_usage_percentage_threshold_Critical }} ) for: 1m labels: severity: warning + alerttype: nodemetrics annotations: message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) is using a LOT of MEMORY. MEMORY usage is over {{ humanize $value}}.'`}} summary: {{`'HIGH MEMORY USAGE warning TASK ON {{ $labels.nodename }}'`}} + alertname: HighMemoryUsageOnK8sNode + metrics: {{`({{ humanize $value }})`}}% + - alert: high_memory_usage_on_node_critical - expr: sum by(nodename) ((((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) >= {{ .Values.node_memory_usage_percentage_threshold_Critical }} and (((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) < {{ .Values.node_memory_usage_percentage_threshold_Fatal }} ) + expr: sum by(nodename) ((((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) >= {{ .Values.node_memory_usage_percentage_threshold_Critical }}) for: 1m labels: severity: critical + alerttype: nodemetrics annotations: message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) is using a LOT of MEMORY. MEMORY usage is over {{ humanize $value}}.'`}} summary: {{`'HIGH MEMORY USAGE warning TASK ON {{ $labels.nodename }}'`}} - - alert: high_memory_usage_on_node_fatal - expr: sum by(nodename) (((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) >= {{ .Values.node_memory_usage_percentage_threshold_Fatal }} - for: 1m - labels: - severity: fatal - annotations: - message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) is using a LOT of MEMORY. MEMORY usage is over {{ humanize $value}}.'`}} - summary: {{`'HIGH MEMORY USAGE warning TASK ON {{ $labels.nodename }}'`}} + alertname: HighMemoryUsageOnK8sNode + metrics: {{`({{ humanize $value }})`}}% + - alert: high_load_on_node_warning - expr: sum by(instance) ((node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Warning }} and (node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) < {{ .Values.node_load_avg_threshold_Critical }}) + expr: sum by(instance) ((node_load1{job="node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Warning }} and (node_load1{job="node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="node-exporter"})) * 100) < {{ .Values.node_load_avg_threshold_Critical }}) for: 5m labels: severity: warning + alerttype: nodemetrics annotations: message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) has a high load average. Load average is {{ $value }}%.'`}} summary: {{`'HIGH LOAD AVERAGE warning ON {{ $labels.nodename }}'`}} + alertname: HighLoadOnK8sNode + metrics: {{`({{ humanize $value }})`}}% + - alert: high_load_on_node_critical - expr: sum by(instance) ((node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Critical }} and (node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) < {{ .Values.node_load_avg_threshold_Fatal }}) + expr: sum by(instance) ((node_load1{job="node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Critical }}) for: 5m labels: severity: critical + alerttype: nodemetrics annotations: message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) has a high load average. Load average is {{ $value }}%.'`}} summary: {{`'HIGH LOAD AVERAGE warning ON {{ $labels.nodename }}'`}} - - alert: high_load_on_node_fatal - expr: sum by(instance) ((node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Fatal }}) - for: 5m - labels: - severity: fatal - annotations: - message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) has a high load average. Load average is {{ $value }}%.'`}} - summary: {{`'HIGH LOAD AVERAGE warning ON {{ $labels.nodename }}'`}} + alertname: HighLoadOnK8sNode + metrics: {{`({{ humanize $value }})`}}% + - alert: node_exporter_down_critical expr: up == 0 for: 1m labels: - severity: critical + severity: warning annotations: message: {{`The node exporter '{{ $labels.job }}' is down.`}} summary: {{`'NODE EXPORTER SERVICE critical: NODE ''{{ $labels.host }}'''`}} + - alert: node_running_out_of_disk_space_warning expr: sum by(nodename) (((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) >= {{ .Values.node_disk_usage_percentage_threshold_Warning }} and ((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) < {{ .Values.node_disk_usage_percentage_threshold_Critical }} ) for: 1m labels: severity: warning + alerttype: nodemetrics annotations: message: {{`'Disk usage is {{ humanize $value }}%'`}} summary: {{`'LOW DISK SPACE WARING: NODE {{ $labels.nodename }}'`}} + alertname: K8sNodeRunningOutOfDiskSpace + metrics: {{`({{ humanize $value }})`}}% + - alert: node_running_out_of_disk_space_warning - expr: sum by(nodename) (((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) >= {{ .Values.node_disk_usage_percentage_threshold_Critical }} and ((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) < {{ .Values.node_disk_usage_percentage_threshold_Fatal }} ) + expr: sum by(nodename) (((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) >= {{ .Values.node_disk_usage_percentage_threshold_Critical }}) for: 1m labels: severity: critical annotations: message: {{`'Disk usage is {{ humanize $value }}%'`}} summary: {{`'LOW DISK SPACE WARING: NODE {{ $labels.nodename }}'`}} - - alert: node_running_out_of_disk_space_fatal - expr: sum by(nodename) ((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) >= {{ .Values.node_disk_usage_percentage_threshold_Fatal }} - for: 1m - labels: - severity: fatal - annotations: - message: {{`'Disk usage is {{ humanize $value }}%'`}} - summary: {{`'LOW DISK SPACE WARING: NODE {{ $labels.nodename }}'`}} + alertname: K8sNodeRunningOutOfDiskSpace + metrics: {{`({{ humanize $value }})`}}% diff --git a/kubernetes/helm_charts/monitoring/alertrules/values.yaml b/kubernetes/helm_charts/monitoring/alertrules/values.yaml index 990ed4e8ef..d6c247afc2 100644 --- a/kubernetes/helm_charts/monitoring/alertrules/values.yaml +++ b/kubernetes/helm_charts/monitoring/alertrules/values.yaml @@ -5,15 +5,15 @@ fullnameOverride: sunbird-monitoring namespace: monitoring # Node Exporter vars -node_cpu_usage_percentage_threshold_Warning: 75 -node_cpu_usage_percentage_threshold_Critical: 85 +node_cpu_usage_percentage_threshold_Warning: 80 +node_cpu_usage_percentage_threshold_Critical: 90 node_cpu_usage_percentage_threshold_Fatal: 95 -node_memory_usage_percentage_threshold_Critical: 85 -node_memory_usage_percentage_threshold_Warning: 75 +node_memory_usage_percentage_threshold_Critical: 90 +node_memory_usage_percentage_threshold_Warning: 80 node_memory_usage_percentage_threshold_Fatal: 95 -node_load_avg_threshold_Critical: 95 -node_load_avg_threshold_Warning: 85 -node_load_avg_threshold_Fatal: 120 -node_disk_usage_percentage_threshold_Warning: 75 -node_disk_usage_percentage_threshold_Critical: 85 +node_load_avg_threshold_Critical: 90 +node_load_avg_threshold_Warning: 80 +node_load_avg_threshold_Fatal: 95 +node_disk_usage_percentage_threshold_Warning: 80 +node_disk_usage_percentage_threshold_Critical: 90 node_disk_usage_percentage_threshold_Fatal: 95 diff --git a/kubernetes/helm_charts/monitoring/ingestion-kafka-exporter b/kubernetes/helm_charts/monitoring/ingestion-kafka-exporter new file mode 120000 index 0000000000..8ed35ad576 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/ingestion-kafka-exporter @@ -0,0 +1 @@ +processing-kafka-exporter \ No newline at end of file diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/.helmignore b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/.helmignore new file mode 100644 index 0000000000..50af031725 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/Chart.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/Chart.yaml new file mode 100644 index 0000000000..3a5191414d --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: processing-kafka-exporter +version: 1.0.0 +home: https://github.com/abhishekjiitr/kafka-exporter-helm +maintainers: + - name: abhishekjiitr + email: abhi2254015@gmail.com diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/NOTES.txt b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/NOTES.txt new file mode 100644 index 0000000000..d9eb9e809b --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/NOTES.txt @@ -0,0 +1,5 @@ +1.To see the metrics +{{- if contains "ClusterIP" .Values.service.type }} + kubectl port-forward svc/{{ include "kafka-exporter.fullname" . }} {{ .Values.service.port }} + echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application" +{{- end }} diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/_helpers.tpl b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/_helpers.tpl new file mode 100644 index 0000000000..bc51bbfcd5 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kafka-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kafka-exporter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kafka-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/alertRules.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/alertRules.yaml new file mode 100644 index 0000000000..e107a6a78d --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/alertRules.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "kafka-exporter.fullname" . }} + {{- if .Values.prometheus.serviceMonitor.namespace }} + namespace: {{ .Values.prometheus.serviceMonitor.namespace }} + {{- end }} + labels: + app.kubernetes.io/name: {{ include "kafka-exporter.name" . }} + helm.sh/chart: {{ include "kafka-exporter.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.prometheus.serviceMonitor.additionalLabels }} +{{ toYaml .Values.prometheus.serviceMonitor.additionalLabels | indent 4 -}} + {{- end }} +spec: + groups: + - name: alertrules.kafkaExporter + rules: + - record: isr_partition_difference + expr: kafka_topic_partition_in_sync_replica{topic!="__consumer_offsets",topic!="__samza_*"} - kafka_topic_partition_replicas{topic!="__consumer_offsets",topic!="__samza_*"} + + - alert: ISR_not_equal_to_partition + annotations: + message: {{`"ISR and replication difference for $labels.topic is > 0 current value: {{ humanize $value }}"`}} + summary: {{`"ISR and Replica Mismatch for {{$labels.topic}}"`}} + expr: isr_partition_difference != 0 + for: 5m + labels: + severity: critical + module: dp + + - alert: Kafka_partition_leader_change_rapidly + annotations: + message: {{`"Kafaka Partition leader changing rapidly for {{ $labels.topic }}"`}} + summary: {{`"Kafaka Partition leader changing rapidly for {{ $labels.topic }}"`}} + expr: rate(kafka_topic_partition_leader{topic!="__consumer_offsets",topic!="__samza_*"}[5m]) > 0 + for: 5m + labels: + severity: critical + module: dp + + - alert: kafka_broker_unavailable_processing_cluster + annotations: + summary: {{`"Kafka brokers unavailable processing cluster"`}} + message: "There are only {{`{{humanize $value}}`}} kafka brokers available; Expected count: {{ len .Values.kafkaExporter.kafka.servers }}" + expr: kafka_brokers{job="processing-kafka-exporter"} < {{ len .Values.kafkaExporter.kafka.servers }} + for: 5m + labels: + severity: critical + module: dp + + - alert: kafka_broker_unavailable_ingestion_cluster + annotations: + summary: {{`"Kafka brokers unavailable ingestion cluster "`}} + message: "There are only {{`{{humanize $value}}`}} kafka brokers available; Expected count: {{ len .Values.ingestion_cluster_kafka_server_count }}" + expr: kafka_brokers{job="ingestion-kafka-exporter"} < {{ len .Values.ingestion_cluster_kafka_server_count }} + for: 5m + labels: + severity: critical + module: dp diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/deployment.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/deployment.yaml new file mode 100644 index 0000000000..d79b57b019 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/deployment.yaml @@ -0,0 +1,91 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kafka-exporter.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "kafka-exporter.name" . }} + helm.sh/chart: {{ include "kafka-exporter.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "kafka-exporter.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "kafka-exporter.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + {{- if .Values.kafkaExporter}} + {{- range .Values.kafkaExporter.kafka.servers }} + - "--kafka.server={{ . }}" + {{- end }} + {{- range .Values.kafkaExporter.zookeeper.servers }} + - "--zookeeper.server={{ . }}" + {{- end }} + {{- range .Values.kafkaExporter.additionalFlags }} + - "{{ . }}" + {{- end }} + {{- if .Values.kafkaExporter.kafka.version }} + - --kafka.version={{ .Values.kafkaExporter.kafka.version }} + {{- end }} + {{- end}} + {{- if .Values.kafkaExporter.sasl.enabled }} + - --sasl.enabled + {{- if not .Values.kafkaExporter.sasl.handshake }} + - --sasl.handshake=false + {{- end }} + - --sasl.username={{ .Values.kafkaExporter.sasl.username }} + - --sasl.password={{ .Values.kafkaExporter.sasl.password }} + {{- end }} + {{- if .Values.kafkaExporter.tls.enabled}} + - --tls.enabled + - --tls.ca-file=/etc/tls-certs/ca-file + - --tls.cert-file=/etc/tls-certs/cert-file + - --tls.key-file=/etc/tls-certs/key-file + {{- end }} + {{- if .Values.kafkaExporter.log }} + - --log.level={{ .Values.kafkaExporter.log.level }} + {{- if .Values.kafkaExporter.log.enableSarama }} + - --log.enable-sarama + {{- end }} + {{- end }} + ports: + - name: metrics + containerPort: 9308 + protocol: TCP + {{- if .Values.kafkaExporter.tls.enabled }} + volumeMounts: + - name: tls-certs + mountPath: "/etc/tls-certs/" + readOnly: true + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.kafkaExporter.tls.enabled }} + volumes: + - name: tls-certs + secret: + secretName: {{ include "kafka-exporter.fullname" . }} + {{- end }} diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/secret.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/secret.yaml new file mode 100644 index 0000000000..82f567f38d --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.kafkaExporter.tls.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kafka-exporter.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "kafka-exporter.name" . }} + helm.sh/chart: {{ include "kafka-exporter.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + ca-file: {{ .Values.kafkaExporter.tls.caFile | b64enc }} + cert-file: {{ .Values.kafkaExporter.tls.certFile | b64enc }} + key-file: {{ .Values.kafkaExporter.tls.keyFile | b64enc }} +{{- end }} diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/service.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/service.yaml new file mode 100644 index 0000000000..049041fb03 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kafka-exporter.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "kafka-exporter.name" . }} + helm.sh/chart: {{ include "kafka-exporter.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + app.kubernetes.io/name: {{ include "kafka-exporter.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/servicemonitor.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..395d617dd3 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "kafka-exporter.fullname" . }} + {{- if .Values.prometheus.serviceMonitor.namespace }} + namespace: {{ .Values.prometheus.serviceMonitor.namespace }} + {{- end }} + labels: + app.kubernetes.io/name: {{ include "kafka-exporter.name" . }} + helm.sh/chart: {{ include "kafka-exporter.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.prometheus.serviceMonitor.additionalLabels }} +{{ toYaml .Values.prometheus.serviceMonitor.additionalLabels | indent 4 -}} + {{- end }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app.kubernetes.io/name: {{ include "kafka-exporter.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + helm.sh/chart: {{ include "kafka-exporter.chart" . }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: metrics + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- if .Values.prometheus.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.serviceMonitor.scrapeTimeout }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/values.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/values.yaml new file mode 100644 index 0000000000..2431eafa2e --- /dev/null +++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/values.yaml @@ -0,0 +1,70 @@ +# Default values for kafka-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 +ingestion_cluster_kafka_server_count: ["{{ ingestion_cluster_kafka | join('","') }}"] + +image: + repository: danielqsj/kafka-exporter + tag: latest + pullPolicy: IfNotPresent + +nameOverride: "" +fullnameOverride: "" + +service: + type: ClusterIP + port: 9308 + +kafkaExporter: + kafka: + servers: [] + zookeeper: + servers: [] + additionalFlags: [] + # - --use.consumelag.zookeeper + + sasl: + enabled: false + handshake: true + username: "" + password: "" + + tls: + enabled: false + insecure-skip-tls-verify: false + caFile: "" + certFile: "" + keyFile: "" + + log: + level: info + enableSarama: false + +prometheus: + serviceMonitor: + enabled: true + namespace: monitoring + interval: "30s" + additionalLabels: + app: kafka-exporter + + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/.helmignore b/kubernetes/helm_charts/monitoring/prometheus-adapter/.helmignore new file mode 100755 index 0000000000..f0c1319444 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/Chart.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/Chart.yaml new file mode 100755 index 0000000000..96efec0851 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +appVersion: v0.7.0 +deprecated: true +description: DEPRECATED A Helm chart for k8s prometheus adapter +home: https://github.com/DirectXMan12/k8s-prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/DirectXMan12/k8s-prometheus-adapter +version: 2.5.1 diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/OWNERS b/kubernetes/helm_charts/monitoring/prometheus-adapter/OWNERS new file mode 100755 index 0000000000..0c15fcf344 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/OWNERS @@ -0,0 +1,8 @@ +approvers: + - mattiasgees + - steven-sheehy + - hectorj2f +reviewers: + - mattiasgees + - steven-sheehy + - hectorj2f diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/README.md b/kubernetes/helm_charts/monitoring/prometheus-adapter/README.md new file mode 100755 index 0000000000..182e633d5a --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/README.md @@ -0,0 +1,162 @@ +# Prometheus Adapter + +DEPRECATED and moved to + +Installs the [Prometheus Adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +$ helm install --name my-release stable/prometheus-adapter +``` + +This command deploys the prometheus adapter with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +## Using the Chart + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/DirectXMan12/k8s-prometheus-adapter/blob/master/docs/config.md) for the proper format). + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +``` +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +``` +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +``` +rules: + resource: + cpu: + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following table lists the configurable parameters of the Prometheus Adapter chart and their default values. + +| Parameter | Description | Default | +| ------------------------------- | ------------------------------------------------------------------------------- | --------------------------------------------| +| `affinity` | Node affinity | `{}` | +| `image.repository` | Image repository | `directxman12/k8s-prometheus-adapter-amd64` | +| `image.tag` | Image tag | `v0.7.0` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets | `{}` | +| `logLevel` | Log level | `4` | +| `listenPort` | Port that application would listen on in the container | `6443` | +| `metricsRelistInterval` | Interval at which to re-list the set of all available metrics from Prometheus | `1m` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `podLabels` | Labels to add to the pod | `{}` | +| `podAnnotations` | Annotations to add to the pod | `{}` | +| `priorityClassName` | Pod priority | `` | +| `prometheus.url` | Url of where we can find the Prometheus service | `http://prometheus.default.svc` | +| `prometheus.port` | Port of where we can find the Prometheus service, zero to omit this option | `9090` | +| `prometheus.path` | Custom path to append to the prometheus URL | `` | +| `rbac.create` | If true, create & use RBAC resources | `true` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `rules.default` | If `true`, enable a set of default rules in the configmap | `true` | +| `rules.custom` | A list of custom configmap rules | `[]` | +| `rules.existing` | The name of an existing configMap with rules. Overrides default, custom and external. | `` | +| `rules.external` | A list of custom rules for external metrics API | `[]` | +| `rules.resource` | `resourceRules` to set in configmap rules | `{}` | +| `service.annotations` | Annotations to add to the service | `{}` | +| `service.port` | Service port to expose | `443` | +| `service.type` | Type of service to create | `ClusterIP` | +| `serviceAccount.create` | If true, create & use Serviceaccount | `true` | +| `serviceAccount.name` | If not set and create is true, a name is generated using the fullname template | `` | +| `tls.enable` | If true, use the provided certificates. If false, generate self-signed certs | `false` | +| `tls.ca` | Public CA file that signed the APIService (ignored if tls.enable=false) | `` | +| `tls.key` | Private key of the APIService (ignored if tls.enable=false) | `` | +| `tls.certificate` | Public key of the APIService (ignored if tls.enable=false) | `` | +| `extraVolumeMounts` | Any extra volumes mounts | `[]` | +| `extraVolumes` | Any extra volumes | `[]` | +| `tolerations` | List of node taints to tolerate | `[]` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name my-release \ + --set logLevel=1 \ + stable/prometheus-adapter +``` + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install --name my-release -f values.yaml stable/prometheus-adapter +``` diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/default-values.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/default-values.yaml new file mode 100755 index 0000000000..e69de29bb2 diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/external-rules-values.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/external-rules-values.yaml new file mode 100755 index 0000000000..2dafb56211 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/external-rules-values.yaml @@ -0,0 +1,9 @@ +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/NOTES.txt b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/NOTES.txt new file mode 100755 index 0000000000..329148be71 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,11 @@ +DEPRECATED and moved to + +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/_helpers.tpl b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/_helpers.tpl new file mode 100755 index 0000000000..5d0b993c52 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,43 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml new file mode 100755 index 0000000000..44761c4569 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml new file mode 100755 index 0000000000..79c0af39f6 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml new file mode 100755 index 0000000000..2f828f4a6d --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml @@ -0,0 +1,120 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: {{ template "k8s-prometheus-adapter.name" . }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/custom-metrics-configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if .Values.tls.enable }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --logtostderr=true + - --prometheus-url={{ .Values.prometheus.url }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + ports: + - containerPort: {{ .Values.listenPort }} + name: https + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if .Values.tls.enable }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: + {{- toYaml .Values.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + tolerations: + {{- toYaml .Values.tolerations | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if .Values.tls.enable }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml new file mode 100755 index 0000000000..ec7e5e4762 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml new file mode 100755 index 0000000000..d73deacb78 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml new file mode 100755 index 0000000000..83dd70a91b --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + app: {{ template "k8s-prometheus-adapter.name" . }} + release: {{ .Release.Name }} + type: {{ .Values.service.type }} + diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100755 index 0000000000..1eb380d70f --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,23 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{ if .Values.tls.enable -}} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100755 index 0000000000..33daf7113b --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-configmap.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-configmap.yaml new file mode 100755 index 0000000000..a191e8add3 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-configmap.yaml @@ -0,0 +1,95 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml new file mode 100755 index 0000000000..319460a332 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-apiservice.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100755 index 0000000000..560b2ffcd8 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rules.external }} +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{ if .Values.tls.enable -}} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100755 index 0000000000..4adbd65372 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml new file mode 100755 index 0000000000..641b9d3a44 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml new file mode 100755 index 0000000000..0776029af2 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100755 index 0000000000..f1a619912f --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rules.resource}} +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{ if .Values.tls.enable -}} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100755 index 0000000000..0534af11e6 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100755 index 0000000000..01a307d69b --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/secret.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/secret.yaml new file mode 100755 index 0000000000..38e7cb624f --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/values.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/values.yaml new file mode 100755 index 0000000000..4cc5e40692 --- /dev/null +++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/values.yaml @@ -0,0 +1,141 @@ +# Default values for k8s-prometheus-adapter.. +affinity: {} + +image: + repository: directxman12/k8s-prometheus-adapter-amd64 + tag: v0.7.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +# Url to access prometheus +prometheus: + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +rules: + default: true + custom: [] +# - seriesQuery: '{__name__=~"^some_metric_count$"}' +# resources: +# template: <<.Resource>> +# name: +# matches: "" +# as: "my_custom_metric" +# metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + external: [] +# - seriesQuery: '{__name__=~"^some_metric_count$"}' +# resources: +# template: <<.Resource>> +# name: +# matches: "" +# as: "my_external_metric" +# metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + resource: {} +# cpu: +# containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) +# nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) +# resources: +# overrides: +# instance: +# resource: node +# namespace: +# resource: namespace +# pod: +# resource: pod +# containerLabel: container +# memory: +# containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) +# nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) +# resources: +# overrides: +# instance: +# resource: node +# namespace: +# resource: namespace +# pod: +# resource: pod +# containerLabel: container +# window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS + enabled: false diff --git a/kubernetes/helm_charts/nginx-ingress/Chart.yaml b/kubernetes/helm_charts/nginx-ingress/Chart.yaml new file mode 100644 index 0000000000..1aa28e1e29 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/Chart.yaml @@ -0,0 +1,16 @@ +name: nginx-ingress +version: 0.10.0 +appVersion: 1.12.0 +apiVersion: v1 +kubeVersion: ">= 1.16.0-0" +description: NGINX Ingress Controller +icon: https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v1.12.0/deployments/helm-chart/chart-icon.png +home: https://github.com/nginxinc/kubernetes-ingress +sources: + - https://github.com/nginxinc/kubernetes-ingress/tree/v1.12.0/deployments/helm-chart +keywords: + - ingress + - nginx +maintainers: + - name: nginxinc + email: kubernetes@nginx.com diff --git a/kubernetes/helm_charts/nginx-ingress/README.md b/kubernetes/helm_charts/nginx-ingress/README.md new file mode 100644 index 0000000000..d0f997f395 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/README.md @@ -0,0 +1,230 @@ +# NGINX Ingress Controller Helm Chart + +## Introduction + +This chart deploys the NGINX Ingress controller in your Kubernetes cluster. + +## Prerequisites + + - A [Kubernetes Version Supported by the Ingress Controller](https://docs.nginx.com/nginx-ingress-controller/technical-specifications/#supported-kubernetes-versions) + - Helm 3.0+. + - Git. + - If you’d like to use NGINX Plus: + - Build an Ingress controller image with NGINX Plus and push it to your private registry by following the instructions from [here](../../build/README.md). + - Update the `controller.image.repository` field of the `values-plus.yaml` accordingly. + +## Getting the Chart Sources + +This step is required if you're installing the chart using its sources. Additionally, the step is also required for managing the custom resource definitions (CRDs), which the Ingress Controller requires by default, or for upgrading/deleting the CRDs. + +1. Clone the Ingress controller repo: + ```console + $ git clone https://github.com/nginxinc/kubernetes-ingress/ + ``` +2. Change your working directory to /deployments/helm-chart: + ```console + $ cd kubernetes-ingress/deployments/helm-chart + $ git checkout v1.12.0 + ``` + +## Adding the Helm Repository + +This step is required if you're installing the chart via the helm repository. + +```console +$ helm repo add nginx-stable https://helm.nginx.com/stable +$ helm repo update +``` + +**Note**: If you wish to use the experimental repository, replace `stable` with `edge`. + +## Installing the Chart + +### Installing the CRDs + +By default, the Ingress Controller requires a number of custom resource definitions (CRDs) installed in the cluster. The Helm client will install those CRDs. + +If you do not use the custom resources that require those CRDs (which corresponds to `controller.enableCustomResources` set to `false` and `controller.appprotect.enable` set to `false`). The installation of the CRDs can be skipped by specifying `--skip-crds` for the helm install command. + +### Installing via Helm Repository + +To install the chart with the release name my-release (my-release is the name that you choose): + +For NGINX: +```console +$ helm install my-release nginx-stable/nginx-ingress +``` + +For NGINX Plus: (assuming you have pushed the Ingress controller image `nginx-plus-ingress` to your private registry `myregistry.example.com`) +```console +$ helm install my-release nginx-stable/nginx-ingress --set controller.image.repository=myregistry.example.com/nginx-plus-ingress --set controller.nginxplus=true +``` + +**Note**: If you wish to use the experimental repository, replace `stable` with `edge` and add the `--devel` flag. + +### Installing Using Chart Sources + +To install the chart with the release name my-release (my-release is the name that you choose): + +For NGINX: +```console +$ helm install my-release . +``` + +For NGINX Plus: +```console +$ helm install my-release -f values-plus.yaml . +``` + +**Note**: If you wish to use the experimental repository, replace the value in the `tag` field inside the yaml files with `edge`. + +The command deploys the Ingress controller in your Kubernetes cluster in the default configuration. The configuration section lists the parameters that can be configured during installation. + +When deploying the Ingress controller, make sure to use your own TLS certificate and key for the default server rather than the default pre-generated ones. Read the [Configuration](#Configuration) section below to see how to configure a TLS certificate and key for the default server. Note that the default server returns the Not Found page with the 404 status code for all requests for domains for which there are no Ingress rules defined. + +## Upgrading the Chart + +### Upgrading the CRDs + +Helm does not upgrade the CRDs during a release upgrade. Before you upgrade a release, run the following command to upgrade the CRDs: + +```console +$ kubectl apply -f crds/ +``` +> **Note**: The following warning is expected and can be ignored: `Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply`. + +> **Note**: Make sure to check the [release notes](https://www.github.com/nginxinc/kubernetes-ingress/releases) for a new release for any special upgrade procedures. + +### Upgrading the Release + +To upgrade the release `my-release`: + +#### Upgrade Using Chart Sources: + +```console +$ helm upgrade my-release . +``` + +#### Upgrade via Helm Repository: + +```console +$ helm upgrade my-release nginx-stable/nginx-ingress +``` + +## Uninstalling the Chart + +### Uninstalling the Release + +To uninstall/delete the release `my-release`: + +```console +$ helm uninstall my-release +``` + +The command removes all the Kubernetes components associated with the release and deletes the release. + +### Uninstalling the CRDs + +Uninstalling the release does not remove the CRDs. To remove the CRDs, run: + +```console +$ kubectl delete -f crds/ +``` +> **Note**: This command will delete all the corresponding custom resources in your cluster across all namespaces. Please ensure there are no custom resources that you want to keep and there are no other Ingress Controller releases running in the cluster. + +## Running Multiple Ingress Controllers + +If you are running multiple Ingress Controller releases in your cluster with enabled custom resources, the releases will share a single version of the CRDs. As a result, make sure that the Ingress Controller versions match the version of the CRDs. Additionally, when uninstalling a release, ensure that you don’t remove the CRDs until there are no other Ingress Controller releases running in the cluster. + +See [running multiple ingress controllers](https://docs.nginx.com/nginx-ingress-controller/installation/running-multiple-ingress-controllers/) for more details. + +## Configuration + +The following tables lists the configurable parameters of the NGINX Ingress controller chart and their default values. + +Parameter | Description | Default +--- | --- | --- +`controller.name` | The name of the Ingress controller daemonset or deployment. | Autogenerated +`controller.kind` | The kind of the Ingress controller installation - deployment or daemonset. | deployment +`controller.nginxplus` | Deploys the Ingress controller for NGINX Plus. | false +`controller.nginxReloadTimeout` | The timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. The default is 4000 (or 20000 if `controller.appprotect.enable` is true). If set to 0, the default value will be used. | 0 +`controller.hostNetwork` | Enables the Ingress controller pods to use the host's network namespace. | false +`controller.nginxDebug` | Enables debugging for NGINX. Uses the `nginx-debug` binary. Requires `error-log-level: debug` in the ConfigMap via `controller.config.entries`. | false +`controller.logLevel` | The log level of the Ingress Controller. | 1 +`controller.image.repository` | The image repository of the Ingress controller. | nginx/nginx-ingress +`controller.image.tag` | The tag of the Ingress controller image. | 1.12.0 +`controller.image.pullPolicy` | The pull policy for the Ingress controller image. | IfNotPresent +`controller.config.name` | The name of the ConfigMap used by the Ingress controller. | Autogenerated +`controller.config.annotations` | The annotations of the Ingress controller configmap. | {} +`controller.config.entries` | The entries of the ConfigMap for customizing NGINX configuration. See [ConfigMap resource docs](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) for the list of supported ConfigMap keys. | {} +`controller.customPorts` | A list of custom ports to expose on the NGINX ingress controller pod. Follows the conventional Kubernetes yaml syntax for container ports. | [] +`controller.defaultTLS.cert` | The base64-encoded TLS certificate for the default HTTPS server. If not specified, a pre-generated self-signed certificate is used. **Note:** It is recommended that you specify your own certificate. | A pre-generated self-signed certificate. +`controller.defaultTLS.key` | The base64-encoded TLS key for the default HTTPS server. **Note:** If not specified, a pre-generated key is used. It is recommended that you specify your own key. | A pre-generated key. +`controller.defaultTLS.secret` | The secret with a TLS certificate and key for the default HTTPS server. The value must follow the following format: `/`. Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters. | None +`controller.wildcardTLS.cert` | The base64-encoded TLS certificate for every Ingress host that has TLS enabled but no secret specified. If the parameter is not set, for such Ingress hosts NGINX will break any attempt to establish a TLS connection. | None +`controller.wildcardTLS.key` | The base64-encoded TLS key for every Ingress host that has TLS enabled but no secret specified. If the parameter is not set, for such Ingress hosts NGINX will break any attempt to establish a TLS connection. | None +`controller.wildcardTLS.secret` | The secret with a TLS certificate and key for every Ingress host that has TLS enabled but no secret specified. The value must follow the following format: `/`. Used as an alternative to specifying a certificate and key using `controller.wildcardTLS.cert` and `controller.wildcardTLS.key` parameters. | None +`controller.nodeSelector` | The node selector for pod assignment for the Ingress controller pods. | {} +`controller.terminationGracePeriodSeconds` | The termination grace period of the Ingress controller pod. | 30 +`controller.tolerations` | The tolerations of the Ingress controller pods. | [] +`controller.affinity` | The affinity of the Ingress controller pods. | {} +`controller.volumes` | The volumes of the Ingress controller pods. | [] +`controller.volumeMounts` | The volumeMounts of the Ingress controller pods. | [] +`controller.resources` | The resources of the Ingress controller pods. | {} +`controller.replicaCount` | The number of replicas of the Ingress controller deployment. | 1 +`controller.ingressClass` | A class of the Ingress controller. For Kubernetes >= 1.18, a corresponding IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. For Kubernetes < 1.18, the Ingress Controller only processes resources that belong to its class - i.e have the annotation "kubernetes.io/ingress.class" (for Ingress resources) or field "ingressClassName" (for VirtualServer/VirtualServerRoute/TransportServer resources) equal to the class. Additionally, the Ingress Controller processes resources that do not have the class set, which can be disabled by setting the `controller.useIngressClassOnly` parameter to `true`. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of kubernetes. | nginx +`controller.useIngressClassOnly` | Ignore Ingress resources without the `"kubernetes.io/ingress.class"` annotation. For kubernetes versions >= 1.18 this flag will be IGNORED. | false +`controller.setAsDefaultIngress` | New Ingresses without an `"ingressClassName"` field specified will be assigned the class specified in `controller.ingressClass`. Only for kubernetes versions >= 1.18. | false +`controller.watchNamespace` | Namespace to watch for Ingress resources. By default the Ingress controller watches all namespaces. | "" +`controller.enableCustomResources` | Enable the custom resources. | true +`controller.enablePreviewPolicies` | Enable preview policies. | false +`controller.enableTLSPassthrough` | Enable TLS Passthrough on port 443. Requires `controller.enableCustomResources`. | false +`controller.globalConfiguration.create` | Creates the GlobalConfiguration custom resource. Requires `controller.enableCustomResources`. | false +`controller.globalConfiguration.spec` | The spec of the GlobalConfiguration for defining the global configuration parameters of the Ingress Controller. | {} +`controller.enableSnippets` | Enable custom NGINX configuration snippets in VirtualServer, VirtualServerRoute and TransportServer resources. | false +`controller.healthStatus` | Add a location "/nginx-health" to the default server. The location responds with the 200 status code for any request. Useful for external health-checking of the Ingress controller. | false +`controller.healthStatusURI` | Sets the URI of health status location in the default server. Requires `controller.healthStatus`. | "/nginx-health" +`controller.nginxStatus.enable` | Enable the NGINX stub_status, or the NGINX Plus API. | true +`controller.nginxStatus.port` | Set the port where the NGINX stub_status or the NGINX Plus API is exposed. | 8080 +`controller.nginxStatus.allowCidrs` | Add IPv4 IP/CIDR blocks to the allow list for NGINX stub_status or the NGINX Plus API. Separate multiple IP/CIDR by commas. | 127.0.0.1 +`controller.priorityClassName` | The PriorityClass of the Ingress controller pods. | None +`controller.service.create` | Creates a service to expose the Ingress controller pods. | true +`controller.service.type` | The type of service to create for the Ingress controller. | LoadBalancer +`controller.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | Local +`controller.service.annotations` | The annotations of the Ingress controller service. | {} +`controller.service.extraLabels` | The extra labels of the service. | {} +`controller.service.loadBalancerIP` | The static IP address for the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | "" +`controller.service.externalIPs` | The list of external IPs for the Ingress controller service. | [] +`controller.service.loadBalancerSourceRanges` | The IP ranges (CIDR) that are allowed to access the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | [] +`controller.service.name` | The name of the service. | Autogenerated +`controller.service.customPorts` | A list of custom ports to expose through the Ingress controller service. Follows the conventional Kubernetes yaml syntax for service ports. | [] +`controller.service.httpPort.enable` | Enables the HTTP port for the Ingress controller service. | true +`controller.service.httpPort.port` | The HTTP port of the Ingress controller service. | 80 +`controller.service.httpPort.nodePort` | The custom NodePort for the HTTP port. Requires `controller.service.type` set to `NodePort`. | "" +`controller.service.httpPort.targetPort` | The target port of the HTTP port of the Ingress controller service. | 80 +`controller.service.httpsPort.enable` | Enables the HTTPS port for the Ingress controller service. | true +`controller.service.httpsPort.port` | The HTTPS port of the Ingress controller service. | 443 +`controller.service.httpsPort.nodePort` | The custom NodePort for the HTTPS port. Requires `controller.service.type` set to `NodePort`. | "" +`controller.service.httpsPort.targetPort` | The target port of the HTTPS port of the Ingress controller service. | 443 +`controller.serviceAccount.name` | The name of the service account of the Ingress controller pods. Used for RBAC. | Autogenerated +`controller.serviceAccount.imagePullSecretName` | The name of the secret containing docker registry credentials. Secret must exist in the same namespace as the helm release. | "" +`controller.reportIngressStatus.enable` | Updates the address field in the status of Ingress resources with an external address of the Ingress controller. You must also specify the source of the external address either through an external service via `controller.reportIngressStatus.externalService`, `controller.reportIngressStatus.ingressLink` or the `external-status-address` entry in the ConfigMap via `controller.config.entries`. **Note:** `controller.config.entries.external-status-address` takes precedence over the others. | true +`controller.reportIngressStatus.externalService` | Specifies the name of the service with the type LoadBalancer through which the Ingress controller is exposed externally. The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. `controller.reportIngressStatus.enable` must be set to `true`. The default is autogenerated and enabled when `controller.service.create` is set to `true` and `controller.service.type` is set to `LoadBalancer`. | Autogenerated +`controller.reportIngressStatus.ingressLink` | Specifies the name of the IngressLink resource, which exposes the Ingress Controller pods via a BIG-IP system. The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. `controller.reportIngressStatus.enable` must be set to `true`. | "" +`controller.reportIngressStatus.enableLeaderElection` | Enable Leader election to avoid multiple replicas of the controller reporting the status of Ingress resources. `controller.reportIngressStatus.enable` must be set to `true`. | true +`controller.reportIngressStatus.leaderElectionLockName` | Specifies the name of the ConfigMap, within the same namespace as the controller, used as the lock for leader election. controller.reportIngressStatus.enableLeaderElection must be set to true. | Autogenerated +`controller.reportIngressStatus.annotations` | The annotations of the leader election configmap. | {} +`controller.pod.annotations` | The annotations of the Ingress Controller pod. | {} +`controller.appprotect.enable` | Enables the App Protect module in the Ingress Controller. | false +`controller.readyStatus.enable` | Enables the readiness endpoint `"/nginx-ready"`. The endpoint returns a success code when NGINX has loaded all the config after the startup. This also configures a readiness probe for the Ingress Controller pods that uses the readiness endpoint. | true +`controller.readyStatus.port` | The HTTP port for the readiness endpoint. | 8081 +`controller.enableLatencyMetrics` | Enable collection of latency metrics for upstreams. Requires `prometheus.create`. | false +`rbac.create` | Configures RBAC. | true +`prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | false +`prometheus.port` | Configures the port to scrape the metrics. | 9113 +`prometheus.scheme` | Configures the HTTP scheme to use for connections to the Prometheus endpoint. | http +`prometheus.secret` | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Prometheus endpoint with TLS connections. | "" + + +## Notes +* The values-icp.yaml file is used for deploying the Ingress controller on IBM Cloud Private. See the [blog post](https://www.nginx.com/blog/nginx-ingress-controller-ibm-cloud-private/) for more details. diff --git a/kubernetes/helm_charts/nginx-ingress/chart-icon.png b/kubernetes/helm_charts/nginx-ingress/chart-icon.png new file mode 100644 index 0000000000..52961c9a6f Binary files /dev/null and b/kubernetes/helm_charts/nginx-ingress/chart-icon.png differ diff --git a/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_aplogconfs.yaml b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_aplogconfs.yaml new file mode 100644 index 0000000000..c06e46cb24 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_aplogconfs.yaml @@ -0,0 +1,65 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + name: aplogconfs.appprotect.f5.com +spec: + group: appprotect.f5.com + names: + kind: APLogConf + listKind: APLogConfList + plural: aplogconfs + singular: aplogconf + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: APLogConf is the Schema for the APLogConfs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: APLogConfSpec defines the desired state of APLogConf + properties: + content: + properties: + format: + enum: + - splunk + - arcsight + - default + - user-defined + - grpc + type: string + format_string: + type: string + max_message_size: + pattern: ^([1-9]|[1-5][0-9]|6[0-4])k$ + type: string + max_request_size: + pattern: ^([1-9]|[1-9][0-9]|[1-9][0-9]{2}|1[0-9]{3}|20[1-3][0-9]|204[1-8]|any)$ + type: string + type: object + filter: + properties: + request_type: + enum: + - all + - illegal + - blocked + type: string + type: object + type: object + type: object + served: true + storage: true diff --git a/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_appolicies.yaml b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_appolicies.yaml new file mode 100644 index 0000000000..b4309db1a3 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_appolicies.yaml @@ -0,0 +1,1790 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + name: appolicies.appprotect.f5.com +spec: + group: appprotect.f5.com + names: + kind: APPolicy + listKind: APPolicyList + plural: appolicies + singular: appolicy + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: APPolicyConfig is the Schema for the APPolicyconfigs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: APPolicySpec defines the desired state of APPolicy + properties: + modifications: + items: + properties: + action: + type: string + description: + type: string + entity: + properties: + name: + type: string + type: object + entityChanges: + properties: + type: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + modificationsReference: + properties: + link: + pattern: ^http + type: string + type: object + policy: + description: Defines the App Protect policy + properties: + applicationLanguage: + enum: + - iso-8859-10 + - iso-8859-6 + - windows-1255 + - auto-detect + - koi8-r + - gb18030 + - iso-8859-8 + - windows-1250 + - iso-8859-9 + - windows-1252 + - iso-8859-16 + - gb2312 + - iso-8859-2 + - iso-8859-5 + - windows-1257 + - windows-1256 + - iso-8859-13 + - windows-874 + - windows-1253 + - iso-8859-3 + - euc-jp + - utf-8 + - gbk + - windows-1251 + - big5 + - iso-8859-1 + - shift_jis + - euc-kr + - iso-8859-4 + - iso-8859-7 + - iso-8859-15 + type: string + blocking-settings: + properties: + evasions: + items: + properties: + description: + enum: + - '%u decoding' + - Apache whitespace + - Bad unescape + - Bare byte decoding + - Directory traversals + - IIS backslashes + - IIS Unicode codepoints + - Multiple decoding + type: string + enabled: + type: boolean + maxDecodingPasses: + type: integer + type: object + type: array + http-protocols: + items: + properties: + description: + enum: + - Unescaped space in URL + - Unparsable request content + - Several Content-Length headers + - 'POST request with Content-Length: 0' + - Null in request + - No Host header in HTTP/1.1 request + - Multiple host headers + - Host header contains IP address + - High ASCII characters in headers + - Header name with no header value + - CRLF characters before request start + - Content length should be a positive number + - Chunked request with Content-Length header + - Check maximum number of parameters + - Check maximum number of headers + - Body in GET or HEAD requests + - Bad multipart/form-data request parsing + - Bad multipart parameters parsing + - Bad HTTP version + - Bad host header value + type: string + enabled: + type: boolean + maxHeaders: + type: integer + maxParams: + type: integer + type: object + type: array + violations: + items: + properties: + alarm: + type: boolean + block: + type: boolean + description: + type: string + name: + enum: + - VIOL_GRPC_FORMAT + - VIOL_GRPC_MALFORMED + - VIOL_GRPC_METHOD + - VIOL_PARAMETER_ARRAY_VALUE + - VIOL_PARAMETER_VALUE_REGEXP + - VIOL_CSRF + - VIOL_PARAMETER_VALUE_BASE64 + - VIOL_MANDATORY_HEADER + - VIOL_HEADER_REPEATED + - VIOL_ASM_COOKIE_MODIFIED + - VIOL_BLACKLISTED_IP + - VIOL_COOKIE_EXPIRED + - VIOL_COOKIE_LENGTH + - VIOL_COOKIE_MALFORMED + - VIOL_COOKIE_MODIFIED + - VIOL_DATA_GUARD + - VIOL_ENCODING + - VIOL_EVASION + - VIOL_FILETYPE + - VIOL_FILE_UPLOAD + - VIOL_FILE_UPLOAD_IN_BODY + - VIOL_HEADER_LENGTH + - VIOL_HEADER_METACHAR + - VIOL_HTTP_PROTOCOL + - VIOL_HTTP_RESPONSE_STATUS + - VIOL_JSON_FORMAT + - VIOL_JSON_MALFORMED + - VIOL_JSON_SCHEMA + - VIOL_MANDATORY_PARAMETER + - VIOL_MANDATORY_REQUEST_BODY + - VIOL_METHOD + - VIOL_PARAMETER + - VIOL_PARAMETER_DATA_TYPE + - VIOL_PARAMETER_EMPTY_VALUE + - VIOL_PARAMETER_LOCATION + - VIOL_PARAMETER_MULTIPART_NULL_VALUE + - VIOL_PARAMETER_NAME_METACHAR + - VIOL_PARAMETER_NUMERIC_VALUE + - VIOL_PARAMETER_REPEATED + - VIOL_PARAMETER_STATIC_VALUE + - VIOL_PARAMETER_VALUE_LENGTH + - VIOL_PARAMETER_VALUE_METACHAR + - VIOL_POST_DATA_LENGTH + - VIOL_QUERY_STRING_LENGTH + - VIOL_RATING_THREAT + - VIOL_RATING_NEED_EXAMINATION + - VIOL_REQUEST_MAX_LENGTH + - VIOL_REQUEST_LENGTH + - VIOL_THREAT_CAMPAIGN + - VIOL_URL + - VIOL_URL_CONTENT_TYPE + - VIOL_URL_LENGTH + - VIOL_URL_METACHAR + - VIOL_XML_FORMAT + - VIOL_XML_MALFORMED + type: string + type: object + type: array + type: object + blockingSettingReference: + properties: + link: + pattern: ^http + type: string + type: object + bot-defense: + properties: + mitigations: + properties: + anomalies: + items: + properties: + action: + enum: + - alarm + - block + - default + - detect + - ignore + type: string + name: + type: string + scoreThreshold: + pattern: '[0-9]|[1-9][0-9]|1[0-4][0-9]|150|default' + type: string + type: object + type: array + browsers: + items: + properties: + $action: + enum: + - delete + type: string + action: + enum: + - alarm + - block + - detect + type: string + maxVersion: + maximum: 2147483647 + minimum: 0 + type: integer + minVersion: + maximum: 2147483647 + minimum: 0 + type: integer + name: + type: string + type: object + type: array + classes: + items: + properties: + action: + enum: + - alarm + - block + - detect + - ignore + type: string + name: + enum: + - browser + - malicious-bot + - suspicious-browser + - trusted-bot + - unknown + - untrusted-bot + type: string + type: object + type: array + signatures: + items: + properties: + action: + enum: + - alarm + - block + - detect + - ignore + type: string + name: + type: string + type: object + type: array + type: object + settings: + properties: + isEnabled: + type: boolean + type: object + type: object + browser-definitions: + items: + properties: + $action: + enum: + - delete + type: string + isUserDefined: + type: boolean + matchRegex: + type: string + matchString: + type: string + name: + type: string + type: object + type: array + caseInsensitive: + type: boolean + character-sets: + items: + properties: + characterSet: + items: + properties: + isAllowed: + type: boolean + metachar: + type: string + type: object + type: array + characterSetType: + enum: + - gwt-content + - header + - json-content + - parameter-name + - parameter-value + - plain-text-content + - url + - xml-content + type: string + type: object + type: array + characterSetReference: + properties: + link: + pattern: ^http + type: string + type: object + cookie-settings: + properties: + maximumCookieHeaderLength: + pattern: any|\d+ + type: string + type: object + cookieReference: + properties: + link: + pattern: ^http + type: string + type: object + cookieSettingsReference: + properties: + link: + pattern: ^http + type: string + type: object + cookies: + items: + properties: + accessibleOnlyThroughTheHttpProtocol: + type: boolean + attackSignaturesCheck: + type: boolean + decodeValueAsBase64: + enum: + - enabled + - disabled + - required + type: string + enforcementType: + type: string + insertSameSiteAttribute: + enum: + - lax + - none + - none-value + - strict + type: string + name: + type: string + securedOverHttpsConnection: + type: boolean + signatureOverrides: + items: + properties: + enabled: + type: boolean + name: + type: string + signatureId: + type: integer + tag: + type: string + type: object + type: array + type: + enum: + - explicit + - wildcard + type: string + wildcardOrder: + type: integer + type: object + type: array + csrf-protection: + properties: + enabled: + type: boolean + expirationTimeInSeconds: + pattern: disabled|\d+ + type: string + sslOnly: + type: boolean + type: object + csrf-urls: + items: + properties: + $action: + enum: + - delete + type: string + enforcementAction: + enum: + - verify-origin + - none + type: string + method: + enum: + - GET + - POST + - any + type: string + url: + type: string + wildcardOrder: + type: integer + type: object + type: array + data-guard: + properties: + creditCardNumbers: + type: boolean + enabled: + type: boolean + enforcementMode: + enum: + - ignore-urls-in-list + - enforce-urls-in-list + type: string + enforcementUrls: + items: + type: string + type: array + lastCcnDigitsToExpose: + type: integer + lastSsnDigitsToExpose: + type: integer + maskData: + type: boolean + usSocialSecurityNumbers: + type: boolean + type: object + dataGuardReference: + properties: + link: + pattern: ^http + type: string + type: object + description: + type: string + enablePassiveMode: + type: boolean + enforcementMode: + enum: + - transparent + - blocking + type: string + filetypeReference: + properties: + link: + pattern: ^http + type: string + type: object + filetypes: + items: + properties: + action: + type: string + allowed: + type: boolean + checkPostDataLength: + type: boolean + checkQueryStringLength: + type: boolean + checkRequestLength: + type: boolean + checkUrlLength: + type: boolean + name: + type: string + postDataLength: + type: integer + queryStringLength: + type: integer + requestLength: + type: integer + responseCheck: + type: boolean + type: + enum: + - explicit + - wildcard + type: string + urlLength: + type: integer + wildcardOrder: + type: integer + type: object + type: array + fullPath: + type: string + general: + properties: + allowedResponseCodes: + items: + format: int32 + maximum: 999 + minimum: 100 + type: integer + type: array + customXffHeaders: + items: + type: string + type: array + maskCreditCardNumbersInRequest: + type: boolean + trustXff: + type: boolean + type: object + generalReference: + properties: + link: + pattern: ^http + type: string + type: object + grpc-profiles: + items: + properties: + associateUrls: + type: boolean + attackSignaturesCheck: + type: boolean + defenseAttributes: + properties: + allowUnknownFields: + type: boolean + maximumDataLength: + pattern: any|\d+ + type: string + type: object + description: + type: string + hasIdlFiles: + type: boolean + idlFiles: + items: + properties: + idlFile: + properties: + contents: + type: string + fileName: + type: string + isBase64: + type: boolean + type: object + isPrimary: + type: boolean + primaryIdlFileName: + type: string + type: object + type: array + metacharElementCheck: + type: boolean + name: + type: string + signatureOverrides: + items: + properties: + enabled: + type: boolean + name: + type: string + signatureId: + type: integer + tag: + type: string + type: object + type: array + type: object + type: array + header-settings: + properties: + maximumHttpHeaderLength: + pattern: any|\d+ + type: string + type: object + headerReference: + properties: + link: + pattern: ^http + type: string + type: object + headerSettingsReference: + properties: + link: + pattern: ^http + type: string + type: object + headers: + items: + properties: + allowRepeatedOccurrences: + type: boolean + base64Decoding: + type: boolean + checkSignatures: + type: boolean + decodeValueAsBase64: + enum: + - enabled + - disabled + - required + type: string + htmlNormalization: + type: boolean + mandatory: + type: boolean + maskValueInLogs: + type: boolean + name: + type: string + normalizationViolations: + type: boolean + percentDecoding: + type: boolean + signatureOverrides: + items: + properties: + enabled: + type: boolean + name: + type: string + signatureId: + type: integer + tag: + type: string + type: object + type: array + type: + enum: + - explicit + - wildcard + type: string + urlNormalization: + type: boolean + wildcardOrder: + type: integer + type: object + type: array + host-names: + items: + properties: + $action: + enum: + - delete + type: string + includeSubdomains: + type: boolean + name: + type: string + type: object + type: array + idl-files: + items: + properties: + contents: + type: string + fileName: + type: string + isBase64: + type: boolean + type: object + type: array + json-profiles: + items: + properties: + attackSignaturesCheck: + type: boolean + defenseAttributes: + properties: + maximumArrayLength: + pattern: any|\d+ + type: string + maximumStructureDepth: + pattern: any|\d+ + type: string + maximumTotalLengthOfJSONData: + pattern: any|\d+ + type: string + maximumValueLength: + pattern: any|\d+ + type: string + tolerateJSONParsingWarnings: + type: boolean + type: object + description: + type: string + handleJsonValuesAsParameters: + type: boolean + hasValidationFiles: + type: boolean + metacharOverrides: + items: + properties: + isAllowed: + type: boolean + metachar: + type: string + type: object + type: array + name: + type: string + signatureOverrides: + items: + properties: + enabled: + type: boolean + name: + type: string + signatureId: + type: integer + tag: + type: string + type: object + type: array + validationFiles: + items: + properties: + importUrl: + type: string + isPrimary: + type: boolean + jsonValidationFile: + properties: + contents: + type: string + fileName: + type: string + isBase64: + type: boolean + type: object + type: object + type: array + type: object + type: array + json-validation-files: + items: + properties: + contents: + type: string + fileName: + type: string + isBase64: + type: boolean + type: object + type: array + jsonProfileReference: + properties: + link: + pattern: ^http + type: string + type: object + jsonValidationFileReference: + properties: + link: + pattern: ^http + type: string + type: object + methodReference: + properties: + link: + pattern: ^http + type: string + type: object + methods: + items: + properties: + action: + type: string + name: + type: string + type: object + type: array + name: + type: string + open-api-files: + items: + properties: + link: + pattern: ^http + type: string + type: object + type: array + parameterReference: + properties: + link: + pattern: ^http + type: string + type: object + parameters: + items: + properties: + allowEmptyValue: + type: boolean + allowRepeatedParameterName: + type: boolean + arraySerializationFormat: + enum: + - csv + - form + - label + - matrix + - multi + - multipart + - pipe + - ssv + - tsv + type: string + attackSignaturesCheck: + type: boolean + checkMaxValue: + type: boolean + checkMaxValueLength: + type: boolean + checkMetachars: + type: boolean + checkMinValue: + type: boolean + checkMinValueLength: + type: boolean + checkMultipleOfValue: + type: boolean + contentProfile: + properties: + name: + type: string + type: object + dataType: + enum: + - alpha-numeric + - binary + - boolean + - decimal + - email + - integer + - none + - phone + type: string + decodeValueAsBase64: + enum: + - enabled + - disabled + - required + type: string + disallowFileUploadOfExecutables: + type: boolean + enableRegularExpression: + type: boolean + exclusiveMax: + type: boolean + exclusiveMin: + type: boolean + isBase64: + type: boolean + isCookie: + type: boolean + isHeader: + type: boolean + level: + enum: + - global + - url + type: string + mandatory: + type: boolean + maximumLength: + type: integer + maximumValue: + type: integer + metacharsOnParameterValueCheck: + type: boolean + minimumLength: + type: integer + minimumValue: + type: integer + multipleOf: + type: integer + name: + type: string + nameMetacharOverrides: + items: + properties: + isAllowed: + type: boolean + metachar: + type: string + type: object + type: array + objectSerializationStyle: + type: string + parameterEnumValues: + items: + type: string + type: array + parameterLocation: + enum: + - any + - cookie + - form-data + - header + - path + - query + type: string + regularExpression: + type: string + sensitiveParameter: + type: boolean + signatureOverrides: + items: + properties: + enabled: + type: boolean + name: + type: string + signatureId: + type: integer + tag: + type: string + type: object + type: array + staticValues: + type: string + type: + enum: + - explicit + - wildcard + type: string + url: + type: object + valueMetacharOverrides: + items: + properties: + isAllowed: + type: boolean + metachar: + type: string + type: object + type: array + valueType: + enum: + - array + - auto-detect + - dynamic-content + - dynamic-parameter-name + - ignore + - json + - object + - openapi-array + - static-content + - user-input + - xml + type: string + wildcardOrder: + type: integer + type: object + type: array + response-pages: + items: + properties: + ajaxActionType: + enum: + - alert-popup + - custom + - redirect + type: string + ajaxCustomContent: + type: string + ajaxEnabled: + type: boolean + ajaxPopupMessage: + type: string + ajaxRedirectUrl: + type: string + grpcStatusCode: + pattern: ABORTED|ALREADY_EXISTS|CANCELLED|DATA_LOSS|DEADLINE_EXCEEDED|FAILED_PRECONDITION|INTERNAL|INVALID_ARGUMENT|NOT_FOUND|OK|OUT_OF_RANGE|PERMISSION_DENIED|RESOURCE_EXHAUSTED|UNAUTHENTICATED|UNAVAILABLE|UNIMPLEMENTED|UNKNOWN|d+ + type: string + grpcStatusMessage: + type: string + responseActionType: + enum: + - custom + - default + - erase-cookies + - redirect + - soap-fault + type: string + responseContent: + type: string + responseHeader: + type: string + responsePageType: + enum: + - ajax + - ajax-login + - captcha + - captcha-fail + - default + - failed-login-honeypot + - failed-login-honeypot-ajax + - hijack + - leaked-credentials + - leaked-credentials-ajax + - mobile + - persistent-flow + - xml + - grpc + type: string + responseRedirectUrl: + type: string + type: object + type: array + responsePageReference: + properties: + link: + pattern: ^http + type: string + type: object + sensitive-parameters: + items: + properties: + name: + type: string + type: object + type: array + sensitiveParameterReference: + properties: + link: + pattern: ^http + type: string + type: object + server-technologies: + items: + properties: + serverTechnologyName: + enum: + - Jenkins + - SharePoint + - Oracle Application Server + - Python + - Oracle Identity Manager + - Spring Boot + - CouchDB + - SQLite + - Handlebars + - Mustache + - Prototype + - Zend + - Redis + - Underscore.js + - Ember.js + - ZURB Foundation + - ef.js + - Vue.js + - UIKit + - TYPO3 CMS + - RequireJS + - React + - MooTools + - Laravel + - GraphQL + - Google Web Toolkit + - Express.js + - CodeIgniter + - Backbone.js + - AngularJS + - JavaScript + - Nginx + - Jetty + - Joomla + - JavaServer Faces (JSF) + - Ruby + - MongoDB + - Django + - Node.js + - Citrix + - JBoss + - Elasticsearch + - Apache Struts + - XML + - PostgreSQL + - IBM DB2 + - Sybase/ASE + - CGI + - Proxy Servers + - SSI (Server Side Includes) + - Cisco + - Novell + - Macromedia JRun + - BEA Systems WebLogic Server + - Lotus Domino + - MySQL + - Oracle + - Microsoft SQL Server + - PHP + - Outlook Web Access + - Apache/NCSA HTTP Server + - Apache Tomcat + - WordPress + - Macromedia ColdFusion + - Unix/Linux + - Microsoft Windows + - ASP.NET + - Front Page Server Extensions (FPSE) + - IIS + - WebDAV + - ASP + - Java Servlets/JSP + - jQuery + type: string + type: object + type: array + serverTechnologyReference: + properties: + link: + pattern: ^http + type: string + type: object + signature-requirements: + items: + properties: + tag: + type: string + type: object + type: array + signature-sets: + items: + properties: + alarm: + type: boolean + block: + type: boolean + name: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + signature-settings: + properties: + attackSignatureFalsePositiveMode: + enum: + - detect + - detect-and-allow + - disabled + type: string + minimumAccuracyForAutoAddedSignatures: + enum: + - high + - low + - medium + type: string + type: object + signatureReference: + properties: + link: + pattern: ^http + type: string + type: object + signatureSetReference: + properties: + link: + pattern: ^http + type: string + type: object + signatureSettingReference: + properties: + link: + pattern: ^http + type: string + type: object + signatures: + items: + properties: + enabled: + type: boolean + name: + type: string + signatureId: + type: integer + tag: + type: string + type: object + type: array + softwareVersion: + type: string + template: + properties: + name: + type: string + type: object + threat-campaigns: + items: + properties: + isEnabled: + type: boolean + name: + type: string + type: object + type: array + threatCampaignReference: + properties: + link: + pattern: ^http + type: string + type: object + urlReference: + properties: + link: + pattern: ^http + type: string + type: object + urls: + items: + properties: + $action: + enum: + - delete + type: string + allowRenderingInFrames: + enum: + - never + - only-same + type: string + allowRenderingInFramesOnlyFrom: + type: string + attackSignaturesCheck: + type: boolean + clickjackingProtection: + type: boolean + description: + type: string + disallowFileUploadOfExecutables: + type: boolean + html5CrossOriginRequestsEnforcement: + properties: + allowOriginsEnforcementMode: + enum: + - replace-with + - unmodified + type: string + checkAllowedMethods: + type: boolean + crossDomainAllowedOrigin: + items: + properties: + includeSubDomains: + type: boolean + originName: + type: string + originPort: + pattern: any|\d+ + type: string + originProtocol: + enum: + - http + - http/https + - https + type: string + type: object + type: array + enforcementMode: + enum: + - disabled + - enforce + type: string + type: object + isAllowed: + type: boolean + mandatoryBody: + type: boolean + metacharOverrides: + items: + properties: + isAllowed: + type: boolean + metachar: + type: string + type: object + type: array + metacharsOnUrlCheck: + type: boolean + method: + enum: + - ACL + - BCOPY + - BDELETE + - BMOVE + - BPROPFIND + - BPROPPATCH + - CHECKIN + - CHECKOUT + - CONNECT + - COPY + - DELETE + - GET + - HEAD + - LINK + - LOCK + - MERGE + - MKCOL + - MKWORKSPACE + - MOVE + - NOTIFY + - OPTIONS + - PATCH + - POLL + - POST + - PROPFIND + - PROPPATCH + - PUT + - REPORT + - RPC_IN_DATA + - RPC_OUT_DATA + - SEARCH + - SUBSCRIBE + - TRACE + - TRACK + - UNLINK + - UNLOCK + - UNSUBSCRIBE + - VERSION_CONTROL + - X-MS-ENUMATTS + - '*' + type: string + methodOverrides: + items: + properties: + allowed: + type: boolean + method: + enum: + - ACL + - BCOPY + - BDELETE + - BMOVE + - BPROPFIND + - BPROPPATCH + - CHECKIN + - CHECKOUT + - CONNECT + - COPY + - DELETE + - GET + - HEAD + - LINK + - LOCK + - MERGE + - MKCOL + - MKWORKSPACE + - MOVE + - NOTIFY + - OPTIONS + - PATCH + - POLL + - POST + - PROPFIND + - PROPPATCH + - PUT + - REPORT + - RPC_IN_DATA + - RPC_OUT_DATA + - SEARCH + - SUBSCRIBE + - TRACE + - TRACK + - UNLINK + - UNLOCK + - UNSUBSCRIBE + - VERSION_CONTROL + - X-MS-ENUMATTS + type: string + type: object + type: array + methodsOverrideOnUrlCheck: + type: boolean + name: + type: string + operationId: + type: string + positionalParameters: + items: + properties: + parameter: + properties: + allowEmptyValue: + type: boolean + allowRepeatedParameterName: + type: boolean + arraySerializationFormat: + enum: + - csv + - form + - label + - matrix + - multi + - multipart + - pipe + - ssv + - tsv + type: string + attackSignaturesCheck: + type: boolean + checkMaxValue: + type: boolean + checkMaxValueLength: + type: boolean + checkMetachars: + type: boolean + checkMinValue: + type: boolean + checkMinValueLength: + type: boolean + checkMultipleOfValue: + type: boolean + contentProfile: + properties: + name: + type: string + type: object + dataType: + enum: + - alpha-numeric + - binary + - boolean + - decimal + - email + - integer + - none + - phone + type: string + decodeValueAsBase64: + enum: + - enabled + - disabled + - required + type: string + disallowFileUploadOfExecutables: + type: boolean + enableRegularExpression: + type: boolean + exclusiveMax: + type: boolean + exclusiveMin: + type: boolean + isBase64: + type: boolean + isCookie: + type: boolean + isHeader: + type: boolean + level: + enum: + - global + - url + type: string + mandatory: + type: boolean + maximumLength: + type: integer + maximumValue: + type: integer + metacharsOnParameterValueCheck: + type: boolean + minimumLength: + type: integer + minimumValue: + type: integer + multipleOf: + type: integer + name: + type: string + nameMetacharOverrides: + items: + properties: + isAllowed: + type: boolean + metachar: + type: string + type: object + type: array + objectSerializationStyle: + type: string + parameterEnumValues: + items: + type: string + type: array + parameterLocation: + enum: + - any + - cookie + - form-data + - header + - path + - query + type: string + regularExpression: + type: string + sensitiveParameter: + type: boolean + signatureOverrides: + items: + properties: + enabled: + type: boolean + name: + type: string + signatureId: + type: integer + tag: + type: string + type: object + type: array + staticValues: + type: string + type: + enum: + - explicit + - wildcard + type: string + url: + type: object + valueMetacharOverrides: + items: + properties: + isAllowed: + type: boolean + metachar: + type: string + type: object + type: array + valueType: + enum: + - array + - auto-detect + - dynamic-content + - dynamic-parameter-name + - ignore + - json + - object + - openapi-array + - static-content + - user-input + - xml + type: string + wildcardOrder: + type: integer + type: object + urlSegmentIndex: + type: integer + type: object + type: array + protocol: + enum: + - http + - https + type: string + signatureOverrides: + items: + properties: + enabled: + type: boolean + name: + type: string + signatureId: + type: integer + tag: + type: string + type: object + type: array + type: + enum: + - explicit + - wildcard + type: string + urlContentProfiles: + items: + properties: + contentProfile: + properties: + name: + type: string + type: object + headerName: + type: string + headerOrder: + type: string + headerValue: + type: string + name: + type: string + type: + enum: + - apply-content-signatures + - apply-value-and-content-signatures + - disallow + - do-nothing + - form-data + - gwt + - json + - xml + - grpc + type: string + type: object + type: array + wildcardOrder: + type: integer + type: object + type: array + whitelist-ips: + items: + properties: + blockRequests: + enum: + - always + - never + - policy-default + type: string + ipAddress: + pattern: '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' + type: string + ipMask: + pattern: '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' + type: string + type: object + type: array + whitelistIpReference: + properties: + link: + pattern: ^http + type: string + type: object + xml-profiles: + items: + properties: + attackSignaturesCheck: + type: boolean + defenseAttributes: + properties: + allowCDATA: + type: boolean + allowDTDs: + type: boolean + allowExternalReferences: + type: boolean + allowProcessingInstructions: + type: boolean + maximumAttributeValueLength: + pattern: any|\d+ + type: string + maximumAttributesPerElement: + pattern: any|\d+ + type: string + maximumChildrenPerElement: + pattern: any|\d+ + type: string + maximumDocumentDepth: + pattern: any|\d+ + type: string + maximumDocumentSize: + pattern: any|\d+ + type: string + maximumElements: + pattern: any|\d+ + type: string + maximumNSDeclarations: + pattern: any|\d+ + type: string + maximumNameLength: + pattern: any|\d+ + type: string + maximumNamespaceLength: + pattern: any|\d+ + type: string + tolerateCloseTagShorthand: + type: boolean + tolerateLeadingWhiteSpace: + type: boolean + tolerateNumericNames: + type: boolean + type: object + description: + type: string + enableWss: + type: boolean + followSchemaLinks: + type: boolean + name: + type: string + signatureOverrides: + items: + properties: + enabled: + type: boolean + name: + type: string + signatureId: + type: integer + tag: + type: string + type: object + type: array + type: object + type: array + xml-validation-files: + items: + properties: + contents: + type: string + fileName: + type: string + isBase64: + type: boolean + type: object + type: array + xmlProfileReference: + properties: + link: + pattern: ^http + type: string + type: object + xmlValidationFileReference: + properties: + link: + pattern: ^http + type: string + type: object + type: object + type: object + type: object + served: true + storage: true diff --git a/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_apusersigs.yaml b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_apusersigs.yaml new file mode 100644 index 0000000000..044eeb4563 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_apusersigs.yaml @@ -0,0 +1,93 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + name: apusersigs.appprotect.f5.com +spec: + group: appprotect.f5.com + names: + kind: APUserSig + listKind: APUserSigList + plural: apusersigs + singular: apusersig + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: APUserSig is the Schema for the apusersigs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: APUserSigSpec defines the desired state of APUserSig + properties: + properties: + type: string + signatures: + items: + properties: + accuracy: + enum: + - high + - medium + - low + type: string + attackType: + properties: + name: + type: string + type: object + description: + type: string + name: + type: string + references: + properties: + type: + enum: + - bugtraq + - cve + - nessus + - url + type: string + value: + type: string + type: object + risk: + enum: + - high + - medium + - low + type: string + rule: + type: string + signatureType: + enum: + - request + - response + type: string + systems: + items: + properties: + name: + type: string + type: object + type: array + type: object + type: array + tag: + type: string + type: object + type: object + served: true + storage: true diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_globalconfigurations.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_globalconfigurations.yaml new file mode 100644 index 0000000000..3f1a248b1f --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_globalconfigurations.yaml @@ -0,0 +1,56 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 + creationTimestamp: null + name: globalconfigurations.k8s.nginx.org +spec: + group: k8s.nginx.org + names: + kind: GlobalConfiguration + listKind: GlobalConfigurationList + plural: globalconfigurations + shortNames: + - gc + singular: globalconfiguration + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: GlobalConfiguration defines the GlobalConfiguration resource. + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GlobalConfigurationSpec is the spec of the GlobalConfiguration resource. + type: object + properties: + listeners: + type: array + items: + description: Listener defines a listener. + type: object + properties: + name: + type: string + port: + type: integer + protocol: + type: string + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_policies.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_policies.yaml new file mode 100644 index 0000000000..ec583f5448 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_policies.yaml @@ -0,0 +1,176 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 + creationTimestamp: null + name: policies.k8s.nginx.org +spec: + group: k8s.nginx.org + names: + kind: Policy + listKind: PolicyList + plural: policies + shortNames: + - pol + singular: policy + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the Policy. If the resource has a valid status, it means it has been validated and accepted by the Ingress Controller. + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: Policy defines a Policy for VirtualServer and VirtualServerRoute resources. + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: PolicySpec is the spec of the Policy resource. The spec includes multiple fields, where each field represents a different policy. Only one policy (field) is allowed. + type: object + properties: + accessControl: + description: 'AccessControl defines an access policy based on the source IP of a request. policy status: production-ready' + type: object + properties: + allow: + type: array + items: + type: string + deny: + type: array + items: + type: string + egressMTLS: + description: 'EgressMTLS defines an Egress MTLS policy. policy status: preview' + type: object + properties: + ciphers: + type: string + protocols: + type: string + serverName: + type: boolean + sessionReuse: + type: boolean + sslName: + type: string + tlsSecret: + type: string + trustedCertSecret: + type: string + verifyDepth: + type: integer + verifyServer: + type: boolean + ingressMTLS: + description: 'IngressMTLS defines an Ingress MTLS policy. policy status: preview' + type: object + properties: + clientCertSecret: + type: string + verifyClient: + type: string + verifyDepth: + type: integer + jwt: + description: 'JWTAuth holds JWT authentication configuration. policy status: preview' + type: object + properties: + realm: + type: string + secret: + type: string + token: + type: string + oidc: + description: OIDC defines an Open ID Connect policy. + type: object + properties: + authEndpoint: + type: string + clientID: + type: string + clientSecret: + type: string + jwksURI: + type: string + redirectURI: + type: string + scope: + type: string + tokenEndpoint: + type: string + rateLimit: + description: 'RateLimit defines a rate limit policy. policy status: preview' + type: object + properties: + burst: + type: integer + delay: + type: integer + dryRun: + type: boolean + key: + type: string + logLevel: + type: string + noDelay: + type: boolean + rate: + type: string + rejectCode: + type: integer + zoneSize: + type: string + waf: + description: 'WAF defines an WAF policy. policy status: preview' + type: object + properties: + apPolicy: + type: string + enable: + type: boolean + securityLog: + description: SecurityLog defines the security log of a WAF policy. + type: object + properties: + apLogConf: + type: string + enable: + type: boolean + logDest: + type: string + status: + description: PolicyStatus is the status of the policy resource + type: object + properties: + message: + type: string + reason: + type: string + state: + type: string + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_transportservers.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_transportservers.yaml new file mode 100644 index 0000000000..4a28dd9f12 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_transportservers.yaml @@ -0,0 +1,157 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 + creationTimestamp: null + name: transportservers.k8s.nginx.org +spec: + group: k8s.nginx.org + names: + kind: TransportServer + listKind: TransportServerList + plural: transportservers + shortNames: + - ts + singular: transportserver + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the TransportServer. If the resource has a valid status, it means it has been validated and accepted by the Ingress Controller. + jsonPath: .status.state + name: State + type: string + - jsonPath: .status.reason + name: Reason + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TransportServer defines the TransportServer resource. + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TransportServerSpec is the spec of the TransportServer resource. + type: object + properties: + action: + description: Action defines an action. + type: object + properties: + pass: + type: string + host: + type: string + ingressClassName: + type: string + listener: + description: TransportServerListener defines a listener for a TransportServer. + type: object + properties: + name: + type: string + protocol: + type: string + serverSnippets: + type: string + sessionParameters: + description: SessionParameters defines session parameters. + type: object + properties: + timeout: + type: string + streamSnippets: + type: string + upstreamParameters: + description: UpstreamParameters defines parameters for an upstream. + type: object + properties: + connectTimeout: + type: string + nextUpstream: + type: boolean + nextUpstreamTimeout: + type: string + nextUpstreamTries: + type: integer + udpRequests: + type: integer + udpResponses: + type: integer + upstreams: + type: array + items: + description: Upstream defines an upstream. + type: object + properties: + failTimeout: + type: string + healthCheck: + description: HealthCheck defines the parameters for active Upstream HealthChecks. + type: object + properties: + enable: + type: boolean + fails: + type: integer + interval: + type: string + jitter: + type: string + match: + description: Match defines the parameters of a custom health check. + type: object + properties: + expect: + type: string + send: + type: string + passes: + type: integer + port: + type: integer + timeout: + type: string + loadBalancingMethod: + type: string + maxConns: + type: integer + maxFails: + type: integer + name: + type: string + port: + type: integer + service: + type: string + status: + description: TransportServerStatus defines the status for the TransportServer resource. + type: object + properties: + message: + type: string + reason: + type: string + state: + type: string + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualserverroutes.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualserverroutes.yaml new file mode 100644 index 0000000000..5d2816cf70 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualserverroutes.yaml @@ -0,0 +1,621 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 + creationTimestamp: null + name: virtualserverroutes.k8s.nginx.org +spec: + group: k8s.nginx.org + names: + kind: VirtualServerRoute + listKind: VirtualServerRouteList + plural: virtualserverroutes + shortNames: + - vsr + singular: virtualserverroute + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the VirtualServerRoute. If the resource has a valid status, it means it has been validated and accepted by the Ingress Controller. + jsonPath: .status.state + name: State + type: string + - jsonPath: .spec.host + name: Host + type: string + - jsonPath: .status.externalEndpoints[*].ip + name: IP + type: string + - jsonPath: .status.externalEndpoints[*].ports + name: Ports + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VirtualServerRoute defines the VirtualServerRoute resource. + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VirtualServerRouteSpec is the spec of the VirtualServerRoute resource. + type: object + properties: + host: + type: string + ingressClassName: + type: string + subroutes: + type: array + items: + description: Route defines a route. + type: object + properties: + action: + description: Action defines an action. + type: object + properties: + pass: + type: string + proxy: + description: ActionProxy defines a proxy in an Action. + type: object + properties: + requestHeaders: + description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy. + type: object + properties: + pass: + type: boolean + set: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + responseHeaders: + description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy. + type: object + properties: + add: + type: array + items: + description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive. + type: object + properties: + always: + type: boolean + name: + type: string + value: + type: string + hide: + type: array + items: + type: string + ignore: + type: array + items: + type: string + pass: + type: array + items: + type: string + rewritePath: + type: string + upstream: + type: string + redirect: + description: ActionRedirect defines a redirect in an Action. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ActionReturn defines a return in an Action. + type: object + properties: + body: + type: string + code: + type: integer + type: + type: string + errorPages: + type: array + items: + description: ErrorPage defines an ErrorPage in a Route. + type: object + properties: + codes: + type: array + items: + type: integer + redirect: + description: ErrorPageRedirect defines a redirect for an ErrorPage. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ErrorPageReturn defines a return for an ErrorPage. + type: object + properties: + body: + type: string + code: + type: integer + headers: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + type: + type: string + location-snippets: + type: string + matches: + type: array + items: + description: Match defines a match. + type: object + properties: + action: + description: Action defines an action. + type: object + properties: + pass: + type: string + proxy: + description: ActionProxy defines a proxy in an Action. + type: object + properties: + requestHeaders: + description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy. + type: object + properties: + pass: + type: boolean + set: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + responseHeaders: + description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy. + type: object + properties: + add: + type: array + items: + description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive. + type: object + properties: + always: + type: boolean + name: + type: string + value: + type: string + hide: + type: array + items: + type: string + ignore: + type: array + items: + type: string + pass: + type: array + items: + type: string + rewritePath: + type: string + upstream: + type: string + redirect: + description: ActionRedirect defines a redirect in an Action. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ActionReturn defines a return in an Action. + type: object + properties: + body: + type: string + code: + type: integer + type: + type: string + conditions: + type: array + items: + description: Condition defines a condition in a MatchRule. + type: object + properties: + argument: + type: string + cookie: + type: string + header: + type: string + value: + type: string + variable: + type: string + splits: + type: array + items: + description: Split defines a split. + type: object + properties: + action: + description: Action defines an action. + type: object + properties: + pass: + type: string + proxy: + description: ActionProxy defines a proxy in an Action. + type: object + properties: + requestHeaders: + description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy. + type: object + properties: + pass: + type: boolean + set: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + responseHeaders: + description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy. + type: object + properties: + add: + type: array + items: + description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive. + type: object + properties: + always: + type: boolean + name: + type: string + value: + type: string + hide: + type: array + items: + type: string + ignore: + type: array + items: + type: string + pass: + type: array + items: + type: string + rewritePath: + type: string + upstream: + type: string + redirect: + description: ActionRedirect defines a redirect in an Action. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ActionReturn defines a return in an Action. + type: object + properties: + body: + type: string + code: + type: integer + type: + type: string + weight: + type: integer + path: + type: string + policies: + type: array + items: + description: PolicyReference references a policy by name and an optional namespace. + type: object + properties: + name: + type: string + namespace: + type: string + route: + type: string + splits: + type: array + items: + description: Split defines a split. + type: object + properties: + action: + description: Action defines an action. + type: object + properties: + pass: + type: string + proxy: + description: ActionProxy defines a proxy in an Action. + type: object + properties: + requestHeaders: + description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy. + type: object + properties: + pass: + type: boolean + set: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + responseHeaders: + description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy. + type: object + properties: + add: + type: array + items: + description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive. + type: object + properties: + always: + type: boolean + name: + type: string + value: + type: string + hide: + type: array + items: + type: string + ignore: + type: array + items: + type: string + pass: + type: array + items: + type: string + rewritePath: + type: string + upstream: + type: string + redirect: + description: ActionRedirect defines a redirect in an Action. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ActionReturn defines a return in an Action. + type: object + properties: + body: + type: string + code: + type: integer + type: + type: string + weight: + type: integer + upstreams: + type: array + items: + description: Upstream defines an upstream. + type: object + properties: + buffer-size: + type: string + buffering: + type: boolean + buffers: + description: UpstreamBuffers defines Buffer Configuration for an Upstream. + type: object + properties: + number: + type: integer + size: + type: string + client-max-body-size: + type: string + connect-timeout: + type: string + fail-timeout: + type: string + healthCheck: + description: HealthCheck defines the parameters for active Upstream HealthChecks. + type: object + properties: + connect-timeout: + type: string + enable: + type: boolean + fails: + type: integer + headers: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + interval: + type: string + jitter: + type: string + passes: + type: integer + path: + type: string + port: + type: integer + read-timeout: + type: string + send-timeout: + type: string + statusMatch: + type: string + tls: + description: UpstreamTLS defines a TLS configuration for an Upstream. + type: object + properties: + enable: + type: boolean + keepalive: + type: integer + lb-method: + type: string + max-conns: + type: integer + max-fails: + type: integer + name: + type: string + next-upstream: + type: string + next-upstream-timeout: + type: string + next-upstream-tries: + type: integer + port: + type: integer + queue: + description: UpstreamQueue defines Queue Configuration for an Upstream. + type: object + properties: + size: + type: integer + timeout: + type: string + read-timeout: + type: string + send-timeout: + type: string + service: + type: string + sessionCookie: + description: SessionCookie defines the parameters for session persistence. + type: object + properties: + domain: + type: string + enable: + type: boolean + expires: + type: string + httpOnly: + type: boolean + name: + type: string + path: + type: string + secure: + type: boolean + slow-start: + type: string + subselector: + type: object + additionalProperties: + type: string + tls: + description: UpstreamTLS defines a TLS configuration for an Upstream. + type: object + properties: + enable: + type: boolean + use-cluster-ip: + type: boolean + status: + description: VirtualServerRouteStatus defines the status for the VirtualServerRoute resource. + type: object + properties: + externalEndpoints: + type: array + items: + description: ExternalEndpoint defines the IP and ports used to connect to this resource. + type: object + properties: + ip: + type: string + ports: + type: string + message: + type: string + reason: + type: string + referencedBy: + type: string + state: + type: string + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualservers.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualservers.yaml new file mode 100644 index 0000000000..ad2c173611 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualservers.yaml @@ -0,0 +1,649 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.0 + creationTimestamp: null + name: virtualservers.k8s.nginx.org +spec: + group: k8s.nginx.org + names: + kind: VirtualServer + listKind: VirtualServerList + plural: virtualservers + shortNames: + - vs + singular: virtualserver + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the VirtualServer. If the resource has a valid status, it means it has been validated and accepted by the Ingress Controller. + jsonPath: .status.state + name: State + type: string + - jsonPath: .spec.host + name: Host + type: string + - jsonPath: .status.externalEndpoints[*].ip + name: IP + type: string + - jsonPath: .status.externalEndpoints[*].ports + name: Ports + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VirtualServer defines the VirtualServer resource. + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VirtualServerSpec is the spec of the VirtualServer resource. + type: object + properties: + host: + type: string + http-snippets: + type: string + ingressClassName: + type: string + policies: + type: array + items: + description: PolicyReference references a policy by name and an optional namespace. + type: object + properties: + name: + type: string + namespace: + type: string + routes: + type: array + items: + description: Route defines a route. + type: object + properties: + action: + description: Action defines an action. + type: object + properties: + pass: + type: string + proxy: + description: ActionProxy defines a proxy in an Action. + type: object + properties: + requestHeaders: + description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy. + type: object + properties: + pass: + type: boolean + set: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + responseHeaders: + description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy. + type: object + properties: + add: + type: array + items: + description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive. + type: object + properties: + always: + type: boolean + name: + type: string + value: + type: string + hide: + type: array + items: + type: string + ignore: + type: array + items: + type: string + pass: + type: array + items: + type: string + rewritePath: + type: string + upstream: + type: string + redirect: + description: ActionRedirect defines a redirect in an Action. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ActionReturn defines a return in an Action. + type: object + properties: + body: + type: string + code: + type: integer + type: + type: string + errorPages: + type: array + items: + description: ErrorPage defines an ErrorPage in a Route. + type: object + properties: + codes: + type: array + items: + type: integer + redirect: + description: ErrorPageRedirect defines a redirect for an ErrorPage. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ErrorPageReturn defines a return for an ErrorPage. + type: object + properties: + body: + type: string + code: + type: integer + headers: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + type: + type: string + location-snippets: + type: string + matches: + type: array + items: + description: Match defines a match. + type: object + properties: + action: + description: Action defines an action. + type: object + properties: + pass: + type: string + proxy: + description: ActionProxy defines a proxy in an Action. + type: object + properties: + requestHeaders: + description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy. + type: object + properties: + pass: + type: boolean + set: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + responseHeaders: + description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy. + type: object + properties: + add: + type: array + items: + description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive. + type: object + properties: + always: + type: boolean + name: + type: string + value: + type: string + hide: + type: array + items: + type: string + ignore: + type: array + items: + type: string + pass: + type: array + items: + type: string + rewritePath: + type: string + upstream: + type: string + redirect: + description: ActionRedirect defines a redirect in an Action. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ActionReturn defines a return in an Action. + type: object + properties: + body: + type: string + code: + type: integer + type: + type: string + conditions: + type: array + items: + description: Condition defines a condition in a MatchRule. + type: object + properties: + argument: + type: string + cookie: + type: string + header: + type: string + value: + type: string + variable: + type: string + splits: + type: array + items: + description: Split defines a split. + type: object + properties: + action: + description: Action defines an action. + type: object + properties: + pass: + type: string + proxy: + description: ActionProxy defines a proxy in an Action. + type: object + properties: + requestHeaders: + description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy. + type: object + properties: + pass: + type: boolean + set: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + responseHeaders: + description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy. + type: object + properties: + add: + type: array + items: + description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive. + type: object + properties: + always: + type: boolean + name: + type: string + value: + type: string + hide: + type: array + items: + type: string + ignore: + type: array + items: + type: string + pass: + type: array + items: + type: string + rewritePath: + type: string + upstream: + type: string + redirect: + description: ActionRedirect defines a redirect in an Action. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ActionReturn defines a return in an Action. + type: object + properties: + body: + type: string + code: + type: integer + type: + type: string + weight: + type: integer + path: + type: string + policies: + type: array + items: + description: PolicyReference references a policy by name and an optional namespace. + type: object + properties: + name: + type: string + namespace: + type: string + route: + type: string + splits: + type: array + items: + description: Split defines a split. + type: object + properties: + action: + description: Action defines an action. + type: object + properties: + pass: + type: string + proxy: + description: ActionProxy defines a proxy in an Action. + type: object + properties: + requestHeaders: + description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy. + type: object + properties: + pass: + type: boolean + set: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + responseHeaders: + description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy. + type: object + properties: + add: + type: array + items: + description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive. + type: object + properties: + always: + type: boolean + name: + type: string + value: + type: string + hide: + type: array + items: + type: string + ignore: + type: array + items: + type: string + pass: + type: array + items: + type: string + rewritePath: + type: string + upstream: + type: string + redirect: + description: ActionRedirect defines a redirect in an Action. + type: object + properties: + code: + type: integer + url: + type: string + return: + description: ActionReturn defines a return in an Action. + type: object + properties: + body: + type: string + code: + type: integer + type: + type: string + weight: + type: integer + server-snippets: + type: string + tls: + description: TLS defines TLS configuration for a VirtualServer. + type: object + properties: + redirect: + description: TLSRedirect defines a redirect for a TLS. + type: object + properties: + basedOn: + type: string + code: + type: integer + enable: + type: boolean + secret: + type: string + upstreams: + type: array + items: + description: Upstream defines an upstream. + type: object + properties: + buffer-size: + type: string + buffering: + type: boolean + buffers: + description: UpstreamBuffers defines Buffer Configuration for an Upstream. + type: object + properties: + number: + type: integer + size: + type: string + client-max-body-size: + type: string + connect-timeout: + type: string + fail-timeout: + type: string + healthCheck: + description: HealthCheck defines the parameters for active Upstream HealthChecks. + type: object + properties: + connect-timeout: + type: string + enable: + type: boolean + fails: + type: integer + headers: + type: array + items: + description: Header defines an HTTP Header. + type: object + properties: + name: + type: string + value: + type: string + interval: + type: string + jitter: + type: string + passes: + type: integer + path: + type: string + port: + type: integer + read-timeout: + type: string + send-timeout: + type: string + statusMatch: + type: string + tls: + description: UpstreamTLS defines a TLS configuration for an Upstream. + type: object + properties: + enable: + type: boolean + keepalive: + type: integer + lb-method: + type: string + max-conns: + type: integer + max-fails: + type: integer + name: + type: string + next-upstream: + type: string + next-upstream-timeout: + type: string + next-upstream-tries: + type: integer + port: + type: integer + queue: + description: UpstreamQueue defines Queue Configuration for an Upstream. + type: object + properties: + size: + type: integer + timeout: + type: string + read-timeout: + type: string + send-timeout: + type: string + service: + type: string + sessionCookie: + description: SessionCookie defines the parameters for session persistence. + type: object + properties: + domain: + type: string + enable: + type: boolean + expires: + type: string + httpOnly: + type: boolean + name: + type: string + path: + type: string + secure: + type: boolean + slow-start: + type: string + subselector: + type: object + additionalProperties: + type: string + tls: + description: UpstreamTLS defines a TLS configuration for an Upstream. + type: object + properties: + enable: + type: boolean + use-cluster-ip: + type: boolean + status: + description: VirtualServerStatus defines the status for the VirtualServer resource. + type: object + properties: + externalEndpoints: + type: array + items: + description: ExternalEndpoint defines the IP and ports used to connect to this resource. + type: object + properties: + ip: + type: string + ports: + type: string + message: + type: string + reason: + type: string + state: + type: string + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubernetes/helm_charts/nginx-ingress/templates/NOTES.txt b/kubernetes/helm_charts/nginx-ingress/templates/NOTES.txt new file mode 100644 index 0000000000..c61fbd1a23 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/NOTES.txt @@ -0,0 +1 @@ +The NGINX Ingress Controller has been installed. \ No newline at end of file diff --git a/kubernetes/helm_charts/nginx-ingress/templates/_helpers.tpl b/kubernetes/helm_charts/nginx-ingress/templates/_helpers.tpl new file mode 100644 index 0000000000..9a73a5656c --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/_helpers.tpl @@ -0,0 +1,71 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "nginx-ingress.name" -}} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create labels +*/}} +{{- define "nginx-ingress.labels" -}} +app.kubernetes.io/name: {{ include "nginx-ingress.name" . }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Expand the name of the configmap. +*/}} +{{- define "nginx-ingress.configName" -}} +{{- default (include "nginx-ingress.name" .) .Values.controller.config.name -}} +{{- end -}} + +{{/* +Expand leader election lock name. +*/}} +{{- define "nginx-ingress.leaderElectionName" -}} +{{- if .Values.controller.reportIngressStatus.leaderElectionLockName -}} +{{ .Values.controller.reportIngressStatus.leaderElectionLockName }} +{{- else -}} +{{- printf "%s-%s" (include "nginx-ingress.name" .) "leader-election" -}} +{{- end -}} +{{- end -}} + +{{/* +Expand service account name. +*/}} +{{- define "nginx-ingress.serviceAccountName" -}} +{{- default (include "nginx-ingress.name" .) .Values.controller.serviceAccount.name -}} +{{- end -}} + +{{/* +Expand service name. +*/}} +{{- define "nginx-ingress.serviceName" -}} +{{- default (include "nginx-ingress.name" .) .Values.controller.service.name }} +{{- end -}} + +{{/* +Expand default TLS name. +*/}} +{{- define "nginx-ingress.defaultTLSName" -}} +{{- printf "%s-%s" (include "nginx-ingress.name" .) "default-server-tls" -}} +{{- end -}} + +{{/* +Expand wildcard TLS name. +*/}} +{{- define "nginx-ingress.wildcardTLSName" -}} +{{- printf "%s-%s" (include "nginx-ingress.name" .) "wildcard-tls" -}} +{{- end -}} + +{{/* +Expand app name. +*/}} +{{- define "nginx-ingress.appName" -}} +{{- default (include "nginx-ingress.name" .) .Values.controller.name -}} +{{- end -}} \ No newline at end of file diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-configmap.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-configmap.yaml new file mode 100644 index 0000000000..971faa84d0 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nginx-ingress.configName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +{{- if .Values.controller.config.annotations }} + annotations: +{{ toYaml .Values.controller.config.annotations | indent 4 }} +{{- end }} +data: +{{- if .Values.controller.config.entries }} +{{ toYaml .Values.controller.config.entries | indent 2 }} +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-daemonset.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-daemonset.yaml new file mode 100644 index 0000000000..e39ab74534 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-daemonset.yaml @@ -0,0 +1,159 @@ +{{- if eq .Values.controller.kind "daemonset" }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ default (include "nginx-ingress.name" .) .Values.controller.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ include "nginx-ingress.appName" . }} + template: + metadata: + labels: + app: {{ include "nginx-ingress.appName" . }} +{{- if or (.Values.prometheus.create) (.Values.controller.pod.annotations) }} + annotations: +{{- if .Values.prometheus.create }} + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.prometheus.port }}" + prometheus.io/scheme: "{{ .Values.prometheus.scheme }}" +{{- end }} +{{- if .Values.controller.pod.annotations }} +{{ toYaml .Values.controller.pod.annotations | indent 8 }} +{{- end }} +{{- end }} + spec: + serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} +{{- if .Values.controller.nodeSelector }} + nodeSelector: +{{ toYaml .Values.controller.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.controller.tolerations }} + tolerations: +{{ toYaml .Values.controller.tolerations | indent 6 }} +{{- end }} +{{- if .Values.controller.affinity }} + affinity: +{{ toYaml .Values.controller.affinity | indent 8 }} +{{- end }} +{{- if .Values.controller.volumes }} + volumes: +{{ toYaml .Values.controller.volumes | indent 6 }} +{{- end }} +{{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName }} +{{- end }} + hostNetwork: {{ .Values.controller.hostNetwork }} + containers: + - name: {{ include "nginx-ingress.name" . }} + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" + imagePullPolicy: "{{ .Values.controller.image.pullPolicy }}" + ports: + - name: http + containerPort: 80 + hostPort: 80 + - name: https + containerPort: 443 + hostPort: 443 +{{ if .Values.controller.customPorts }} +{{ toYaml .Values.controller.customPorts | indent 8 }} +{{ end }} +{{- if .Values.prometheus.create }} + - name: prometheus + containerPort: {{ .Values.prometheus.port }} +{{- end }} +{{- if .Values.controller.readyStatus.enable }} + - name: readiness-port + containerPort: {{ .Values.controller.readyStatus.port}} + readinessProbe: + httpGet: + path: /nginx-ready + port: readiness-port + periodSeconds: 1 +{{- end }} + securityContext: + allowPrivilegeEscalation: true + runAsUser: 101 #nginx + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE +{{- if .Values.controller.volumeMounts }} + volumeMounts: +{{ toYaml .Values.controller.volumeMounts | indent 8 }} +{{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + resources: +{{ toYaml .Values.controller.resources | indent 10 }} + args: + - -nginx-plus={{ .Values.controller.nginxplus }} + - -nginx-reload-timeout={{ .Values.controller.nginxReloadTimeout }} + - -enable-app-protect={{ .Values.controller.appprotect.enable }} + - -nginx-configmaps=$(POD_NAMESPACE)/{{ include "nginx-ingress.configName" . }} +{{- if .Values.controller.defaultTLS.secret }} + - -default-server-tls-secret={{ .Values.controller.defaultTLS.secret }} +{{ else }} + - -default-server-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.defaultTLSName" . }} +{{- end }} + - -ingress-class={{ .Values.controller.ingressClass }} +{{- if semverCompare "<1.18.0" .Capabilities.KubeVersion.GitVersion }} + - -use-ingress-class-only={{ .Values.controller.useIngressClassOnly }} +{{- end }} +{{- if .Values.controller.watchNamespace }} + - -watch-namespace={{ .Values.controller.watchNamespace }} +{{- end }} + - -health-status={{ .Values.controller.healthStatus }} + - -health-status-uri={{ .Values.controller.healthStatusURI }} + - -nginx-debug={{ .Values.controller.nginxDebug }} + - -v={{ .Values.controller.logLevel }} + - -nginx-status={{ .Values.controller.nginxStatus.enable }} +{{- if .Values.controller.nginxStatus.enable }} + - -nginx-status-port={{ .Values.controller.nginxStatus.port }} + - -nginx-status-allow-cidrs={{ .Values.controller.nginxStatus.allowCidrs }} +{{- end }} +{{- if .Values.controller.reportIngressStatus.enable }} + - -report-ingress-status +{{- if .Values.controller.reportIngressStatus.ingressLink }} + - -ingresslink={{ .Values.controller.reportIngressStatus.ingressLink }} +{{- else if .Values.controller.reportIngressStatus.externalService }} + - -external-service={{ .Values.controller.reportIngressStatus.externalService }} +{{- else if and (.Values.controller.service.create) (eq .Values.controller.service.type "LoadBalancer") }} + - -external-service={{ include "nginx-ingress.serviceName" . }} +{{- end }} + - -enable-leader-election={{ .Values.controller.reportIngressStatus.enableLeaderElection }} + - -leader-election-lock-name={{ include "nginx-ingress.leaderElectionName" . }} +{{- end }} +{{- if .Values.controller.wildcardTLS.secret }} + - -wildcard-tls-secret={{ .Values.controller.wildcardTLS.secret }} +{{- else if and .Values.controller.wildcardTLS.cert .Values.controller.wildcardTLS.key }} + - -wildcard-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.wildcardTLSName" . }} +{{- end }} + - -enable-prometheus-metrics={{ .Values.prometheus.create }} + - -prometheus-metrics-listen-port={{ .Values.prometheus.port }} + - -prometheus-tls-secret={{ .Values.prometheus.secret }} + - -enable-custom-resources={{ .Values.controller.enableCustomResources }} +{{- if .Values.controller.enableCustomResources }} + - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }} + - -enable-snippets={{ .Values.controller.enableSnippets }} + - -enable-preview-policies={{ .Values.controller.enablePreviewPolicies }} +{{- if .Values.controller.globalConfiguration.create }} + - -global-configuration=$(POD_NAMESPACE)/{{ include "nginx-ingress.name" . }} +{{- end }} +{{- end }} + - -ready-status={{ .Values.controller.readyStatus.enable }} + - -ready-status-port={{ .Values.controller.readyStatus.port }} + - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }} +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-deployment.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-deployment.yaml new file mode 100644 index 0000000000..4b6cd91746 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-deployment.yaml @@ -0,0 +1,157 @@ +{{- if eq .Values.controller.kind "deployment" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ default (include "nginx-ingress.name" .) .Values.controller.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controller.replicaCount }} + selector: + matchLabels: + app: {{ include "nginx-ingress.appName" . }} + template: + metadata: + labels: + app: {{ include "nginx-ingress.appName" . }} +{{- if or (.Values.prometheus.create) (.Values.controller.pod.annotations) }} + annotations: +{{- if .Values.prometheus.create }} + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.prometheus.port }}" + prometheus.io/scheme: "{{ .Values.prometheus.scheme }}" +{{- end }} +{{- if .Values.controller.pod.annotations }} +{{ toYaml .Values.controller.pod.annotations | indent 8 }} +{{- end }} +{{- end }} + spec: +{{- if .Values.controller.nodeSelector }} + nodeSelector: +{{ toYaml .Values.controller.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.controller.tolerations }} + tolerations: +{{ toYaml .Values.controller.tolerations | indent 6 }} +{{- end }} +{{- if .Values.controller.affinity }} + affinity: +{{ toYaml .Values.controller.affinity | indent 8 }} +{{- end }} +{{- if .Values.controller.volumes }} + volumes: +{{ toYaml .Values.controller.volumes | indent 6 }} +{{- end }} +{{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName }} +{{- end }} + serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }} + hostNetwork: {{ .Values.controller.hostNetwork }} + containers: + - image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" + name: {{ include "nginx-ingress.name" . }} + imagePullPolicy: "{{ .Values.controller.image.pullPolicy }}" + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 +{{ if .Values.controller.customPorts }} +{{ toYaml .Values.controller.customPorts | indent 8 }} +{{ end }} +{{- if .Values.prometheus.create }} + - name: prometheus + containerPort: {{ .Values.prometheus.port }} +{{- end }} +{{- if .Values.controller.readyStatus.enable }} + - name: readiness-port + containerPort: {{ .Values.controller.readyStatus.port}} + readinessProbe: + httpGet: + path: /nginx-ready + port: readiness-port + periodSeconds: 1 +{{- end }} + resources: +{{ toYaml .Values.controller.resources | indent 10 }} + securityContext: + allowPrivilegeEscalation: true + runAsUser: 101 #nginx + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE +{{- if .Values.controller.volumeMounts }} + volumeMounts: +{{ toYaml .Values.controller.volumeMounts | indent 8 }} +{{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + args: + - -nginx-plus={{ .Values.controller.nginxplus }} + - -nginx-reload-timeout={{ .Values.controller.nginxReloadTimeout }} + - -enable-app-protect={{ .Values.controller.appprotect.enable }} + - -nginx-configmaps=$(POD_NAMESPACE)/{{ include "nginx-ingress.configName" . }} +{{- if .Values.controller.defaultTLS.secret }} + - -default-server-tls-secret={{ .Values.controller.defaultTLS.secret }} +{{ else }} + - -default-server-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.defaultTLSName" . }} +{{- end }} + - -ingress-class={{ .Values.controller.ingressClass }} +{{- if semverCompare "<1.18.0" .Capabilities.KubeVersion.GitVersion }} + - -use-ingress-class-only={{ .Values.controller.useIngressClassOnly }} +{{- end }} +{{- if .Values.controller.watchNamespace }} + - -watch-namespace={{ .Values.controller.watchNamespace }} +{{- end }} + - -health-status={{ .Values.controller.healthStatus }} + - -health-status-uri={{ .Values.controller.healthStatusURI }} + - -nginx-debug={{ .Values.controller.nginxDebug }} + - -v={{ .Values.controller.logLevel }} + - -nginx-status={{ .Values.controller.nginxStatus.enable }} +{{- if .Values.controller.nginxStatus.enable }} + - -nginx-status-port={{ .Values.controller.nginxStatus.port }} + - -nginx-status-allow-cidrs={{ .Values.controller.nginxStatus.allowCidrs }} +{{- end }} +{{- if .Values.controller.reportIngressStatus.enable }} + - -report-ingress-status +{{- if .Values.controller.reportIngressStatus.ingressLink }} + - -ingresslink={{ .Values.controller.reportIngressStatus.ingressLink }} +{{- else if .Values.controller.reportIngressStatus.externalService }} + - -external-service={{ .Values.controller.reportIngressStatus.externalService }} +{{- else if and (.Values.controller.service.create) (eq .Values.controller.service.type "LoadBalancer") }} + - -external-service={{ include "nginx-ingress.serviceName" . }} +{{- end }} + - -enable-leader-election={{ .Values.controller.reportIngressStatus.enableLeaderElection }} + - -leader-election-lock-name={{ include "nginx-ingress.leaderElectionName" . }} +{{- end }} +{{- if .Values.controller.wildcardTLS.secret }} + - -wildcard-tls-secret={{ .Values.controller.wildcardTLS.secret }} +{{- else if and .Values.controller.wildcardTLS.cert .Values.controller.wildcardTLS.key }} + - -wildcard-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.wildcardTLSName" . }} +{{- end }} + - -enable-prometheus-metrics={{ .Values.prometheus.create }} + - -prometheus-metrics-listen-port={{ .Values.prometheus.port }} + - -prometheus-tls-secret={{ .Values.prometheus.secret }} + - -enable-custom-resources={{ .Values.controller.enableCustomResources }} +{{- if .Values.controller.enableCustomResources }} + - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }} + - -enable-snippets={{ .Values.controller.enableSnippets }} + - -enable-preview-policies={{ .Values.controller.enablePreviewPolicies }} +{{- if .Values.controller.globalConfiguration.create }} + - -global-configuration=$(POD_NAMESPACE)/{{ include "nginx-ingress.name" . }} +{{- end }} +{{- end }} + - -ready-status={{ .Values.controller.readyStatus.enable }} + - -ready-status-port={{ .Values.controller.readyStatus.port }} + - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }} +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-globalconfiguration.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-globalconfiguration.yaml new file mode 100644 index 0000000000..b0bba48704 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-globalconfiguration.yaml @@ -0,0 +1,11 @@ +{{ if .Values.controller.globalConfiguration.create }} +apiVersion: k8s.nginx.org/v1alpha1 +kind: GlobalConfiguration +metadata: + name: {{ include "nginx-ingress.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +spec: +{{ toYaml .Values.controller.globalConfiguration.spec | indent 2 }} +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-ingress-class.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-ingress-class.yaml new file mode 100644 index 0000000000..04df66a30f --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-ingress-class.yaml @@ -0,0 +1,12 @@ +{{- if semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.GitVersion }} +apiVersion: networking.k8s.io/v1beta1 +kind: IngressClass +metadata: + name: {{ .Values.controller.ingressClass }} +{{- if .Values.controller.setAsDefaultIngress }} + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +{{- end }} +spec: + controller: nginx.org/ingress-controller +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-leader-election-configmap.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-leader-election-configmap.yaml new file mode 100644 index 0000000000..ef6f79a581 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-leader-election-configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nginx-ingress.leaderElectionName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +{{- if .Values.controller.reportIngressStatus.annotations }} + annotations: +{{ toYaml .Values.controller.reportIngressStatus.annotations | indent 4 }} +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-secret.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-secret.yaml new file mode 100644 index 0000000000..4c9e693bb5 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-secret.yaml @@ -0,0 +1,13 @@ +{{ if not .Values.controller.defaultTLS.secret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "nginx-ingress.defaultTLSName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +type: kubernetes.io/tls +data: + tls.crt: {{ .Values.controller.defaultTLS.cert }} + tls.key: {{ .Values.controller.defaultTLS.key }} +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-service.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-service.yaml new file mode 100644 index 0000000000..11d542271f --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-service.yaml @@ -0,0 +1,60 @@ +{{- if .Values.controller.service.create }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nginx-ingress.serviceName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +{{- if .Values.controller.service.extraLabels }} +{{ toYaml .Values.controller.service.extraLabels | indent 4 }} +{{- end }} +{{- if .Values.controller.service.annotations }} + annotations: +{{ toYaml .Values.controller.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if or (eq .Values.controller.service.type "LoadBalancer") (eq .Values.controller.service.type "NodePort") }} + {{- if .Values.controller.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.controller.service.externalTrafficPolicy }} + {{- end }} +{{- end }} +{{- if eq .Values.controller.service.type "LoadBalancer" }} + {{- if .Values.controller.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.controller.service.loadBalancerIP }} + {{- end }} + {{- if .Values.controller.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.controller.service.loadBalancerSourceRanges | indent 4 }} + {{- end }} +{{- end }} + type: {{ .Values.controller.service.type }} + ports: +{{- if .Values.controller.service.customPorts }} +{{ toYaml .Values.controller.service.customPorts | indent 2 }} +{{ end }} +{{- if .Values.controller.service.httpPort.enable }} + - port: {{ .Values.controller.service.httpPort.port }} + targetPort: {{ .Values.controller.service.httpPort.targetPort }} + protocol: TCP + name: http + {{- if eq .Values.controller.service.type "NodePort" }} + nodePort: {{ .Values.controller.service.httpPort.nodePort }} + {{- end }} +{{- end }} +{{- if .Values.controller.service.httpsPort.enable }} + - port: {{ .Values.controller.service.httpsPort.port }} + targetPort: {{ .Values.controller.service.httpsPort.targetPort }} + protocol: TCP + name: https + {{- if eq .Values.controller.service.type "NodePort" }} + nodePort: {{ .Values.controller.service.httpsPort.nodePort }} + {{- end }} +{{- end }} + selector: + app: {{ include "nginx-ingress.appName" . }} + {{- if .Values.controller.service.externalIPs }} + externalIPs: +{{ toYaml .Values.controller.service.externalIPs | indent 4 }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-serviceaccount.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-serviceaccount.yaml new file mode 100644 index 0000000000..c03c6a793b --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.rbac.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nginx-ingress.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +{{- if .Values.controller.serviceAccount.imagePullSecretName }} +imagePullSecrets: +- name: {{ .Values.controller.serviceAccount.imagePullSecretName }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-wildcard-secret.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-wildcard-secret.yaml new file mode 100644 index 0000000000..3abe16c4c3 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-wildcard-secret.yaml @@ -0,0 +1,13 @@ +{{ if and (not .Values.controller.wildcardTLS.secret) (and .Values.controller.wildcardTLS.cert .Values.controller.wildcardTLS.key) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "nginx-ingress.wildcardTLSName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +type: kubernetes.io/tls +data: + tls.crt: {{ .Values.controller.wildcardTLS.cert }} + tls.key: {{ .Values.controller.wildcardTLS.key }} +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/templates/rbac.yaml b/kubernetes/helm_charts/nginx-ingress/templates/rbac.yaml new file mode 100644 index 0000000000..7538b4830d --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/templates/rbac.yaml @@ -0,0 +1,135 @@ +{{- if .Values.rbac.create }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "nginx-ingress.name" . }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +rules: +{{- if .Values.controller.appprotect.enable }} +- apiGroups: + - appprotect.f5.com + resources: + - appolicies + - aplogconfs + - apusersigs + verbs: + - get + - watch + - list +{{- end }} +- apiGroups: + - "" + resources: + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +{{- if .Values.controller.reportIngressStatus.enableLeaderElection }} + - update + - create +{{- end }} +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - list +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get +{{- if .Values.controller.reportIngressStatus.enable }} +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +{{- end }} +{{- if .Values.controller.enableCustomResources }} +- apiGroups: + - k8s.nginx.org + resources: + - virtualservers + - virtualserverroutes + - globalconfigurations + - transportservers + - policies + verbs: + - list + - watch + - get +- apiGroups: + - k8s.nginx.org + resources: + - virtualservers/status + - virtualserverroutes/status + - policies/status + - transportservers/status + verbs: + - update +{{- end }} +{{- if .Values.controller.reportIngressStatus.ingressLink }} +- apiGroups: + - cis.f5.com + resources: + - ingresslinks + verbs: + - list + - watch + - get +{{- end }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "nginx-ingress.name" . }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +subjects: +- kind: ServiceAccount + name: {{ include "nginx-ingress.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "nginx-ingress.name" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/kubernetes/helm_charts/nginx-ingress/values-icp.yaml b/kubernetes/helm_charts/nginx-ingress/values-icp.yaml new file mode 100644 index 0000000000..6b74235fa1 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/values-icp.yaml @@ -0,0 +1,16 @@ +controller: + kind: daemonset + nginxplus: true + image: + repository: mycluster.icp:8500/kube-system/nginx-plus-ingress + tag: "1.12.0" + nodeSelector: + beta.kubernetes.io/arch: "amd64" + proxy: true + terminationGracePeriodSeconds: 60 + tolerations: + - key: "dedicated" + operator: "Exists" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" diff --git a/kubernetes/helm_charts/nginx-ingress/values-plus.yaml b/kubernetes/helm_charts/nginx-ingress/values-plus.yaml new file mode 100644 index 0000000000..24ab2844e2 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/values-plus.yaml @@ -0,0 +1,5 @@ +controller: + nginxplus: true + image: + repository: nginx-plus-ingress + tag: "1.12.0" diff --git a/kubernetes/helm_charts/nginx-ingress/values.yaml b/kubernetes/helm_charts/nginx-ingress/values.yaml new file mode 100644 index 0000000000..1316234c01 --- /dev/null +++ b/kubernetes/helm_charts/nginx-ingress/values.yaml @@ -0,0 +1,318 @@ +controller: + ## The name of the Ingress controller daemonset or deployment. + ## Autogenerated if not set or set to "". + # name: nginx-ingress + + ## The kind of the Ingress controller installation - deployment or daemonset. + kind: deployment + + ## Deploys the Ingress controller for NGINX Plus. + nginxplus: false + + # Timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. + # Default is 4000 (default is 20000 instead if enable-app-protect is true) + # If set to 0, default values will be used. + nginxReloadTimeout: 0 + + ## Support for App Protect + appprotect: + ## Enable the App Protect module in the Ingress Controller. + enable: false + + ## Enables the Ingress controller pods to use the host's network namespace. + hostNetwork: false + + ## Enables debugging for NGINX. Uses the nginx-debug binary. Requires error-log-level: debug in the ConfigMap via `controller.config.entries`. + nginxDebug: false + + ## The log level of the Ingress Controller. + logLevel: 1 + + ## A list of custom ports to expose on the NGINX ingress controller pod. Follows the conventional Kubernetes yaml syntax for container ports. + customPorts: [] + + image: + ## The image repository of the Ingress controller. + repository: nginx/nginx-ingress + + ## The tag of the Ingress controller image. + tag: "1.12.0" + + ## The pull policy for the Ingress controller image. + pullPolicy: IfNotPresent + + config: + ## The name of the ConfigMap used by the Ingress controller. + ## Autogenerated if not set or set to "". + # name: nginx-config + + ## The annotations of the Ingress Controller configmap. + annotations: {} + + ## The entries of the ConfigMap for customizing NGINX configuration. + entries: {} + + ## It is recommended to use your own TLS certificates and keys + defaultTLS: + ## The base64-encoded TLS certificate for the default HTTPS server. If not specified, a pre-generated self-signed certificate is used. + ## Note: It is recommended that you specify your own certificate. + cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN2akNDQWFZQ0NRREFPRjl0THNhWFhEQU5CZ2txaGtpRzl3MEJBUXNGQURBaE1SOHdIUVlEVlFRRERCWk8KUjBsT1dFbHVaM0psYzNORGIyNTBjbTlzYkdWeU1CNFhEVEU0TURreE1qRTRNRE16TlZvWERUSXpNRGt4TVRFNApNRE16TlZvd0lURWZNQjBHQTFVRUF3d1dUa2RKVGxoSmJtZHlaWE56UTI5dWRISnZiR3hsY2pDQ0FTSXdEUVlKCktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwvN2hIUEtFWGRMdjNyaUM3QlBrMTNpWkt5eTlyQ08KR2xZUXYyK2EzUDF0azIrS3YwVGF5aGRCbDRrcnNUcTZzZm8vWUk1Y2Vhbkw4WGM3U1pyQkVRYm9EN2REbWs1Qgo4eDZLS2xHWU5IWlg0Rm5UZ0VPaStlM2ptTFFxRlBSY1kzVnNPazFFeUZBL0JnWlJVbkNHZUtGeERSN0tQdGhyCmtqSXVuektURXUyaDU4Tlp0S21ScUJHdDEwcTNRYzhZT3ExM2FnbmovUWRjc0ZYYTJnMjB1K1lYZDdoZ3krZksKWk4vVUkxQUQ0YzZyM1lma1ZWUmVHd1lxQVp1WXN2V0RKbW1GNWRwdEMzN011cDBPRUxVTExSakZJOTZXNXIwSAo1TmdPc25NWFJNV1hYVlpiNWRxT3R0SmRtS3FhZ25TZ1JQQVpQN2MwQjFQU2FqYzZjNGZRVXpNQ0F3RUFBVEFOCkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQWpLb2tRdGRPcEsrTzhibWVPc3lySmdJSXJycVFVY2ZOUitjb0hZVUoKdGhrYnhITFMzR3VBTWI5dm15VExPY2xxeC9aYzJPblEwMEJCLzlTb0swcitFZ1U2UlVrRWtWcitTTFA3NTdUWgozZWI4dmdPdEduMS9ienM3bzNBaS9kclkrcUI5Q2k1S3lPc3FHTG1US2xFaUtOYkcyR1ZyTWxjS0ZYQU80YTY3Cklnc1hzYktNbTQwV1U3cG9mcGltU1ZmaXFSdkV5YmN3N0NYODF6cFErUyt1eHRYK2VBZ3V0NHh3VlI5d2IyVXYKelhuZk9HbWhWNThDd1dIQnNKa0kxNXhaa2VUWXdSN0diaEFMSkZUUkk3dkhvQXprTWIzbjAxQjQyWjNrN3RXNQpJUDFmTlpIOFUvOWxiUHNoT21FRFZkdjF5ZytVRVJxbStGSis2R0oxeFJGcGZnPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + + ## The base64-encoded TLS key for the default HTTPS server. Note: If not specified, a pre-generated key is used. + ## Note: It is recommended that you specify your own key. + key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBdi91RWM4b1JkMHUvZXVJTHNFK1RYZUprckxMMnNJNGFWaEMvYjVyYy9XMlRiNHEvClJOcktGMEdYaVN1eE9ycXgrajlnamx4NXFjdnhkenRKbXNFUkJ1Z1B0ME9hVGtIekhvb3FVWmcwZGxmZ1dkT0EKUTZMNTdlT1l0Q29VOUZ4amRXdzZUVVRJVUQ4R0JsRlNjSVo0b1hFTkhzbysyR3VTTWk2Zk1wTVM3YUhudzFtMApxWkdvRWEzWFNyZEJ6eGc2clhkcUNlUDlCMXl3VmRyYURiUzc1aGQzdUdETDU4cGszOVFqVUFQaHpxdmRoK1JWClZGNGJCaW9CbTVpeTlZTW1hWVhsMm0wTGZzeTZuUTRRdFFzdEdNVWozcGJtdlFmazJBNnljeGRFeFpkZFZsdmwKMm82MjBsMllxcHFDZEtCRThCay90elFIVTlKcU56cHpoOUJUTXdJREFRQUJBb0lCQVFDZklHbXowOHhRVmorNwpLZnZJUXQwQ0YzR2MxNld6eDhVNml4MHg4Mm15d1kxUUNlL3BzWE9LZlRxT1h1SENyUlp5TnUvZ2IvUUQ4bUFOCmxOMjRZTWl0TWRJODg5TEZoTkp3QU5OODJDeTczckM5bzVvUDlkazAvYzRIbjAzSkVYNzZ5QjgzQm9rR1FvYksKMjhMNk0rdHUzUmFqNjd6Vmc2d2szaEhrU0pXSzBwV1YrSjdrUkRWYmhDYUZhNk5nMUZNRWxhTlozVDhhUUtyQgpDUDNDeEFTdjYxWTk5TEI4KzNXWVFIK3NYaTVGM01pYVNBZ1BkQUk3WEh1dXFET1lvMU5PL0JoSGt1aVg2QnRtCnorNTZud2pZMy8yUytSRmNBc3JMTnIwMDJZZi9oY0IraVlDNzVWYmcydVd6WTY3TWdOTGQ5VW9RU3BDRkYrVm4KM0cyUnhybnhBb0dCQU40U3M0ZVlPU2huMVpQQjdhTUZsY0k2RHR2S2ErTGZTTXFyY2pOZjJlSEpZNnhubmxKdgpGenpGL2RiVWVTbWxSekR0WkdlcXZXaHFISy9iTjIyeWJhOU1WMDlRQ0JFTk5jNmtWajJTVHpUWkJVbEx4QzYrCk93Z0wyZHhKendWelU0VC84ajdHalRUN05BZVpFS2FvRHFyRG5BYWkyaW5oZU1JVWZHRXFGKzJyQW9HQkFOMVAKK0tZL0lsS3RWRzRKSklQNzBjUis3RmpyeXJpY05iWCtQVzUvOXFHaWxnY2grZ3l4b25BWlBpd2NpeDN3QVpGdwpaZC96ZFB2aTBkWEppc1BSZjRMazg5b2pCUmpiRmRmc2l5UmJYbyt3TFU4NUhRU2NGMnN5aUFPaTVBRHdVU0FkCm45YWFweUNweEFkREtERHdObit3ZFhtaTZ0OHRpSFRkK3RoVDhkaVpBb0dCQUt6Wis1bG9OOTBtYlF4VVh5YUwKMjFSUm9tMGJjcndsTmVCaWNFSmlzaEhYa2xpSVVxZ3hSZklNM2hhUVRUcklKZENFaHFsV01aV0xPb2I2NTNyZgo3aFlMSXM1ZUtka3o0aFRVdnpldm9TMHVXcm9CV2xOVHlGanIrSWhKZnZUc0hpOGdsU3FkbXgySkJhZUFVWUNXCndNdlQ4NmNLclNyNkQrZG8wS05FZzFsL0FvR0FlMkFVdHVFbFNqLzBmRzgrV3hHc1RFV1JqclRNUzRSUjhRWXQKeXdjdFA4aDZxTGxKUTRCWGxQU05rMXZLTmtOUkxIb2pZT2pCQTViYjhibXNVU1BlV09NNENoaFJ4QnlHbmR2eAphYkJDRkFwY0IvbEg4d1R0alVZYlN5T294ZGt5OEp0ek90ajJhS0FiZHd6NlArWDZDODhjZmxYVFo5MWpYL3RMCjF3TmRKS2tDZ1lCbyt0UzB5TzJ2SWFmK2UwSkN5TGhzVDQ5cTN3Zis2QWVqWGx2WDJ1VnRYejN5QTZnbXo5aCsKcDNlK2JMRUxwb3B0WFhNdUFRR0xhUkcrYlNNcjR5dERYbE5ZSndUeThXczNKY3dlSTdqZVp2b0ZpbmNvVlVIMwphdmxoTUVCRGYxSjltSDB5cDBwWUNaS2ROdHNvZEZtQktzVEtQMjJhTmtsVVhCS3gyZzR6cFE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= + + ## The secret with a TLS certificate and key for the default HTTPS server. + ## The value must follow the following format: `/`. + ## Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters. + ## Format: / + secret: + + wildcardTLS: + ## The base64-encoded TLS certificate for every Ingress host that has TLS enabled but no secret specified. + ## If the parameter is not set, for such Ingress hosts NGINX will break any attempt to establish a TLS connection. + cert: "" + + ## The base64-encoded TLS key for every Ingress host that has TLS enabled but no secret specified. + ## If the parameter is not set, for such Ingress hosts NGINX will break any attempt to establish a TLS connection. + key: "" + + ## The secret with a TLS certificate and key for every Ingress host that has TLS enabled but no secret specified. + ## The value must follow the following format: `/`. + ## Used as an alternative to specifying a certificate and key using `controller.wildcardTLS.cert` and `controller.wildcardTLS.key` parameters. + ## Format: / + secret: + + ## The node selector for pod assignment for the Ingress controller pods. + nodeSelector: {} + + ## The termination grace period of the Ingress controller pod. + terminationGracePeriodSeconds: 30 + + ## The resources of the Ingress controller pods. + resources: {} + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 100m + # memory: 64Mi + + ## The tolerations of the Ingress controller pods. + tolerations: [] + + ## The affinity of the Ingress controller pods. + affinity: {} + + ## The volumes of the Ingress controller pods. + volumes: [] + # - name: extra-conf + # configMap: + # name: extra-conf + + ## The volumeMounts of the Ingress controller pods. + volumeMounts: [] + # - name: extra-conf + # mountPath: /etc/nginx/conf.d/extra.conf + # subPath: extra.conf + + ## The number of replicas of the Ingress controller deployment. + replicaCount: 1 + + ## A class of the Ingress controller. + + ## For Kubernetes >= 1.18, a corresponding IngressClass resource with the name equal to the class must be deployed. Otherwise, + ## the Ingress Controller will fail to start. + ## The Ingress controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. + + ## For Kubernetes < 1.18, the Ingress Controller only processes resources that belong to its class - + ## i.e have the annotation "kubernetes.io/ingress.class" (for Ingress resources) + ## or field "ingressClassName" (for VirtualServer/VirtualServerRoute/TransportServer resources) equal to the class. + ## Additionally, the Ingress Controller processes resources that do not have the class set, + ## which can be disabled by setting the controller.useIngressClassOnly parameter to true. + + ## The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of kubernetes. + ingressClass: nginx + + ## For kubernetes versions >= 1.18 this flag will be IGNORED. + ## Ignore Ingress resources without the "kubernetes.io/ingress.class" annotation + useIngressClassOnly: false + + ## Only for Kubernetes >= 1.18 + ## New Ingresses without an ingressClassName field specified will be assigned the class specified in `controller.ingressClass`. + setAsDefaultIngress: false + + ## Namespace to watch for Ingress resources. By default the Ingress controller watches all namespaces. + watchNamespace: "" + + ## Enable the custom resources. + enableCustomResources: false + + ## Enable preview policies. + enablePreviewPolicies: false + + ## Enable TLS Passthrough on port 443. Requires controller.enableCustomResources. + enableTLSPassthrough: false + + globalConfiguration: + ## Creates the GlobalConfiguration custom resource. Requires controller.enableCustomResources. + create: false + + ## The spec of the GlobalConfiguration for defining the global configuration parameters of the Ingress Controller. + spec: {} + # listeners: + # - name: dns-udp + # port: 5353 + # protocol: UDP + # - name: dns-tcp + # port: 5353 + # protocol: TCP + + ## Enable custom NGINX configuration snippets in VirtualServer, VirtualServerRoute and TransportServer resources. + enableSnippets: false + + ## Add a location based on the value of health-status-uri to the default server. The location responds with the 200 status code for any request. + ## Useful for external health-checking of the Ingress controller. + healthStatus: false + + ## Sets the URI of health status location in the default server. Requires controller.healthStatus. + healthStatusURI: "/nginx-health" + + nginxStatus: + ## Enable the NGINX stub_status, or the NGINX Plus API. + enable: true + + ## Set the port where the NGINX stub_status or the NGINX Plus API is exposed. + port: 8080 + + ## Add IPv4 IP/CIDR blocks to the allow list for NGINX stub_status or the NGINX Plus API. Separate multiple IP/CIDR by commas. + allowCidrs: "127.0.0.1" + + service: + ## Creates a service to expose the Ingress controller pods. + create: true + + ## The type of service to create for the Ingress controller. + type: LoadBalancer + + ## The externalTrafficPolicy of the service. The value Local preserves the client source IP. + externalTrafficPolicy: Local + + ## The annotations of the Ingress controller service. + annotations: {} + + ## The extra labels of the service. + extraLabels: {} + + ## The static IP address for the load balancer. Requires controller.service.type set to LoadBalancer. The cloud provider must support this feature. + loadBalancerIP: "" + + ## The list of external IPs for the Ingress controller service. + externalIPs: [] + + ## The IP ranges (CIDR) that are allowed to access the load balancer. Requires controller.service.type set to LoadBalancer. The cloud provider must support this feature. + loadBalancerSourceRanges: [] + + ## The name of the service + ## Autogenerated if not set or set to "". + # name: nginx-ingress + + httpPort: + ## Enables the HTTP port for the Ingress controller service. + enable: true + + ## The HTTP port of the Ingress controller service. + port: 80 + + ## The custom NodePort for the HTTP port. Requires controller.service.type set to NodePort. + nodePort: "" + + ## The HTTP port on the POD where the Ingress controller service is running. + targetPort: 80 + + httpsPort: + ## Enables the HTTPS port for the Ingress controller service. + enable: true + + ## The HTTPS port of the Ingress controller service. + port: 443 + + ## The custom NodePort for the HTTPS port. Requires controller.service.type set to NodePort. + nodePort: "" + + ## The HTTPS port on the POD where the Ingress controller service is running. + targetPort: 443 + + ## A list of custom ports to expose through the Ingress controller service. Follows the conventional Kubernetes yaml syntax for service ports. + customPorts: [] + + serviceAccount: + ## The name of the service account of the Ingress controller pods. Used for RBAC. + ## Autogenerated if not set or set to "". + # name: nginx-ingress + + ## The name of the secret containing docker registry credentials. + ## Secret must exist in the same namespace as the helm release. + imagePullSecretName: "" + + reportIngressStatus: + ## Updates the address field in the status of Ingress resources with an external address of the Ingress controller. + ## You must also specify the source of the external address either through an external service via controller.reportIngressStatus.externalService, + ## controller.reportIngressStatus.ingressLink or the external-status-address entry in the ConfigMap via controller.config.entries. + ## Note: controller.config.entries.external-status-address takes precedence over the others. + enable: true + + ## Specifies the name of the service with the type LoadBalancer through which the Ingress controller is exposed externally. + ## The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. + ## controller.reportIngressStatus.enable must be set to true. + ## The default is autogenerated and matches the created service (see controller.service.create). + # externalService: nginx-ingress + + ## Specifies the name of the IngressLink resource, which exposes the Ingress Controller pods via a BIG-IP system. + ## The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. + ## controller.reportIngressStatus.enable must be set to true. + ingressLink: "" + + ## Enable Leader election to avoid multiple replicas of the controller reporting the status of Ingress resources. controller.reportIngressStatus.enable must be set to true. + enableLeaderElection: true + + ## Specifies the name of the ConfigMap, within the same namespace as the controller, used as the lock for leader election. controller.reportIngressStatus.enableLeaderElection must be set to true. + ## Autogenerated if not set or set to "". + # leaderElectionLockName: "nginx-ingress-leader-election" + + ## The annotations of the leader election configmap. + annotations: {} + + pod: + ## The annotations of the Ingress Controller pod. + annotations: {} + + ## The PriorityClass of the ingress controller pods. + priorityClassName: + + readyStatus: + ## Enables readiness endpoint "/nginx-ready". The endpoint returns a success code when NGINX has loaded all the config after startup. + enable: true + + ## Set the port where the readiness endpoint is exposed. + port: 8081 + + ## Enable collection of latency metrics for upstreams. Requires prometheus.create. + enableLatencyMetrics: false + +rbac: + ## Configures RBAC. + create: true + +prometheus: + ## Expose NGINX or NGINX Plus metrics in the Prometheus format. + create: true + + ## Configures the port to scrape the metrics. + port: 9113 + + ## Specifies the namespace/name of a Kubernetes TLS Secret which will be used to protect the Prometheus endpoint. + secret: "" + + ## Configures the HTTP scheme used. + scheme: http diff --git a/kubernetes/helm_charts/secor/.helmignore b/kubernetes/helm_charts/secor/.helmignore new file mode 100644 index 0000000000..fbe01f88f2 --- /dev/null +++ b/kubernetes/helm_charts/secor/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/kubernetes/helm_charts/secor/Chart.yaml b/kubernetes/helm_charts/secor/Chart.yaml new file mode 100644 index 0000000000..1f85986391 --- /dev/null +++ b/kubernetes/helm_charts/secor/Chart.yaml @@ -0,0 +1,17 @@ +name: secor +version: 0.1.0 +appVersion: 0.25 +description: Secor is a service implementing Kafka log persistence +keywords: +- kafka +- secor +maintainers: +- name: secor-maintainers +sources: +- https://github.com/pinterest/secor +engine: gotpl +home: https://github.com/pinterest/secor + +dependencies: + - name: alertrules + condition: alertrules.enabled diff --git a/kubernetes/helm_charts/secor/charts/alertrules/.helmignore b/kubernetes/helm_charts/secor/charts/alertrules/.helmignore new file mode 100644 index 0000000000..50af031725 --- /dev/null +++ b/kubernetes/helm_charts/secor/charts/alertrules/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/secor/charts/alertrules/Chart.yaml b/kubernetes/helm_charts/secor/charts/alertrules/Chart.yaml new file mode 100644 index 0000000000..85e318cf75 --- /dev/null +++ b/kubernetes/helm_charts/secor/charts/alertrules/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: alertrules +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 1.16.0 diff --git a/kubernetes/helm_charts/secor/charts/alertrules/templates/_helpers.tpl b/kubernetes/helm_charts/secor/charts/alertrules/templates/_helpers.tpl new file mode 100644 index 0000000000..95896c3e8c --- /dev/null +++ b/kubernetes/helm_charts/secor/charts/alertrules/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "alertrules.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "alertrules.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "alertrules.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "alertrules.labels" -}} +helm.sh/chart: {{ include "alertrules.chart" . }} +{{ include "alertrules.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "alertrules.selectorLabels" -}} +app.kubernetes.io/name: {{ include "alertrules.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "alertrules.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "alertrules.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/secor/charts/alertrules/templates/promrulesSecorLag.yaml b/kubernetes/helm_charts/secor/charts/alertrules/templates/promrulesSecorLag.yaml new file mode 100644 index 0000000000..01395cc960 --- /dev/null +++ b/kubernetes/helm_charts/secor/charts/alertrules/templates/promrulesSecorLag.yaml @@ -0,0 +1,66 @@ +{{- if .Values.secor_alertrule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + role: alert-rules + app: {{ .Values.prometheus_rule_selector_app }} + release: {{ .Values.prometheus_rule_selector_release }} + name: sunbird-monitoring-{{ .Release.Name }}-secor-rules + namespace: monitoring +spec: + groups: + - name: alertrules.secor_jobs_lag_rule + rules: + - alert: secor job {{ .Release.Name }} lag + expr: kafka_consumergroup_lag_sum{consumergroup="{{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}", job="processing-kafka-exporter"} > {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_critical" }} + for: 30m + labels: + severity: critical + module: dp_lag + alerttype: lag + annotations: + message: The secor job {{ .Release.Name }} lag is above threshold {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_critical" }} + lag: {{`({{ humanize $value }})`}} + job_id: {{ .Release.Name }} + alertname: SecorJobsLag + + - alert: secor job {{ .Release.Name }} lag + expr: kafka_consumergroup_lag_sum{consumergroup="{{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}", job="processing-kafka-exporter"} > {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_warning" }} + for: 30m + labels: + severity: warning + module: dp_lag + alerttype: lag + annotations: + message: The secor job {{ .Release.Name }} lag is above threshold {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_warning" }} + lag: {{`({{ humanize $value }})`}} + job_id: {{ .Release.Name }} + alertname: SecorJobsLag + + - alert: secor job {{ .Release.Name }} lag + expr: kafka_consumergroup_lag_sum{consumergroup="{{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}", job="ingestion-kafka-exporter"} > {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_critical" }} + for: 30m + labels: + severity: critical + module: dp_lag + alerttype: lag + annotations: + message: The secor job {{ .Release.Name }} lag is above threshold {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_critical" }} + lag: {{`({{ humanize $value }})`}} + job_id: {{ .Release.Name }} + alertname: SecorJobsLag + + - alert: secor job {{ .Release.Name }} lag + expr: kafka_consumergroup_lag_sum{consumergroup="{{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}", job="ingestion-kafka-exporter"} > {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_warning" }} + for: 30m + labels: + severity: warning + module: dp_lag + alerttype: lag + annotations: + message: The secor job {{ .Release.Name }} lag is above threshold {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_warning" }} + lag: {{`({{ humanize $value }})`}} + job_id: {{ .Release.Name }} + alertname: SecorJobsLag +{{- end }} diff --git a/kubernetes/helm_charts/secor/config/log4j.docker.properties b/kubernetes/helm_charts/secor/config/log4j.docker.properties new file mode 100644 index 0000000000..06a1176823 --- /dev/null +++ b/kubernetes/helm_charts/secor/config/log4j.docker.properties @@ -0,0 +1,8 @@ +# log4j logging configuration. + +# root logger. +log4j.rootLogger=INFO, CONSOLE + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] (%c) %-5p %m%n diff --git a/kubernetes/helm_charts/secor/config/secor.azure.properties b/kubernetes/helm_charts/secor/config/secor.azure.properties new file mode 100644 index 0000000000..23f8b76dc9 --- /dev/null +++ b/kubernetes/helm_charts/secor/config/secor.azure.properties @@ -0,0 +1,27 @@ +include=secor.properties + +# Configure upload manager class to use Azure blob storage upload manager +secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager + +############ +# MUST SET # +############ + +# Microsoft Azure blob storage default endpoint protocol +secor.azure.endpoints.protocol=https + +# Microsoft Azure authentication credentials. +# https://azure.microsoft.com/en-us/documentation/articles/storage-create-storage-account +secor.azure.account.name={{ $.Values.azure_account }} +secor.azure.account.key={{ $.Values.azure_secret }} + +# Microsoft Azure blob storage container name. Container is a grouping of a set +# of blobs. https://msdn.microsoft.com/en-us/library/dd135715.aspx +secor.azure.container.name={{ $.Values.azure_container_name }} + +# Microsoft Azure blob storage path where files are stored within the container. +secor.azure.path={{- get (get $.Values.secor_jobs $.Release.Name) "base_path" }} + +################ +# END MUST SET # +################ \ No newline at end of file diff --git a/kubernetes/helm_charts/secor/config/secor.common.properties b/kubernetes/helm_charts/secor/config/secor.common.properties new file mode 100644 index 0000000000..7050ebcf1b --- /dev/null +++ b/kubernetes/helm_charts/secor/config/secor.common.properties @@ -0,0 +1,495 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############ +# MUST SET # +############ + +# Regular expression matching names of consumed topics. +secor.kafka.topic_filter={{ get (get $.Values.secor_jobs $.Release.Name) "topic" }} +secor.kafka.topic_blacklist= + +# Choose what to fill according to the service you are using +# in the choice option you can fill S3, GS, Swift or Azure +cloud.service=Azure + +# AWS authentication credentials. +# Leave empty if using IAM role-based authentication with s3a filesystem. +aws.access.key= +aws.secret.key= +aws.role= + +# Optional Proxy Setting. Set to true to enable proxy +# Only applicable to S3UploadManager +aws.proxy.isEnabled=false +aws.proxy.http.host= +aws.proxy.http.port= + +################ +# END MUST SET # +################ + + +# AWS region or endpoint. region should be a known region name (eg. +# us-east-1). endpoint should be a known S3 endpoint url. If neither +# are specified, then the default region (us-east-1) is used. If both +# are specified then endpoint is used. +# +# Only apply if the the S3UploadManager is used - see +# secor.upload.manager.class. +# +# http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region +aws.region= +aws.endpoint= + +# Toggle the AWS S3 client between virtual host style access and path style +# access. See http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html +aws.client.pathstyleaccess=false + +########################### +# START AWS S3 ENCRYPTION # +########################### + +# AWS specify type of server-side encryption, if any +# set to S3 to enable S3-managed encryption +# set to KMS to enable AWS KMS-managed encryption (see aws.sse.kms.key) +# set to customer to enable customer-managed encryption (see aws.sse.customer.key) +# set empty to disable encryption +aws.sse.type= + +# Key to use for S3 server-side encryption, base64-encoded +# Note: requires aws.sse.type to be set to customer to be used +aws.sse.customer.key= + +# KMS Key to use for S3 server-side encryption, base64-encoded +# Leave empty to use default generated key +# Note: requires aws.sse.type to be set to KMS to be used +aws.sse.kms.key= + +######################### +# END AWS S3 ENCRYPTION # +######################### + +# Hadoop filesystem to use. Choices are s3n or s3a. +# See https://wiki.apache.org/hadoop/AmazonS3 for details. +secor.s3.filesystem=s3n + +# Swift config, MUST configure if cloud.service=Swift + +# Swift Login Details: +swift.use.get.auth=true +swift.auth.url= +swift.tenant= +swift.username= +swift.port=8080 +swift.public=true + +# only needed if "swift.use.get.auth" = false +swift.password= + +# only needed if "swift.use.get.auth" = true +swift.api.key= + +# GS config, MUST configure if gcloud.service=GS + +# Name of the Google cloud storage bucket where log files are stored. +secor.gs.bucket=secor_gs + +# Google cloud storage path where files are stored within the bucket. +secor.gs.path=data + +# Use direct uploads +# WARNING: disables resumable uploads, files are uploaded in a single request +# This may help prevent IOException: insufficient data written, +# see https://github.com/pinterest/secor/issues/177 +# https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload +secor.gs.upload.direct=false + +# Application credentials configuration file +# https://developers.google.com/identity/protocols/application-default-credentials +# It can be empty when secor running in Google Cloud VMs with proper scopes +secor.gs.credentials.path= + +# Zookeeper config. +zookeeper.session.timeout.ms=3000 +zookeeper.sync.time.ms=200 + +# Zookeeper path (chroot) under which secor data will be placed. +secor.zookeeper.path=/ + +# Impacts how frequently the upload logic is triggered if no messages are delivered. +kafka.consumer.timeout.ms=10000 + +# Where consumer should read from if no committed offset in zookeeper. +# "smallest" -> read from earliest offset +# "largest" -> read from latest offset +# Always use "smallest" unless you know what you're doing and are willing to risk +# data loss for new topics or topics whose number of partitions has changed. +# See the kafka docs for "auto.offset.reset". +kafka.consumer.auto.offset.reset=smallest + +# Choose between range and roundrobin partition assignment strategy for kafka +# high level consumers. Check PartitionAssignor.scala in kafa 821 module for +# the differences between the two. +# In kafka 811, only range strategy is supported. +kafka.partition.assignment.strategy=range + +# Max number of retries during rebalance. +kafka.rebalance.max.retries= + +# Rebalance backoff. +kafka.rebalance.backoff.ms= + +# Kafka consumer receive buffer size (socket.receive.buffer.bytes) +kafka.socket.receive.buffer.bytes= + +# Kafka fetch max size (fetch.message.max.bytes) +kafka.fetch.message.max.bytes=5242880 + +# Kafka fetch min bytes (fetch.fetch.min.bytes) +kafka.fetch.min.bytes= + +# Kafka fetch max wait ms (fetch.max.wait.ms) +kafka.fetch.wait.max.ms= + +# Port of the broker serving topic partition metadata. +#kafka.seed.broker.port=9092 + +# Zookeeper path at which kafka is registered. In Zookeeper parlance, this is referred +# to as the chroot. +kafka.zookeeper.path=/ + +# Store offset in zookeeper and kafka consumer topic. +# Only used if kafka.offsets.storage is set to "kafka" +# http://kafka.apache.org/documentation.html#oldconsumerconfigs +# Possible values: true or false +kafka.dual.commit.enabled=true + +# Storage offset. +# Possible values: "zookeeper" to read offset from zookeeper or "kafka" to read offset from kafka consumer topic +kafka.offsets.storage=zookeeper + +# Parameter which tells whether to extract Kafka message timestamp. This value is to be chose in case of 0.10.x kafka brokers. +# Default value is false. Also specify `kafka.message.timestamp.className` as `com.pinterest.secor.timestamp.Kafka10MessageTimestamp`, +# in case you are enabling this parameter as `true`. +kafka.useTimestamp=false + +# Classname for the timestamp field you want to use. Default is `com.pinterest.secor.timestamp.Kafka10MessageTimestamp` +# for 0.10 build profile. Basically, it will be `Kafka8MessageTimestamp` for 0.8 kafka and `Kafka10MessageTimestamp` +# for 0.10 kafka. Fully classified names are `com.pinterest.secor.timestamp.Kafka8MessageTimestamp` and +# `com.pinterest.secor.timestamp.Kafka10MessageTimestamp`. +kafka.message.timestamp.className=com.pinterest.secor.timestamp.Kafka10MessageTimestamp + +# Secor generation is a version that should be incremented during non-backwards-compatible +# Secor releases. Generation number is one of the components of generated log file names. +# Generation number makes sure that outputs of different Secor versions are isolated. +secor.generation=1 + +# Number of consumer threads per Secor process. +secor.consumer.threads=7 + +# Consumption rate limit enforced at the process level (not a consumer-thread level). +secor.messages.per.second=10000 + +# Used by the "backup" consumer group only. +# Number of continuous message offsets that constitute a single offset= partition on s3. +# Example: +# if set to 10, +# messages with offsets 0 to 9 will be written to s3 path s3n://.../offset=0/... +# messages with offsets 10 to 19 will be written to s3 path s3n://.../offset=10/... +# ... +secor.offsets.per.partition=10000000 +secor.offsets.prefix=offset= +# How long does it take for secor to forget a topic partition. Applies to stats generation only. +secor.topic_partition.forget.seconds=600 + +# Setting the partitioner to use hourly partition +# By default, the partitioner will do daily partition, so the data will be +# written into +# s3n://.../topic/dt=2015-07-07/ +# If this parameter is set to true, the data will be written into +# s3n://.../topic/dt=2015-07-07/hr=02 +# The hour folder ranges from 00 to 23 +partitioner.granularity.hour=false +partitioner.granularity.minute=false + +partitioner.granularity.date.prefix=dt= +partitioner.granularity.hour.prefix=hr= +partitioner.granularity.minute.prefix=min= + +partitioner.granularity.date.format=yyyy-MM-dd +partitioner.granularity.hour.format=HH +partitioner.granularity.minute.format=mm + +# how many seconds should the finalizer wait to finalize a partition +partitioner.finalizer.delay.seconds=3600 + +# During partition finalization, the finalizer will start from the last +# time partition (e.g. dt=2015-07-17) and traverse backwards for n +# partition periods (e.g. dt=2015-07-16, dt=2015-07-15 ...) +# This parameter controls how many partition periods to traverse back +# The default is 10 +# secor.finalizer.lookback.periods=10 + +# If greater than 0, upon startup Secor will clean up directories and files under secor.local.path +# that are older than this value. +secor.local.log.delete.age.hours=-1 + +# Secor comes with a tool that adds Hive partitions for finalized topics. Currently, we support +# only Hive clusters accessible through Qubole. The token gives access to the Qubole API. +# It is available at https://api.qubole.com/users/edit +qubole.api.token= + +# hive tables are generally named after the topics. For instance if the topic +# is request_log the hive table is also called request_log. If you want this +# to be pinlog_request_log you can set this config to "pinlog_". This affects +# all topics. +hive.table.prefix= + +# You can also name your hive table directly if your hive table doesn't +# follow the pattern of +# E.g. hive.table.name.topic1=table1 to indicate that hive table for +# kafka topic will be named + +# Secor can export stats such as consumption lag (in seconds and offsets) per topic partition. +# Leave empty to disable this functionality. +tsdb.hostport= + +# Regex of topics that are not exported to TSDB. +monitoring.blacklist.topics= + +# Prefix of exported stats. +monitoring.prefix=secor + +# Monitoring interval. +# Set to 0 to disable - the progress monitor will run once and exit. +monitoring.interval.seconds=0 + +# Secor can export stats to statsd such as consumption lag (in seconds and offsets) per topic partition. +# Leave empty to disable this functionality. +statsd.hostport= + +# Thrift protocol class. It applies to timestamp extractor below and parquet output for thrift messages. +# TBinaryProtocol by default +secor.thrift.protocol.class= + +# Thrift message class. It applies to parquet output. +# If all Kafka topics transfer the same thrift message type, set secor.thrift.message.class.*= +secor.thrift.message.class.*= + +# If true, the consumer group will be the initial prefix of all +# exported metrics, before `monitoring.prefix` (if set). +# +# Setting to false and use monitoring.prefix can lead to nice paths. +# For example, +# secor.kafka.group = secor_hr_partition +# monitoring.prefix = secor.hr +# statsd.prefixWithConsumerGroup = false +# => secor.hr.lag.offsets.. +# +# secor.kafka.group = secor_hr_partition +# monitoring.prefix = secor +# statsd.prefixWithConsumerGroup = true +# => secor_hr_partition.secor.lag.offsets.. +statsd.prefixWithConsumerGroup=true + +# Name of field that contains timestamp for JSON, MessagePack, or Thrift message parser. (1405970352123) +message.timestamp.name={{ get (get $.Values.secor_jobs $.Release.Name) "timestamp_key" }} +message.fallback.timestamp.name={{ get (get $.Values.secor_jobs $.Release.Name) "fallback_timestamp_key" }} + +# Separator for defining message.timestamp.name in a nested structure. E.g. +# {"meta_data": {"created": "1405911096123", "last_modified": "1405912096123"}, "data": "test"} +# message.timestamp.name=meta_data.created +# message.timestamp.name.separator=. +message.timestamp.name.separator= + +# Field ID of the field that contains timestamp for Thrift message parser. +# N.B. setting this past 1 will come with a performance penalty +message.timestamp.id=1 + +# Data type of the timestamp field for thrift message parser. +# Supports i64 and i32. +message.timestamp.type=i64 + +# Name of field that contains a timestamp, as a date Format, for JSON. (2014-08-07, Jul 23 02:16:57 2005, etc...) +# Should be used when there is no timestamp in a Long format. Also ignore time zones. +message.timestamp.input.pattern=yyyy-MM-dd + +# whether timestamp field is required, it should always be required. But +# for historical reason, we didn't enforce this check, there might exist some +# installations with messages missing timestamp field +message.timestamp.required=true + +# To enable compression, set this to a valid compression codec implementing +# org.apache.hadoop.io.compress.CompressionCodec interface, such as +# 'org.apache.hadoop.io.compress.GzipCodec'. +secor.compression.codec=org.apache.hadoop.io.compress.GzipCodec + +# To set a custom file extension set this to a valid file suffix, such as +# '.gz', '.part', etc. +secor.file.extension=.gz + +# The secor file reader/writer used to read/write the data, by default we write sequence files +#secor.file.reader.writer.factory=com.pinterest.secor.io.impl.SequenceFileReaderWriterFactory +secor.file.reader.writer.factory=com.pinterest.secor.io.impl.DelimitedTextFileReaderWriterFactory +#if left blank defaults to \n +secor.file.reader.Delimiter=\n +#if left blank no Delimiter is added. do not use \ as that needs to be escaped and is an escape +#character and not a delimtier. +secor.file.writer.Delimiter=\n + +# Max message size in bytes to retrieve via KafkaClient. This is used by ProgressMonitor and PartitionFinalizer. +# This should be set large enough to accept the max message size configured in your kafka broker +# Default is 0.1 MB +secor.max.message.size.bytes=100000 + +# Class that will manage uploads. Default is to use the hadoop +# interface to S3. +secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager + +#Set below property to your timezone, and the events will be parsed and converted to the timezone specified +secor.message.timezone=UTC + +#Set below property to your timezone, and partitions in s3 will be created as per timezone provided +secor.parser.timezone=Asia/Kolkata + +# Transformer class that transform message accordingly. +secor.message.transformer.class=com.pinterest.secor.transformer.IdentityMessageTransformer + +# Set below property to true if you want to have the md5hash appended to your s3 path. +# This helps for better partitioning of the data on s3. Which gives better performance while reading and writing on s3 +secor.s3.prefix.md5hash=false + +# After the given date, secor will upload files to the supplied s3 alternative path +secor.s3.alter.path.date= + +# An alternative S3 path for secor to upload files to +secor.s3.alternative.path= + +# If enabled, add calls will be made to qubole, otherwise, skip qubole call for finalization +secor.enable.qubole=true + +# Timeout value for qubole calls +secor.qubole.timeout.ms=300000 + +# Topics to upload at a fixed minute mark +secor.kafka.upload_at_minute_mark.topic_filter= + +# What the minute mark is. This isn't triggered unless the topic name matches +secor.upload.minute_mark=0 + +# File age per topic and per partition is checked against secor.max.file.age.seconds by looking at +# the youngest file when true or at the oldest file when false. +secor.file.age.youngest=true + +# Class that manages metric collection. +# Sending metrics to Ostrich is the default implementation. +secor.monitoring.metrics.collector.class=com.pinterest.secor.monitoring.OstrichMetricCollector + +# Row group size in bytes for Parquet writers. Specifies how much data will be buffered in memory before flushing a +# block to disk. Larger values allow for larger column chinks which makes it possible to do larger sequential IO. +# Should be aligned with HDFS blocks. Defaults to 128MB in Parquet 1.9. +parquet.block.size=134217728 + +# Page group size in bytes for Parquet writers. Indivisible unit for columnar data. Smaller data pages allow for more +# fine grained reading but have higher space overhead. Defaults to 1MB in Parquet 1.9. +parquet.page.size=1048576 + +# Enable or disable dictionary encoding for Parquet writers. The dictionary encoding builds a dictionary of values +# encountered in a given column. Defaults to true in Parquet 1.9. +parquet.enable.dictionary=true + +# Enable or disable validation for Parquet writers. Validates records written against the schema. Defaults to false in +# Parquet 1.9. +parquet.validation=false + +# User can configure ORC schema for each Kafka topic. Common schema is also possible. This property is mandatory +# if DefaultORCSchemaProvider is used. ORC schema for all the topics should be defined like this: +secor.orc.message.schema.*=struct\,f:array\,g:int> +# Below config used for defining ORC schema provider class name. User can use the custom implementation for orc schema provider +secor.orc.schema.provider=com.pinterest.secor.util.orc.schema.DefaultORCSchemaProvider + +# Changes related to 0.29 upgrade + +# If true, runs a final upload when the JVM is shut down (eg, from SIGTERM). +# You may want to ensure that whatever system you use to run secor sets an +# appropriate grace period to allow a full upload before a forced termination. +secor.upload.on.shutdown=false + +# If true, uploads lastSeen offset onto ZK, see https://github.com/pinterest/secor/issues/600 for details +secor.upload.secor.upload.last.seen.offset=false + +# If true, uploads are entirely deterministic, which can avoid some race conditions +# which can lead to messages being backed up multiple times. This is incompatible with +# secor.upload.on.shutdown=true, and ignores the values of secor.max.file.size.bytes, +# secor.max.file.age.seconds, and secor.upload.minute_mark. +# +# In deterministic mode, you must set one or both of secor.max.file.timestamp.range.millis and +# secor.max.input.payload.size.bytes. These determine when to upload, and are ignored outside +# of deterministic mode. +secor.upload.deterministic=false + +# Classname for the message iterator you want to use. The message iterator determines what kind of consumer +# secor will use to communicate with kafka. com.pinterest.secor.reader.LegacyKafkaMessageIterator uses +# the old kafka consumer written scala. Its not recommended to use the legacy iterator with kafka version >= 1.0 since it +# does not support the new broker protocols. You may face significant performance degradation on your brokers if you use it +kafka.message.iterator.className=com.pinterest.secor.reader.SecorKafkaMessageIterator + +# Classname for the kafka client used by utility classes like PartitionFinalizer and Progress monitor. +# Legacy client is kept for compatibility purposes and is deprecated +kafka.client.className=com.pinterest.secor.common.SecorKafkaClient + + +######################################## +# Optional: Kafka New Consumer Configs # +######################################## + +# Same as old configuration. Except accepted values are earliest and latest instead of smallest +# and largest +kafka.new.consumer.auto.offset.reset=earliest + +# Comma-separated list of topics to consume. Please note that this is not a regular expression. +# If that's what you want, you can use "secor.kafka.topic_filter" instead. +kafka.new.consumer.topic.list={{ get (get $.Values.secor_jobs $.Release.Name) "topic" }} + +kafka.new.consumer.poll.timeout.seconds=10 +kafka.new.consumer.request.timeout.ms=10000 +kafka.new.consumer.ssl.key.password= +kafka.new.consumer.ssl.keystore.location= +kafka.new.consumer.ssl.keystore.password= +kafka.new.consumer.ssl.truststore.location= +kafka.new.consumer.ssl.truststore.password= +kafka.new.consumer.isolation.level= +kafka.new.consumer.max.poll.interval.ms= +kafka.new.consumer.max.poll.records= +kafka.new.consumer.sasl.client.callback.handler.class= +kafka.new.consumer.sasl.jaas.config= +kafka.new.consumer.sasl.kerberos.service.name= +kafka.new.consumer.sasl.login.callback.handler.class= +kafka.new.consumer.sasl.login.class= +kafka.new.consumer.sasl.mechanism= +kafka.new.consumer.security.protocol= +kafka.new.consumer.ssl.enabled.protocols= +kafka.new.consumer.ssl.keystore.type= +kafka.new.consumer.ssl.protocol= +kafka.new.consumer.ssl.provider= +kafka.new.consumer.ssl.truststore.type= +kafka.new.consumer.partition.assignment.strategy.class= + +######################################## +# End: Kafka New Consumer Configs # +######################################## + +kafka.fetch.max.bytes= \ No newline at end of file diff --git a/kubernetes/helm_charts/secor/config/secor.partition.properties b/kubernetes/helm_charts/secor/config/secor.partition.properties new file mode 100644 index 0000000000..743e1bab86 --- /dev/null +++ b/kubernetes/helm_charts/secor/config/secor.partition.properties @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include=secor.properties +include=secor.azure.properties + +# Name of the Kafka consumer group. +secor.kafka.group={{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }} + +# Parser class that extracts s3 partitions from consumed messages. +secor.message.parser.class={{ get (get $.Values.secor_jobs $.Release.Name) "message_parser" }} + +# S3 path where sequence files are stored. +secor.s3.path= + +# Swift path where sequence files are stored. +secor.swift.path=secor_dev/partition + +# Azure path where sequence files are stored. +secor.azure.path={{ get (get $.Values.secor_jobs $.Release.Name) "base_path" }} + +# Local path where sequence files are stored before they are uploaded to s3. +secor.local.path=/mnt/data/{{ get (get $.Values.secor_jobs $.Release.Name) "service_name" }}/message_logs/partition + +# Port of the Ostrich server. +#ostrich.port= + +# Secor custom properties + +# Partition Date Output format. This is used along with PatternDateMessageParser. Defaults to 'yyyy-MM-dd' *New* +secor.partition.output_dt_format=yyyy-MM-dd + +secor.partition.prefix.enable={{ get (get $.Values.secor_jobs $.Release.Name) "partition_prefix_enabled" }} +# Name of field that contains timestamp for JSON, MessagePack, or Thrift message parser. (1405970352123) +secor.partition.prefix.identifier={{ get (get $.Values.secor_jobs $.Release.Name) "partition_prefix_key" }} + +secor.partition.prefix.mapping={{ get (get $.Values.secor_jobs $.Release.Name) "partition_prefix_mapping" }} + +secor.max.file.age.policy=oldest + +# Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz. +# Available placeholders are +# topic - The topic name the data is being fetched +# partition - The partition name +# generation - Generation +# kafkaPartition - The kafka partition +# fmOffset - First Message offset in the file. +# randomHex - A 4 character random hex to append to the file name +# currentTimestamp - Time of upload in epoch format +# currentTime - Time of upload in HH-mm format +# currentDate - Time of upload in YYYYMMDD format +# folder - Folder to use based on message id map lookup +secor.s3.output_file_pattern={{ get (get $.Values.secor_jobs $.Release.Name) "output_file_pattern" }} + +secor.partition.message.channel.identifier={{ get (get $.Values.secor_jobs $.Release.Name) "message_channel_identifier" }} diff --git a/kubernetes/helm_charts/secor/config/secor.properties b/kubernetes/helm_charts/secor/config/secor.properties new file mode 100644 index 0000000000..6f2876d1de --- /dev/null +++ b/kubernetes/helm_charts/secor/config/secor.properties @@ -0,0 +1,40 @@ +include=secor.common.properties + +############ +# MUST SET # +############ + +# Fill the section which fits your needs +############### +# Using S3 # +############### + +# Name of the s3 bucket where log files are stored. +secor.s3.bucket= + +############### +# Using Swift # +############### + +# Boolean variable which determines if each topic will be uploaded to different container +# The Containers for the topics will be Created automatically +# If true, then the setting "secor.swift.container" will be ignored +secor.swift.containers.for.each.topic=false + +# Name of swift container where log files are stored. +secor.swift.container=logsContainer + +################ +# END MUST SET # +################ + +kafka.seed.broker.host={{ get (get $.Values.secor_jobs $.Release.Name) "kafka_broker_host" }} +kafka.seed.broker.port=9092 + +zookeeper.quorum={{ get (get $.Values.secor_jobs $.Release.Name) "zookeeper_quorum" }} + +# Upload policies. +# 10K +secor.max.file.size.bytes={{ get (get $.Values.secor_jobs $.Release.Name) "max_file_size" }} +# 10 seconds +secor.max.file.age.seconds={{ get (get $.Values.secor_jobs $.Release.Name) "max_file_age" }} diff --git a/kubernetes/helm_charts/secor/templates/_helpers.tpl b/kubernetes/helm_charts/secor/templates/_helpers.tpl new file mode 100644 index 0000000000..a63432d8bf --- /dev/null +++ b/kubernetes/helm_charts/secor/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "secor.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "name" -}} +{{- default .Release.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "secor.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "secor.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +App config names +*/}} +{{- define "secor.appConfigName" -}} +{{- $.Release.Name -}} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "secor.selectorLabels" -}} +app.kubernetes.io/name: {{ include "secor.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/kubernetes/helm_charts/secor/templates/secor-config.yaml b/kubernetes/helm_charts/secor/templates/secor-config.yaml new file mode 100644 index 0000000000..f3d9cf3a55 --- /dev/null +++ b/kubernetes/helm_charts/secor/templates/secor-config.yaml @@ -0,0 +1,23 @@ +{{- range $job_name, $job_config := .Values.secor_jobs }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $.Release.Name }}-config + namespace: {{ $.Values.namespace }} + labels: + app: {{ $.Release.Name }}-config + chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} + release: {{ $.Release.Name }} + heritage: {{ $.Release.Service }} +data: + secor.properties: |+ +{{- (tpl ($.Files.Get "config/secor.properties") $) | nindent 4 }} + secor.common.properties: |+ +{{- (tpl ($.Files.Get "config/secor.common.properties") $) | nindent 4 }} + secor.azure.properties: |+ +{{- (tpl ($.Files.Get "config/secor.azure.properties") $) | nindent 4 }} + secor.partition.properties: |+ +{{- (tpl ($.Files.Get "config/secor.partition.properties") $) | nindent 4 }} + log4j.properties: |+ +{{ ($.Files.Glob "config/log4j*.properties").AsConfig | nindent 4 }} +{{- end }} diff --git a/kubernetes/helm_charts/secor/templates/secor-statefulset.yaml b/kubernetes/helm_charts/secor/templates/secor-statefulset.yaml new file mode 100644 index 0000000000..bc20b514e1 --- /dev/null +++ b/kubernetes/helm_charts/secor/templates/secor-statefulset.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + configmap.reloader.stakater.com/reload: {{ .Release.Name }}-config + name: {{ .Release.Name }} + namespace: {{ .Values.namespace }} + labels: + app: secor + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + serviceName: secor + updateStrategy: + type: RollingUpdate + replicas: {{ get (get $.Values.secor_jobs $.Release.Name) "replicas" }} + selector: + matchLabels: + app: secor + release: {{ .Release.Name }} + template: + metadata: + labels: + app: secor + release: {{ .Release.Name }} + spec: + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} + initContainers: + - name: change-pv-dir-ownership + image: alpine:3 + command: ["/bin/sh", "-c"] + args: ["chown -R 9999:9999 /mnt/data"] + volumeMounts: + - name: data-path + mountPath: /mnt/data + containers: + - name: {{ .Release.Name }}-secor + image: {{ .Values.image.prefix | default "" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: "CONFIG_FILE" + value: "/opt/secor/config/secor.partition.properties" + - name: "LOG4J_CONFIGURATION" + value: "/opt/secor/config/log4j.docker.properties" + + volumeMounts: + - mountPath: /opt/secor/config + name: secor-config + - name: "data-path" + mountPath: /mnt/data + resources: + requests: + cpu: {{ get (get (get $.Values.secor_jobs $.Release.Name) "requests") "cpu" }} + memory: {{ get (get (get $.Values.secor_jobs $.Release.Name) "requests") "memory"}} + securityContext: + runAsUser: 9999 + + volumes: + - configMap: + name: {{ .Release.Name }}-config + name: secor-config + + volumeClaimTemplates: + - metadata: + name: data-path + labels: + app: secor + spec: + storageClassName: "{{- .Values.storageClass }}" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ get (get (get $.Values.secor_jobs $.Release.Name) "storage") "size" }} + diff --git a/kubernetes/helm_charts/secor/values.j2 b/kubernetes/helm_charts/secor/values.j2 new file mode 100644 index 0000000000..4aa2e0ee83 --- /dev/null +++ b/kubernetes/helm_charts/secor/values.j2 @@ -0,0 +1,63 @@ +azure_account: "{{ sunbird_private_storage_account_name }}" +azure_secret: "{{ sunbird_private_storage_account_key }}" +azure_container_name: "telemetry-data-store" + +namespace: {{ secor_namespace }} +storageClass: {{ secor_storage_class | default('default') }} +imagepullsecrets: {{ imagepullsecrets }} + +secor_jobs: +{% for item in secor_job_list %} + {{ item }}: + replicas: "{{ secor_jobs['%s'|format(item)].replicas }}" + consumer_group: "{{ secor_jobs['%s'|format(item)].consumer_group }}" + service_name: "{{ secor_jobs['%s'|format(item)].service_name }}" + base_path: "{{ secor_jobs['%s'|format(item)].base_path }}" + timestamp_key: "{{ secor_jobs['%s'|format(item)].timestamp_key }}" + fallback_timestamp_key: "{{ secor_jobs['%s'|format(item)].fallback_timestamp_key }}" + topic: "{{ secor_jobs['%s'|format(item)].topic }}" + kafka_broker_host: "{{ secor_jobs['%s'|format(item)].kafka_broker_host }}" + zookeeper_quorum: "{{ secor_jobs['%s'|format(item)].zookeeper_quorum }}" + max_file_size: "{{ secor_jobs['%s'|format(item)].max_file_size }}" + max_file_age: "{{ secor_jobs['%s'|format(item)].max_file_age }}" + partition_prefix_enabled: "{{ secor_jobs['%s'|format(item)].partition_prefix_enabled }}" + partition_prefix_key: "{{ secor_jobs['%s'|format(item)].partition_prefix_key }}" + partition_prefix_mapping: '{{ secor_jobs['%s'|format(item)].partition_prefix_mapping }}' + message_channel_identifier: "{{ secor_jobs['%s'|format(item)].message_channel_identifier }}" + output_file_pattern: "{{ secor_jobs['%s'|format(item)].output_file_pattern }}" + message_parser: "{{ secor_jobs['%s'|format(item)].message_parser }}" + storage: + size: "{{ secor_jobs['%s'|format(item)].storage.size }}" + requests: + cpu: "{{ secor_jobs['%s'|format(item)].requests.cpu }}" + memory: "{{ secor_jobs['%s'|format(item)].requests.memory }}" + lag_threshold_warning: "{{ secor_jobs['%s'|format(item)].lag_threshold_warning }}" + lag_threshold_critical: "{{ secor_jobs['%s'|format(item)].lag_threshold_critical }}" +{% endfor %} + + +image: + prefix: {{ dockerhub }} + repository: {{ image_repository }} + tag: {{ image_tag }} + pullPolicy: {{ image_pullPolicy }} + +exporter: + image: + repository: prom/statsd-exporter + tag: latest + pullPolicy: IfNotPresent + +prometheus_rule_selector_app: prometheus-operator +prometheus_rule_selector_release: prometheus-operator + +# If you enable this, secor lag alert rules will be created in the flink cluster. +# In our case the consumer group lag metrics available in core prometheus. +# So we need to create the secor lag alert rule in core prometheus. +# By adding this condition we are avoiding creating the secor lag alert rule in flink cluster. +alertrules: + enabled: false + +# This condition is whether to create the secor lag alert rule or not. +secor_alertrule: + enabled: {{ secor_alertrule_enabled | lower}} diff --git a/kubernetes/pipelines/bootstrap/Jenkinsfile b/kubernetes/pipelines/bootstrap/Jenkinsfile new file mode 100644 index 0000000000..e70b782daf --- /dev/null +++ b/kubernetes/pipelines/bootstrap/Jenkinsfile @@ -0,0 +1,46 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + stage('deploy') { + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "$currentWs/kubernetes/ansible/bootstrap_minimal.yaml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + ansible_playbook_run(values) + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + summary() + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/kubernetes/pipelines/build/Jenkinsfile b/kubernetes/pipelines/build/Jenkinsfile index 1660222db2..52004a4d94 100644 --- a/kubernetes/pipelines/build/Jenkinsfile +++ b/kubernetes/pipelines/build/Jenkinsfile @@ -14,19 +14,10 @@ node('build-slave') { } else println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL) } - cleanWs() - if (params.github_release_tag == "") { - checkout scm - commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim() - branch_name = sh(script: 'git name-rev --name-only HEAD | rev | cut -d "/" -f1| rev', returnStdout: true).trim() - build_tag = branch_name + "_" + commit_hash + "_" + env.BUILD_NUMBER - println(ANSI_BOLD + ANSI_YELLOW + "github_release_tag not specified, using the latest commit hash: " + commit_hash + ANSI_NORMAL) - } else { - def scmVars = checkout scm - checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$params.github_release_tag"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]] - build_tag = params.github_release_tag + "_" + env.BUILD_NUMBER - println(ANSI_BOLD + ANSI_YELLOW + "github_release_tag specified, building from tag: " + params.github_release_tag + ANSI_NORMAL) - } + cleanWs() + checkout scm + commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim() + build_tag = sh(script: "echo " + params.github_release_tag.split('/')[-1] + "_" + commit_hash + "_" + env.BUILD_NUMBER, returnStdout: true).trim() echo "build_tag: " + build_tag stage('Build') { diff --git a/kubernetes/pipelines/build/auto_build_deploy b/kubernetes/pipelines/build/auto_build_deploy new file mode 100644 index 0000000000..bae59da55b --- /dev/null +++ b/kubernetes/pipelines/build/auto_build_deploy @@ -0,0 +1,66 @@ +@Library('deploy-conf') _ +node('build-slave') { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + ansiColor('xterm') { + withEnv(["JAVA_HOME=${JAVA11_HOME}"]) { + stage('Checkout') { + tag_name = env.JOB_NAME.split("/")[-1] + pre_checks() + if (!env.hub_org) { + println(ANSI_BOLD + ANSI_RED + "Uh Oh! Please set a Jenkins environment variable named hub_org with value as registery/sunbidrded" + ANSI_NORMAL) + error 'Please resolve the errors and rerun..' + } else + println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL) + } + cleanWs() + def scmVars = checkout scm + checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$tag_name"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]] + build_tag = tag_name + "_" + env.BUILD_NUMBER + commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim() + artifact_version = tag_name + "_" + commit_hash + echo "build_tag: " + build_tag + + // stage Build + env.NODE_ENV = "build" + print "Environment will be : ${env.NODE_ENV}" + dir('data-pipeline-flink'){ + sh '/opt/apache-maven-3.6.3/bin/mvn3.6 clean install -DskipTests' + } + + // stage Package + dir('data-pipeline-flink/sunbird-dp-distribution') { + sh "/opt/apache-maven-3.6.3/bin/mvn3.6 package -Pbuild-docker-image -Drelease-version=${build_tag}" + } + + + // stage Retagging + sh """ + docker tag sunbird-datapipeline:${build_tag} ${hub_org}/sunbird-datapipeline:${build_tag} + echo {\\"image_name\\" : \\"sunbird-datapipeline\\", \\"image_tag\\" : \\"${build_tag}\\", \\"node_name\\" : \\"${env.NODE_NAME}\\"} > metadata.json + """ + + // stage ArchiveArtifacts + archiveArtifacts "metadata.json" + currentBuild.description = "${build_tag}" + + + } + currentBuild.result = "SUCCESS" + slack_notify(currentBuild.result, tag_name) + email_notify() + auto_build_deploy() + } + } + catch (err) { + currentBuild.result = "FAILURE" + slack_notify(currentBuild.result, tag_name) + email_notify() + throw err + } +} diff --git a/kubernetes/pipelines/deploy/Jenkinsfile b/kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile similarity index 100% rename from kubernetes/pipelines/deploy/Jenkinsfile rename to kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile diff --git a/kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile.startstop b/kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile.startstop new file mode 100644 index 0000000000..75bcc043f3 --- /dev/null +++ b/kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile.startstop @@ -0,0 +1,51 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/kubernetes/ansible/flink_jobs_stop_start.yml" + ansibleExtraArgs = "--extra-vars \"job_names=${params.job_names}\" --tags ${params.action} --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + // slack_notify(currentBuild.result) + // email_notify() + } +} diff --git a/kubernetes/pipelines/deploy/secor/Jenkinsfile b/kubernetes/pipelines/deploy/secor/Jenkinsfile new file mode 100644 index 0000000000..5d10e799fd --- /dev/null +++ b/kubernetes/pipelines/deploy/secor/Jenkinsfile @@ -0,0 +1,53 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + stage('deploy') { + values = docker_params() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim().toLowerCase() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "$currentWs/kubernetes/ansible/secor_deploy.yaml" + ansibleExtraArgs = "--extra-vars \"chart_path=${currentWs}/kubernetes/helm_charts/secor job_names_to_deploy=${params.job_names_to_deploy} image_tag=$values.image_tag\" --skip-tags alertrule --vault-password-file /var/lib/jenkins/secrets/vault-pass -v" + values.put('currentWs', currentWs) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + ansible_playbook_run(values) + archiveArtifacts 'metadata.json' + currentBuild.description = "Image: ${values.image_tag}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + if (params.monitoring_enabled) { + ansiblePlaybook = "${currentWs}/kubernetes/ansible/secor_deploy.yaml" + ansibleExtraArgs = "--extra-vars \"chart_path=${currentWs}/kubernetes/helm_charts/secor job_names_to_deploy=${params.job_names_to_deploy}\" --skip-tags secor_deploy --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + } else { + echo "Alert rule creation skipped" + } + } + summary() + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/kubernetes/pipelines/monitoring/Jenkinsfile b/kubernetes/pipelines/monitoring/Jenkinsfile index 8f95cdd3e3..1e9f2d4935 100644 --- a/kubernetes/pipelines/monitoring/Jenkinsfile +++ b/kubernetes/pipelines/monitoring/Jenkinsfile @@ -24,7 +24,7 @@ node() { jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim().toLowerCase() currentWs = sh(returnStdout: true, script: 'pwd').trim() ansiblePlaybook = "$currentWs/kubernetes/ansible/monitoring.yaml" - ansibleExtraArgs = "--extra-vars \"chart_path=${currentWs}/kubernetes/helm_charts/$jobName kubeconfig_path=${params.kubeconfigpath} release_name=$jobName\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -v" + ansibleExtraArgs = "--extra-vars \"chart_path=${currentWs}/kubernetes/helm_charts/$jobName release_name=$jobName\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -v" values.put('currentWs', currentWs) values.put('env', envDir) values.put('module', module) diff --git a/kubernetes/pipelines/provision/druid/Jenkinsfile b/kubernetes/pipelines/provision/druid/Jenkinsfile new file mode 100644 index 0000000000..c7ce8493d4 --- /dev/null +++ b/kubernetes/pipelines/provision/druid/Jenkinsfile @@ -0,0 +1,50 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + stage('deploy') { + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "$currentWs/kubernetes/ansible/druid_provision.yml" + ansibleExtraArgs = "--extra-vars \"chart_base_path=${currentWs}/kubernetes/helm_charts cluster_type=${params.druid_cluster_type}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -v" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = "SUCCESS" + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + + summary() + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/pipelines/backup/redis-restore/Jenkinsfile b/pipelines/backup/redis-restore/Jenkinsfile index fd7ae88f7d..adfae91886 100644 --- a/pipelines/backup/redis-restore/Jenkinsfile +++ b/pipelines/backup/redis-restore/Jenkinsfile @@ -26,7 +26,7 @@ node("ops-slave") { module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() ansiblePlaybook = "${currentWs}/ansible/redis-restore.yml" - ansibleExtraArgs = "--extra-vars \"redis_restore_file_name=${redis_restore_file_name}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + ansibleExtraArgs = "--extra-vars \"redis_restore_file_name=${redis_restore_file_name} redis_restore_process=${redis_restore_process}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" values.put('currentWs', currentWs) values.put('env', envDir) values.put('module', module) diff --git a/pipelines/deploy/PopulatePSQL-ConsumerChannelMapping/Jenkinsfile b/pipelines/deploy/PopulatePSQL-ConsumerChannelMapping/Jenkinsfile index e5433ab0df..50895c39a0 100644 --- a/pipelines/deploy/PopulatePSQL-ConsumerChannelMapping/Jenkinsfile +++ b/pipelines/deploy/PopulatePSQL-ConsumerChannelMapping/Jenkinsfile @@ -25,8 +25,8 @@ node() { module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() currentWs = sh(returnStdout: true, script: 'pwd').trim() - ansiblePlaybook = "${currentWs}/ansible/postgres-provision.yml" - ansibleExtraArgs = "--connection=local --extra-vars \"consumer_id=${params.Consumer_id} channel=${params.Channel_id} status=${params.status} created_by=${params.Created_by}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + ansiblePlaybook = "${currentWs}/ansible/postgres-db-update.yml" + ansibleExtraArgs = "--connection=local --tags insert --extra-vars \"consumer_id=${params.Consumer_id} channel=${params.Channel_id} status=${params.status} created_by=${params.Created_by}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" values.put('currentWs', currentWs) values.put('env', envDir) values.put('module', module) @@ -35,14 +35,6 @@ node() { values.put('ansibleExtraArgs', ansibleExtraArgs) println values ansible_playbook_run(values) - - - ansiblePlaybook = "${currentWs}/ansible/lpa_api_deploy.yml" - ansibleExtraArgs = "--tags refresh-cache --vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) currentBuild.result = "SUCCESS" currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" } diff --git a/pipelines/deploy/analytics-api/Jenkinsfile b/pipelines/deploy/analytics-api/Jenkinsfile deleted file mode 100644 index e0e164d2cd..0000000000 --- a/pipelines/deploy/analytics-api/Jenkinsfile +++ /dev/null @@ -1,63 +0,0 @@ -@Library('deploy-conf') _ -node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - folder = new File("$WORKSPACE/.git") - if (folder.exists()) - { - println "Found .git folder. Clearing it.." - sh'git clean -fxd' - } - checkout scm - - } - - ansiColor('xterm') { - values = lp_dp_params() - stage('get artifact') { - currentWs = sh(returnStdout: true, script: 'pwd').trim() - artifact = values.artifact_name + ":" + values.artifact_version - values.put('currentWs', currentWs) - values.put('artifact', artifact) - artifact_download(values) - } - stage('deploy artifact'){ - sh """ - unzip ${artifact} - mv analytics-api-2.0-dist.zip ansible - """ - ansiblePlaybook = "${currentWs}/ansible/lpa_api_deploy.yml" - ansibleExtraArgs = " --vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true - archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true - ansiblePlaybook = "${currentWs}/ansible/analyticsapi_logstash.yml" - ansibleExtraArgs = " --vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = "SUCCESS" - currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" - } - } - summary() - } - catch (err) { - currentBuild.result = "FAILURE" - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } -} diff --git a/pipelines/deploy/content-snapshot-indexer/Jenkinsfile b/pipelines/deploy/content-snapshot-indexer/Jenkinsfile index 74299bc1bf..c4591aa4aa 100644 --- a/pipelines/deploy/content-snapshot-indexer/Jenkinsfile +++ b/pipelines/deploy/content-snapshot-indexer/Jenkinsfile @@ -30,11 +30,11 @@ node() { stage('deploy artifact'){ sh """ unzip ${artifact} - mv adhoc-jobs-1.0.jar ansible/roles/content-snapshot-indexer/files/ - mv adhoc-jobs-1.0-distribution.tar.gz ansible/roles/content-snapshot-indexer/files/ + mv etl-jobs-1.0.jar ansible/roles/content-snapshot-indexer/files/ + mv etl-jobs-1.0-distribution.tar.gz ansible/roles/content-snapshot-indexer/files/ """ ansiblePlaybook = "${currentWs}/ansible/content-snapshot-indexer.yml" - ansibleExtraArgs = " --tags ${invoke_type} --extra-vars \"script_to_run=${params.script_to_run}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + ansibleExtraArgs = " --tags ${invoke_type} --extra-vars \"script_to_run=${params.script_to_run} date=${params.date} identifier=${params.user_id} populate_anonymous_user=${params.populate_anonymous_user} refresh_data=${params.refresh_data}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" values.put('ansiblePlaybook', ansiblePlaybook) values.put('ansibleExtraArgs', ansibleExtraArgs) println values diff --git a/pipelines/deploy/data-products/Jenkinsfile b/pipelines/deploy/data-products/Jenkinsfile index cd76b86290..80cd2b3e91 100644 --- a/pipelines/deploy/data-products/Jenkinsfile +++ b/pipelines/deploy/data-products/Jenkinsfile @@ -43,9 +43,9 @@ node() { echo "analytics-framework-2.0.jar exist" mv analytics-framework-2.0.jar ansible fi - if [ -f scruid_2.11-2.3.2.jar ]; then - echo "scruid_2.11-2.3.2.jar exist" - mv scruid_2.11-2.3.2.jar ansible + if [ -f scruid_2.11-2.4.0.jar ]; then + echo "scruid_2.11-2.4.0.jar exist" + mv scruid_2.11-2.4.0.jar ansible fi """ ansiblePlaybook = "--extra-vars \"remote=${params.remote}\" ${currentWs}/ansible/lpa_data-products_deploy.yml" @@ -67,7 +67,7 @@ node() { throw err } finally { - slack_notify(currentBuild.result) - email_notify() + // slack_notify(currentBuild.result) + // email_notify() } } diff --git a/pipelines/deploy/datascience/Jenkinsfile b/pipelines/deploy/datascience/Jenkinsfile index c6f9cdbce0..c567e434a1 100644 --- a/pipelines/deploy/datascience/Jenkinsfile +++ b/pipelines/deploy/datascience/Jenkinsfile @@ -33,7 +33,7 @@ node() { mv dataproducts.tar.gz ansible """ ansiblePlaybook = "${currentWs}/ansible/portal.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass --tags common" + ansibleExtraArgs = "--extra-vars \"date=${params.date}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass --tags ${params.tags}" values.put('ansiblePlaybook', ansiblePlaybook) values.put('ansibleExtraArgs', ansibleExtraArgs) println values diff --git a/pipelines/deploy/druid-anomaly-detection/Jenkinsfile b/pipelines/deploy/druid-anomaly-detection/Jenkinsfile new file mode 100644 index 0000000000..81c60ea6b4 --- /dev/null +++ b/pipelines/deploy/druid-anomaly-detection/Jenkinsfile @@ -0,0 +1,57 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + + } + + ansiColor('xterm') { + values = lp_dp_params() + stage('get artifact') { + currentWs = sh(returnStdout: true, script: 'pwd').trim() + artifact = values.artifact_name + ":" + values.artifact_version + values.put('currentWs', currentWs) + values.put('artifact', artifact) + artifact_download(values) + } + stage('deploy artifact'){ + sh """ + unzip ${artifact} + mv anomaly_detection.tar.gz ansible + """ + ansiblePlaybook = "${currentWs}/ansible/druid_anomaly_detection.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true + archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true + } + } + summary() + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/pipelines/deploy/druid-run-report-job/Jenkinsfile b/pipelines/deploy/druid-run-report-job/Jenkinsfile index 57cae469ac..e49e7a4b8d 100644 --- a/pipelines/deploy/druid-run-report-job/Jenkinsfile +++ b/pipelines/deploy/druid-run-report-job/Jenkinsfile @@ -26,7 +26,7 @@ node() { jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() currentWs = sh(returnStdout: true, script: 'pwd').trim() ansiblePlaybook = "${currentWs}/ansible/druid-run-report.yml" - ansibleExtraArgs = "--extra-vars \"report_id=${params.report_id}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -vv" + ansibleExtraArgs = "--extra-vars \"report_id=${params.report_id} report_start_date=${params.report_start_date} report_end_date=${params.report_end_date}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -vv" values.put('currentWs', currentWs) values.put('env', envDir) values.put('module', module) diff --git a/pipelines/deploy/druid-summary-monthly-ingestion/Jenkinsfile b/pipelines/deploy/druid-summary-monthly-ingestion/Jenkinsfile new file mode 100644 index 0000000000..e4ea9da9b9 --- /dev/null +++ b/pipelines/deploy/druid-summary-monthly-ingestion/Jenkinsfile @@ -0,0 +1,51 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/druid-summary-monthly-ingestion.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -vv" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/pipelines/deploy/geolocation-db-setup/Jenkinsfile b/pipelines/deploy/geolocation-db-setup/Jenkinsfile index dd5934c746..45c7c19ac8 100644 --- a/pipelines/deploy/geolocation-db-setup/Jenkinsfile +++ b/pipelines/deploy/geolocation-db-setup/Jenkinsfile @@ -1,47 +1,61 @@ @Library('deploy-conf') _ node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - cleanWs() - checkout scm - } + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" - ansiColor('xterm') { - stage('deploy'){ - values = [:] - envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() - module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() - jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() - currentWs = sh(returnStdout: true, script: 'pwd').trim() - ansiblePlaybook = "${currentWs}/ansible/lpa_provision_geo_location_db.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('currentWs', currentWs) - values.put('env', envDir) - values.put('module', module) - values.put('jobName', jobName) - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = "SUCCESS" - currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" - } - } - } + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } - catch (err) { - currentBuild.result = "FAILURE" - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } -} + ansiColor('xterm') { + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + artifact = params.geoip_zip_csv_file_name + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('artifact', artifact) + values.put('artifact', artifact) + values.put('ignore_job_parameter', true) + } + + stage('get artifact') { + artifact_download(values) + } + stage('deploy'){ + sh "mv ${artifact} ansible/roles/provision-geo-location-db/files" + ansiblePlaybook = "${currentWs}/ansible/lpa_provision_geo_location_db.yml" + ansibleExtraArgs = "--extra-vars \"geoip_zip_csv_file_name=${artifact}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/pipelines/deploy/replay-jobs/Jenkinsfile b/pipelines/deploy/replay-jobs/Jenkinsfile index 245915d4a4..e8c247bc4f 100644 --- a/pipelines/deploy/replay-jobs/Jenkinsfile +++ b/pipelines/deploy/replay-jobs/Jenkinsfile @@ -37,7 +37,7 @@ node() { ansible_playbook_run(values) ansiblePlaybook = "${currentWs}/ansible/lpa_fetch_logs.yml" - ansibleExtraArgs = "--limit spark --tags ${params.job_type} --extra-vars \"job_id=${params.job_id} start_date=${params.start_date} end_date=${params.end_date} pause_min=${params.pause_min}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + ansibleExtraArgs = "--limit spark --tags ${params.job_type} --extra-vars \"job_id=${params.job_id} start_date=${params.start_date} end_date=${params.end_date} batch_id=${params.batch_identifier} pause_min=${params.pause_min} keyword=${params.keyword}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" values.put('ansiblePlaybook', ansiblePlaybook) values.put('ansibleExtraArgs', ansibleExtraArgs) println values diff --git a/pipelines/deploy/secor/Jenkinsfile.flink b/pipelines/deploy/secor/Jenkinsfile.flink new file mode 100644 index 0000000000..3efeaf1a3d --- /dev/null +++ b/pipelines/deploy/secor/Jenkinsfile.flink @@ -0,0 +1,65 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + + } + ansiColor('xterm') { + values = lp_dp_params() + stage('get artifact') { + currentWs = sh(returnStdout: true, script: 'pwd').trim() + artifact = values.artifact_name + ":" + values.artifact_version + values.put('currentWs', currentWs) + values.put('artifact', artifact) + artifact_download(values) + } + stage('deploy artifact'){ + sh """ + unzip ${artifact} + mkdir -p ansible/artifacts/target + mv secor-*.tar.gz ansible/artifacts/target + """ + ansiblePlaybook = "${currentWs}/ansible/lpa_secor_deploy.yml" + ansibleExtraArgs = " --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + + ansiblePlaybook = "${currentWs}/ansible/secor_telemetry_backup_deploy.yaml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true + archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true + } + } + summary() + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} + diff --git a/pipelines/deploy/spark-cluster-deploy/Jenkinsfile b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile new file mode 100644 index 0000000000..9749d35b36 --- /dev/null +++ b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile @@ -0,0 +1,53 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/spark-cluster-job-submit.yml" + ansibleExtraArgs = "--tags ${params.job_type} --extra-vars \"job_id=${params.job_id} mode=${params.mode} partitions=${params.partitions} parallelisation=${params.parallelisation} start_date=${params.start_date} end_date=${params.end_date} batch_id=${params.batch_identifier} sparkMaster=${params.sparkMaster} pause_min=${params.pause_min} selected_partitions=${params.selected_partitions}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -vvvv " + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + summary() + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel new file mode 100644 index 0000000000..4b9891d62a --- /dev/null +++ b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel @@ -0,0 +1,53 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/spark-cluster-job-submit.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass --extra-vars \"jobs=${params.jobs_to_submit} batch_size=${params.batch_size}\" --tags config-update,${params.jobs_submit_type} -vvvv " + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + summary() + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} \ No newline at end of file diff --git a/pipelines/deploy/yarn/Jenkinsfile b/pipelines/deploy/yarn/Jenkinsfile deleted file mode 100644 index 185a2da8ac..0000000000 --- a/pipelines/deploy/yarn/Jenkinsfile +++ /dev/null @@ -1,63 +0,0 @@ -@Library('deploy-conf') _ -node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - folder = new File("$WORKSPACE/.git") - if (folder.exists()) - { - println "Found .git folder. Clearing it.." - sh'git clean -fxd' - } - checkout scm - - } - - ansiColor('xterm') { - values = lp_dp_params() - stage('get artifact') { - currentWs = sh(returnStdout: true, script: 'pwd').trim() - artifact = values.artifact_name + ":" + values.artifact_version - values.put('currentWs', currentWs) - values.put('artifact', artifact) - artifact_download(values) - } - stage('deploy artifact'){ - sh """ - unzip ${artifact} - mv distribution-*.tar.gz ansible - rm -rf ansible/roles/samza-jobs/defaults/jobs - mkdir ansible/roles/samza-jobs/defaults/jobs ansible/roles/samza-jobs/defaults/allfiles - chmod 777 ansible/roles/samza-jobs/defaults/jobs - tar -xvf ansible/distribution-*.tar.gz -C ansible/roles/samza-jobs/defaults/allfiles/ - - """ - ansiblePlaybook = "${currentWs}/ansible/samza_deploy.yml" - ansibleExtraArgs = "--extra-vars \"job_names_to_kill=${params.job_names_to_deploy} job_workspace=${WORKSPACE}/ansible\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = "SUCCESS" - currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" - archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true - archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true - } - } - summary() - } - catch (err) { - currentBuild.result = "FAILURE" - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } -} - diff --git a/pipelines/ops/druid/Jenkinsfile.druidingestionalert b/pipelines/ops/druid/Jenkinsfile.druidingestionalert index 3cdee9ac38..fe62c3b805 100644 --- a/pipelines/ops/druid/Jenkinsfile.druidingestionalert +++ b/pipelines/ops/druid/Jenkinsfile.druidingestionalert @@ -1,5 +1,5 @@ @Library('deploy-conf') _ -node("ops-slave") { +node() { try { String ANSI_GREEN = "\u001B[32m" String ANSI_NORMAL = "\u001B[0m" diff --git a/pipelines/ops/druid/Jenkinsfile.druidsegmentalert b/pipelines/ops/druid/Jenkinsfile.druidsegmentalert index 73a6fcb8ca..52f3c20f17 100644 --- a/pipelines/ops/druid/Jenkinsfile.druidsegmentalert +++ b/pipelines/ops/druid/Jenkinsfile.druidsegmentalert @@ -1,5 +1,5 @@ @Library('deploy-conf') _ -node("ops-slave") { +node() { try { String ANSI_GREEN = "\u001B[32m" String ANSI_NORMAL = "\u001B[0m" diff --git a/pipelines/ops/druid/Jenkinsfile.ingestion b/pipelines/ops/druid/Jenkinsfile.ingestion index 58a42ea69d..990f3952f5 100644 --- a/pipelines/ops/druid/Jenkinsfile.ingestion +++ b/pipelines/ops/druid/Jenkinsfile.ingestion @@ -1,5 +1,5 @@ @Library('deploy-conf') _ -node("ops-slave") { +node() { try { String ANSI_GREEN = "\u001B[32m" String ANSI_NORMAL = "\u001B[0m" @@ -25,8 +25,20 @@ node("ops-slave") { module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() currentWs = sh(returnStdout: true, script: 'pwd').trim() + // populating the vars for raw-overlord and rollup-overlord cluster respectively. + task_names="${params.ingestion_task_names}"; + println(task_names); + List task_names_list = task_names.split(','); + List rollup_task_names = [], raw_task_names = []; + task_names_list.each { + if(it.startsWith("rollup")) { + rollup_task_names.add(it); + } + else + raw_task_names.add(it); + } ansiblePlaybook = "${currentWs}/ansible/druid-ingestion.yml" - ansibleExtraArgs = "--tags ${params.action} --vault-password-file /var/lib/jenkins/secrets/vault-pass -vv" + ansibleExtraArgs = "--extra-vars \"rollup_ingestion_task_names=${rollup_task_names.join(",")} raw_ingestion_task_names=${raw_task_names.join(",")}\" --tags populate_var,${params.action} --tags druid --skip-tags ml-druid --vault-password-file /var/lib/jenkins/secrets/vault-pass -vv" values.put('currentWs', currentWs) values.put('env', envDir) values.put('module', module) diff --git a/pipelines/ops/druid/Jenkinsfile.ml.ingestion b/pipelines/ops/druid/Jenkinsfile.ml.ingestion new file mode 100644 index 0000000000..ef68443f0e --- /dev/null +++ b/pipelines/ops/druid/Jenkinsfile.ml.ingestion @@ -0,0 +1,63 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + // populating the vars for raw-overlord and rollup-overlord cluster respectively. + task_names="${params.ingestion_task_names}"; + println(task_names); + List task_names_list = task_names.split(','); + List rollup_task_names = [], raw_task_names = []; + task_names_list.each { + if(it.startsWith("rollup")) { + rollup_task_names.add(it); + } + else + raw_task_names.add(it); + } + ansiblePlaybook = "${currentWs}/ansible/druid-ingestion.yml" + ansibleExtraArgs = "--extra-vars \"rollup_ingestion_task_names=${rollup_task_names.join(",")} raw_ingestion_task_names=${raw_task_names.join(",")}\" --tags populate_var,${params.action} --tags ml-druid --skip-tags druid --vault-password-file /var/lib/jenkins/secrets/vault-pass -vv" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/pipelines/provision/analytics-api/Jenkinsfile b/pipelines/provision/analytics-api/Jenkinsfile deleted file mode 100644 index 7c77425079..0000000000 --- a/pipelines/provision/analytics-api/Jenkinsfile +++ /dev/null @@ -1,53 +0,0 @@ -@Library('deploy-conf') _ -node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - folder = new File("$WORKSPACE/.git") - if (folder.exists()) - { - println "Found .git folder. Clearing it.." - sh'git clean -fxd' - } - checkout scm - - } - - ansiColor('xterm') { - stage('deploy'){ - values = [:] - envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() - module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() - jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() - currentWs = sh(returnStdout: true, script: 'pwd').trim() - ansiblePlaybook = "${currentWs}/ansible/lpa_api_provision.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('currentWs', currentWs) - values.put('env', envDir) - values.put('module', module) - values.put('jobName', jobName) - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = "SUCCESS" - currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" - } - } - summary() - } - catch (err) { - currentBuild.result = "FAILURE" - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } - -} \ No newline at end of file diff --git a/pipelines/provision/cassandra/Jenkinsfile b/pipelines/provision/cassandra/Jenkinsfile deleted file mode 100644 index 36ecc9dd9b..0000000000 --- a/pipelines/provision/cassandra/Jenkinsfile +++ /dev/null @@ -1,53 +0,0 @@ -@Library('deploy-conf') _ -node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - folder = new File("$WORKSPACE/.git") - if (folder.exists()) - { - println "Found .git folder. Clearing it.." - sh'git clean -fxd' - } - checkout scm - - } - - ansiColor('xterm') { - stage('deploy'){ - values = [:] - envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() - module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() - jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() - currentWs = sh(returnStdout: true, script: 'pwd').trim() - ansiblePlaybook = "${currentWs}/ansible/lpa_cassandra_provision.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('currentWs', currentWs) - values.put('env', envDir) - values.put('module', module) - values.put('jobName', jobName) - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = "SUCCESS" - currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" - } - } - summary() - } - catch (err) { - currentBuild.result = "FAILURE" - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } - -} \ No newline at end of file diff --git a/pipelines/provision/druid/Jenkinsfile b/pipelines/provision/druid/Jenkinsfile index 4895b6536d..a234c28d3a 100644 --- a/pipelines/provision/druid/Jenkinsfile +++ b/pipelines/provision/druid/Jenkinsfile @@ -26,7 +26,7 @@ node() { jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() currentWs = sh(returnStdout: true, script: 'pwd').trim() ansiblePlaybook = "${currentWs}/ansible/druid-provision.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" + ansibleExtraArgs = "--extra-vars \"remote=${params.remote} deploy=${params.service}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" values.put('currentWs', currentWs) values.put('env', envDir) values.put('module', module) diff --git a/pipelines/provision/graphite-exporter/Jenkinsfile b/pipelines/provision/graphite-exporter/Jenkinsfile new file mode 100644 index 0000000000..99742d268e --- /dev/null +++ b/pipelines/provision/graphite-exporter/Jenkinsfile @@ -0,0 +1,51 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + currentWs = sh(returnStdout: true, script: 'pwd').trim() + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + ansiblePlaybook = "${currentWs}/ansible/graphite-exporter.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + summary() + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/pipelines/provision/kibana/Jenkinsfile b/pipelines/provision/kibana/Jenkinsfile deleted file mode 100644 index 791994178a..0000000000 --- a/pipelines/provision/kibana/Jenkinsfile +++ /dev/null @@ -1,54 +0,0 @@ - -@Library('deploy-conf') _ -node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - folder = new File("$WORKSPACE/.git") - if (folder.exists()) - { - println "Found .git folder. Clearing it.." - sh'git clean -fxd' - } - checkout scm - - } - - ansiColor('xterm') { - stage('deploy'){ - values = [:] - envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() - module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() - jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() - currentWs = sh(returnStdout: true, script: 'pwd').trim() - ansiblePlaybook = "${currentWs}/ansible/kibana_provision.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('currentWs', currentWs) - values.put('env', envDir) - values.put('module', module) - values.put('jobName', jobName) - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = "SUCCESS" - currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" - } - } - summary() - } - catch (err) { - currentBuild.result = "FAILURE" - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } - -} \ No newline at end of file diff --git a/pipelines/provision/postgres-db-update/Jenkinsfile b/pipelines/provision/postgres-db-update/Jenkinsfile new file mode 100644 index 0000000000..3fbb046008 --- /dev/null +++ b/pipelines/provision/postgres-db-update/Jenkinsfile @@ -0,0 +1,53 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + currentWs = sh(returnStdout: true, script: 'pwd').trim() + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + ansiblePlaybook = "${currentWs}/ansible/postgres-db-update.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = "SUCCESS" + currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + summary() + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } + +} diff --git a/pipelines/provision/postgres-managed/Jenkinsfile b/pipelines/provision/postgres-managed/Jenkinsfile deleted file mode 100644 index d98eee299e..0000000000 --- a/pipelines/provision/postgres-managed/Jenkinsfile +++ /dev/null @@ -1,53 +0,0 @@ -@Library('deploy-conf') _ -node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - folder = new File("$WORKSPACE/.git") - if (folder.exists()) - { - println "Found .git folder. Clearing it.." - sh'git clean -fxd' - } - checkout scm - - } - - ansiColor('xterm') { - stage('deploy'){ - values = [:] - currentWs = sh(returnStdout: true, script: 'pwd').trim() - envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() - module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() - jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() - ansiblePlaybook = "${currentWs}/ansible/postgres-manage.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v" - values.put('currentWs', currentWs) - values.put('env', envDir) - values.put('module', module) - values.put('jobName', jobName) - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = "SUCCESS" - currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" - } - } - summary() - } - catch (err) { - currentBuild.result = "FAILURE" - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } - -} diff --git a/pipelines/provision/redis/Jenkinsfile b/pipelines/provision/redis/Jenkinsfile index 8b4f864fae..95b2a1fc7d 100644 --- a/pipelines/provision/redis/Jenkinsfile +++ b/pipelines/provision/redis/Jenkinsfile @@ -26,7 +26,7 @@ node() { jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() currentWs = sh(returnStdout: true, script: 'pwd').trim() ansiblePlaybook = "${currentWs}/ansible/redis_provision.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" + ansibleExtraArgs = "--tags ${params.redis_type} --vault-password-file /var/lib/jenkins/secrets/vault-pass" values.put('currentWs', currentWs) values.put('env', envDir) values.put('module', module) diff --git a/pipelines/provision/spark/Jenkinsfile.delete b/pipelines/provision/spark/Jenkinsfile.delete new file mode 100644 index 0000000000..93aed171cb --- /dev/null +++ b/pipelines/provision/spark/Jenkinsfile.delete @@ -0,0 +1,51 @@ +@Library('deploy-conf') _ +node('build-slave') { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + ansiColor('xterm') { + stage('Checkout') { + checkout scm + } + + stage('copy cluster creation script') { + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/azure-hdinsight-spark.provision.yml" + ansibleExtraArgs = "--extra-vars \"azure_resource_group=${params.resource_group} subscription_id=${env.subscription_id} tenant_id=${env.tenant_id} cluster_state=${params.type}\" --tags copy-script --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + } + stage('delete spark HDinsight cluster') { + storage_container=params.storage_container + withCredentials([usernamePassword(credentialsId: 'azure-service-principal', passwordVariable: 'sppass', usernameVariable: 'spuser')]) { + sh ''' + cd /tmp + ./delete-cluster.sh $spuser $sppass + ''' + } + + + } + + } + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + +} diff --git a/pipelines/provision/spark/Jenkinsfile.hdinsight b/pipelines/provision/spark/Jenkinsfile.hdinsight new file mode 100644 index 0000000000..4d8f24fd39 --- /dev/null +++ b/pipelines/provision/spark/Jenkinsfile.hdinsight @@ -0,0 +1,56 @@ + +@Library('deploy-conf') _ +node('build-slave') { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + ansiColor('xterm') { + stage('Checkout') { + checkout scm + } + + stage('copy cluster creation script') { + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/azure-hdinsight-spark.provision.yml" + ansibleExtraArgs = "--extra-vars \"azure_resource_group=${params.resource_group} location=${params.location} component_version=${params.component_version} version=${params.version} headnode_size=${params.headnode_size} workernode_size=${params.workernode_size} subnet_name=${subnet_name} vnet_name=${vnet_name} workernode_count=${workernode_count} subscription_id=${env.subscription_id} tenant_id=${env.tenant_id} cluster_state=${params.type}\" --tags copy-script,copy-creation-script --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + } + stage('create and provision spark HDinsight cluster') { + storage_container=params.storage_container + withCredentials([usernamePassword(credentialsId: 'azure-service-principal', passwordVariable: 'sppass', usernameVariable: 'spuser')]) { + sh ''' + currentws=$(pwd) + ansibleplaybook="$currentws/ansible/azure-hdinsight-spark.provision.yml" + cd /tmp + ./create-cluster.sh $spuser $sppass + vars="storage_container=$storage_container" + export ANSIBLE_HOST_KEY_CHECKING=False + ansible-playbook -i $currentws/ansible/inventory/env $ansibleplaybook --extra-vars '"'$vars'"' --tags spark-provision --vault-password-file /var/lib/jenkins/secrets/vault-pass + ''' + } + + } + + } + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + +} diff --git a/pipelines/provision/telemetry-elasticsearch/Jenkinsfile b/pipelines/provision/telemetry-elasticsearch/Jenkinsfile deleted file mode 100644 index 43b2d89719..0000000000 --- a/pipelines/provision/telemetry-elasticsearch/Jenkinsfile +++ /dev/null @@ -1,60 +0,0 @@ - -@Library('deploy-conf') _ -node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - folder = new File("$WORKSPACE/.git") - if (folder.exists()) - { - println "Found .git folder. Clearing it.." - sh'git clean -fxd' - } - checkout scm - - } - - ansiColor('xterm') { - stage('deploy'){ - values = [:] - envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() - module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() - jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() - currentWs = sh(returnStdout: true, script: 'pwd').trim() - ansiblePlaybook = "${currentWs}/ansible/es_telemetry_cluster_setup.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('currentWs', currentWs) - values.put('env', envDir) - values.put('module', module) - values.put('jobName', jobName) - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - - ansiblePlaybook = "${currentWs}/ansible/es_azure_plugin.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = "SUCCESS" - currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" - } - } - summary() - } - catch (err) { - currentBuild.result = "FAILURE" - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } -} diff --git a/pipelines/provision/yarn/Jenkinsfile b/pipelines/provision/yarn/Jenkinsfile deleted file mode 100644 index 1fd44f5df9..0000000000 --- a/pipelines/provision/yarn/Jenkinsfile +++ /dev/null @@ -1,53 +0,0 @@ - -@Library('deploy-conf') _ -node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - folder = new File("$WORKSPACE/.git") - if (folder.exists()) - { - println "Found .git folder. Clearing it.." - sh'git clean -fxd' - } - checkout scm - - } - - ansiColor('xterm') { - stage('deploy'){ - values = [:] - envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() - module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() - jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() - currentWs = sh(returnStdout: true, script: 'pwd').trim() - ansiblePlaybook = "${currentWs}/ansible/dp_yarn_provision.yml" - ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('currentWs', currentWs) - values.put('env', envDir) - values.put('module', module) - values.put('jobName', jobName) - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = "SUCCESS" - currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" - } - } - summary() - } - catch (err) { - currentBuild.result = "FAILURE" - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } -} diff --git a/pipelines/upload/Jenkinsfile b/pipelines/upload/Jenkinsfile index b9d8f45ef6..eeb7371cb1 100644 --- a/pipelines/upload/Jenkinsfile +++ b/pipelines/upload/Jenkinsfile @@ -35,7 +35,7 @@ node() { throw err } finally { - slack_notify(currentBuild.result) - email_notify() + // slack_notify(currentBuild.result) + // email_notify() } } diff --git a/schemas/1.0/common.json b/schemas/1.0/common.json deleted file mode 100644 index 57e5ed81c0..0000000000 --- a/schemas/1.0/common.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "type": "object", - "properties": { - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": ["1.0"] - }, - "gdata": { - "$ref": "http://localhost:7070/schemas/1.0/gdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 1 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 1 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - } - } -} diff --git a/schemas/1.0/common_empty_sid_uid.json b/schemas/1.0/common_empty_sid_uid.json deleted file mode 100644 index 91120d65ae..0000000000 --- a/schemas/1.0/common_empty_sid_uid.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "type": "object", - "properties": { - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "$ref": "http://localhost:7070/schemas/1.0/gdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - } - } -} diff --git a/schemas/1.0/event.json b/schemas/1.0/event.json deleted file mode 100644 index 5db20d0647..0000000000 --- a/schemas/1.0/event.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/event", - "type": "object", - "oneOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/ge_genie_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_genie_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_signup.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_session_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_session_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_launch_game.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_game_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_profile_set.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_view_progress.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_genie_update.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_game_update.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_api_call.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_genie_resume.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_interact.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_interrupt.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_error.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_misc.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/oe_assess.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/oe_level_set.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/oe_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/oe_learn.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/oe_earn.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/oe_interact.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/oe_interrupt.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/oe_misc.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/oe_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_create_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/1.0/ge_update_profile.json" - } - ] -} diff --git a/schemas/1.0/gdata.json b/schemas/1.0/gdata.json deleted file mode 100644 index 1b603a1b7b..0000000000 --- a/schemas/1.0/gdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/schemas/1.0/ge_api_call.json b/schemas/1.0/ge_api_call.json deleted file mode 100644 index 768c587382..0000000000 --- a/schemas/1.0/ge_api_call.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_api_call", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_API_CALL" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "api", - "msgid", - "isize", - "osize", - "length", - "err" - ], - "properties": { - "api": { - "type": "string", - "minLength": 1 - }, - "msgid": { - "type": "string", - "minLength": 1 - }, - "isize": { - "type": "number", - "minimum": 0 - }, - "osize": { - "type": "number", - "minimum": 0 - }, - "length": { - "type": "number", - "minimum": 0 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_create_profile.json b/schemas/1.0/ge_create_profile.json deleted file mode 100644 index 45a90134f2..0000000000 --- a/schemas/1.0/ge_create_profile.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_CREATE_PROFILE" - ] - }, - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "loc": { - "type": "string" - }, - "uid": { - "type": "string" - }, - "handle": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "age": { - "type": "number" - }, - "standard": { - "type": "number" - }, - "language": { - "type": "string" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/1.0/ge_create_user.json b/schemas/1.0/ge_create_user.json deleted file mode 100644 index 58ac62cef1..0000000000 --- a/schemas/1.0/ge_create_user.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_CREATE_USER" - ] - }, - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "loc": { - "type": "string" - }, - "uid": { - "type": "string" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/1.0/ge_error.json b/schemas/1.0/ge_error.json deleted file mode 100644 index efdb81e393..0000000000 --- a/schemas/1.0/ge_error.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id", - "err", - "eventId", - "data" - ], - "properties": { - "err": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "SYSTEM","GENIE","GAME","GENIESERVICES" - ] - }, - "id": { - "type": "string" - }, - "eventId": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_export.json b/schemas/1.0/ge_export.json deleted file mode 100644 index e1d78df0cc..0000000000 --- a/schemas/1.0/ge_export.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_EXPORT" - ] - }, - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "telemetryEvents": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/1.0/ge_game_end.json b/schemas/1.0/ge_game_end.json deleted file mode 100644 index be7b0a868a..0000000000 --- a/schemas/1.0/ge_game_end.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_game_end", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GAME_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "gid", - "length", - "tmsize", - "err" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "length": { - "type": "number", - "minimum": 0 - }, - "tmsize": { - "type": "number", - "minimum": 0 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_game_update.json b/schemas/1.0/ge_game_update.json deleted file mode 100644 index e7a739c1d0..0000000000 --- a/schemas/1.0/ge_game_update.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_game_update", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GAME_UPDATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "mode", - "ver", - "size", - "err", - "gid" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "mode":{ - "enum": [ - "MDATA", - "WIFI", - "SDCARD", - "BT", - "OTHER" - ] - }, - "ver": { - "type": "string", - "minLength": 1 - }, - "size": { - "type": "number", - "minimum": -1 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_genie_end.json b/schemas/1.0/ge_genie_end.json deleted file mode 100644 index c6fd9f367a..0000000000 --- a/schemas/1.0/ge_genie_end.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_end", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_genie_resume.json b/schemas/1.0/ge_genie_resume.json deleted file mode 100644 index 37b6640d67..0000000000 --- a/schemas/1.0/ge_genie_resume.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_resume", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_RESUME" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "loc" - ], - "properties": { - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_genie_start.json b/schemas/1.0/ge_genie_start.json deleted file mode 100644 index de40a918d7..0000000000 --- a/schemas/1.0/ge_genie_start.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_start", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "dspec", - "loc" - ], - "properties": { - "dspec": { - "id": "http://api.ekstep.org/telemetry/edata/dspec", - "type": "object", - "additionalProperties": false, - "required": [ - "os", - "make", - "dname", - "dlocname", - "mem", - "id", - "idisk", - "edisk", - "scrn", - "camera", - "cpu", - "sims", - "cap" - ], - "properties": { - "os": { - "type": "string", - "minLength": 1 - }, - "dname": { - "type": "string" - }, - "dlocname": { - "type": "string" - }, - "make": { - "type": "string", - "minLength": 1 - }, - "id": { - "type": "string" - }, - "idisk": { - "type": "number", - "minimum": -1 - }, - "edisk": { - "type": "number", - "minimum": -1 - }, - "mem": { - "type": "number", - "minimum": -1 - }, - "sims":{ - "type": "number", - "minimum": -1 - }, - "scrn": { - "type": "number", - "minimum": -1 - }, - "camera": { - "type": "string" - }, - "cpu": { - "type": "string" - }, - "cap": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_genie_update.json b/schemas/1.0/ge_genie_update.json deleted file mode 100644 index 209c139258..0000000000 --- a/schemas/1.0/ge_genie_update.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_update", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_UPDATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "mode", - "ver", - "size", - "err" - ], - "properties": { - "mode":{ - "enum": [ - "MDATA", - "WIFI" - ] - }, - "ver": { - "type": "string", - "minLength": 1 - }, - "size": { - "type": "number", - "minimum": -1 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_import.json b/schemas/1.0/ge_import.json deleted file mode 100644 index cbea1af453..0000000000 --- a/schemas/1.0/ge_import.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_IMPORT" - ] - }, - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "did": { - "type": "string" - }, - "telemetryEvents": { - "type": "number" - }, - "userProfiles": { - "type": "number" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/1.0/ge_interact.json b/schemas/1.0/ge_interact.json deleted file mode 100644 index 0b0c66e399..0000000000 --- a/schemas/1.0/ge_interact.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_interact", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "extype", - "id", - "uri" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "TOUCH","DRAG","DROP","PINCH","ZOOM","SHAKE","ROTATE","SPEAK","LISTEN","WRITE","DRAW","START","END","CHOOSE","ACTIVATE","OTHER" - ] - }, - "extype": { - "type": "string" - }, - "id": { - "type": "string" - }, - "uri": { - "type": "string" - } - } - }, - "ext": { - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_interrupt.json b/schemas/1.0/ge_interrupt.json deleted file mode 100644 index 7141cacf68..0000000000 --- a/schemas/1.0/ge_interrupt.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_interrupt", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND" - ] - }, - "id": { - "type": "string" - } - } - }, - "ext": { - "type": "object" - } - } - } - } - } - ] -} diff --git a/schemas/1.0/ge_launch_game.json b/schemas/1.0/ge_launch_game.json deleted file mode 100644 index 281dce4d12..0000000000 --- a/schemas/1.0/ge_launch_game.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_launch_game", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_LAUNCH_GAME" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "gid", - "tmschm", - "err" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "tmschm": { - "enum": [ - "NONE", - "INTENT", - "FILE" - ] - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_misc.json b/schemas/1.0/ge_misc.json deleted file mode 100644 index eaecb27168..0000000000 --- a/schemas/1.0/ge_misc.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_misc", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_MISC" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "ext" - ], - "properties": { - "ext": { - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_partner_data.json b/schemas/1.0/ge_partner_data.json deleted file mode 100644 index 77faff5e5d..0000000000 --- a/schemas/1.0/ge_partner_data.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_PARTNER_DATA" - ] - }, - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "partnerid": { - "type": "string" - }, - "publickeyid": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/1.0/ge_profile_set.json b/schemas/1.0/ge_profile_set.json deleted file mode 100644 index 53e9be281b..0000000000 --- a/schemas/1.0/ge_profile_set.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_profile_set", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_PROFILE_SET" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "ueksid", - "attrs" - ], - "properties": { - "ueksid": { - "type": "string", - "minLength": 1 - }, - "attrs": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_register_partner.json b/schemas/1.0/ge_register_partner.json deleted file mode 100644 index 9d89ad8785..0000000000 --- a/schemas/1.0/ge_register_partner.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_REGISTER_PARTNER" - ] - }, - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "partnerid": { - "type": "string" - }, - "publickey": { - "type": "string" - }, - "publickeyid": { - "type": "string" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/1.0/ge_service_api_call.json b/schemas/1.0/ge_service_api_call.json deleted file mode 100644 index 76f301f308..0000000000 --- a/schemas/1.0/ge_service_api_call.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SERVICE_API_CALL" - ] - }, - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "message": { - "type": "string" - }, - "service": { - "type": "string" - }, - "result": { - "type": "object", - "additionalProperties": true - }, - "method": { - "type": "string" - }, - "errorMessages": { - "type": "array", - "items": { - "type": "string" - } - }, - "error": { - "type": "string" - }, - "status": { - "type": "string" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/1.0/ge_session_end.json b/schemas/1.0/ge_session_end.json deleted file mode 100644 index ffce9c5942..0000000000 --- a/schemas/1.0/ge_session_end.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_session_end", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SESSION_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/1.0/ge_session_start.json b/schemas/1.0/ge_session_start.json deleted file mode 100644 index 5cbb89d3ae..0000000000 --- a/schemas/1.0/ge_session_start.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_session_start", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SESSION_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "loc" - ], - "properties": { - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/1.0/ge_signup.json b/schemas/1.0/ge_signup.json deleted file mode 100644 index 238ad22db9..0000000000 --- a/schemas/1.0/ge_signup.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_signup", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SIGNUP" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "ueksid", - "utype", - "loc", - "err" - ], - "properties": { - "ueksid": { - "type": "string", - "minLength": 1 - }, - "loc": { - "type": "string" - }, - "err": { - "type": "string" - }, - "utype": { - "enum": [ - "CHILD", - "FACILITATOR" - ] - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/ge_start_partner_session.json b/schemas/1.0/ge_start_partner_session.json deleted file mode 100644 index 1e6b9ff3cf..0000000000 --- a/schemas/1.0/ge_start_partner_session.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_START_PARTNER_SESSION" - ] - }, - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": {} - } - } - } - } -} \ No newline at end of file diff --git a/schemas/1.0/ge_stop_partner_session.json b/schemas/1.0/ge_stop_partner_session.json deleted file mode 100644 index 6edfb65357..0000000000 --- a/schemas/1.0/ge_stop_partner_session.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"http://api.ekstep.org/telemetry/ge_error","type":"object","required":["eid","ts","ver","gdata","sid","uid","did","edata"],"properties":{"eid":{"id":"http://api.ekstep.org/telemetry/eid","enum":["GE_STOP_PARTNER_SESSION"]},"ts":{"id":"http://api.ekstep.org/telemetry/ts","type":"string","format":"date-time"},"ver":{"id":"http://api.ekstep.org/telemetry/ver","type":"string","minLength":1},"gdata":{"type":"object","properties":{"id":{"id":"http://api.ekstep.org/telemetry/gdata/id","type":"string"},"ver":{"id":"http://api.ekstep.org/telemetry/gdata/ver","type":"string"}},"additionalProperties":false,"required":["id","ver"]},"sid":{"id":"http://api.ekstep.org/telemetry/sid","type":"string","minLength":0},"uid":{"id":"http://api.ekstep.org/telemetry/uid","type":"string","minLength":0},"did":{"id":"http://api.ekstep.org/telemetry/did","type":"string","minLength":1},"tags":{"type":"array","items":{"type":"object"}},"edata":{"id":"http://api.ekstep.org/telemetry/edata","type":"object","additionalProperties":false,"required":["eks"],"properties":{"eks":{"id":"http://api.ekstep.org/telemetry/edata/eks","type":"object","additionalProperties":true,"properties":{"length":{"type":"number"}}}}}}} \ No newline at end of file diff --git a/schemas/1.0/ge_update_profile.json b/schemas/1.0/ge_update_profile.json deleted file mode 100644 index 40f90da856..0000000000 --- a/schemas/1.0/ge_update_profile.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_UPDATE_PROFILE" - ] - }, - "ts": { - "id": "http://api.ekstep.org/telemetry/ts", - "type": "string", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "uid": { - "type": "string" - }, - "handle": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "age": { - "type": "number" - }, - "standard": { - "type": "number" - }, - "language": { - "type": "string" - }, - "day": { - "type": "number" - }, - "month": { - "type": "number" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/1.0/ge_view_progress.json b/schemas/1.0/ge_view_progress.json deleted file mode 100644 index 14450c3632..0000000000 --- a/schemas/1.0/ge_view_progress.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_view_progress", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_VIEW_PROGRESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "ueksid" - ], - "properties": { - "ueksid": { - "type": "string", - "minLength": 1 - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/oe_assess.json b/schemas/1.0/oe_assess.json deleted file mode 100644 index 0d5445173f..0000000000 --- a/schemas/1.0/oe_assess.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/oe_assess", - "type": "object", - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ASSESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "subj", - "mc", - "qid", - "qtype", - "qlevel", - "pass", - "score", - "maxscore" - ], - "properties": { - "subj": { - "id": "http://api.ekstep.org/telemetry/edata/eks/subj", - "enum": [ - "NUM", - "LIT", - "COG" - ] - }, - "mc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mc", - "type": "array", - "items": { - "type": "string" - } - }, - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qid", - "type": "string" - }, - "qtype": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qtype", - "type": "string" - }, - "qlevel": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qlevel", - "type": "string" - }, - "pass": { - "id": "http://api.ekstep.org/telemetry/edata/eks/pass", - "enum": [ - "Yes", - "No" - ] - }, - "mmc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mmc", - "type": "array", - "items": { - "type": "string" - } - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/eks/score", - "type": "number", - "minimum": 0 - }, - "maxscore": { - "id": "http://api.ekstep.org/telemetry/edata/eks/maxscore", - "type": "number", - "minimum": 0 - }, - "res": { - "id": "http://api.ekstep.org/telemetry/edata/eks/res", - "type": "array", - "items": { - "type": "string" - } - }, - "exres": { - "id": "http://api.ekstep.org/telemetry/edata/eks/exres", - "type": "array", - "items": { - "type": "string" - } - }, - "length": { - "id": "http://api.ekstep.org/telemetry/edata/eks/length", - "type": "number", - "minimum": 0 - }, - "exlength": { - "id": "http://api.ekstep.org/telemetry/edata/eks/exlength", - "type": "number", - "minimum": 0 - }, - "atmpts": { - "id": "http://api.ekstep.org/telemetry/edata/eks/atmpts", - "type": "number", - "multipleOf": 1, - "minimum": 0 - }, - "failedatmpts": { - "id": "http://api.ekstep.org/telemetry/edata/eks/failedatmpts", - "type": "number", - "minimum": 0 - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/eks/uri", - "type": "string" - } - } - }, - "ext": { - "id": "http://api.ekstep.org/telemetry/edata/ext", - "type": "object" - } - } - } - } - } - ], - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ] -} diff --git a/schemas/1.0/oe_earn.json b/schemas/1.0/oe_earn.json deleted file mode 100644 index 6d8c2a04e5..0000000000 --- a/schemas/1.0/oe_earn.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_earn", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_EARN" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object" - }, - "ext": { - "id": "http://api.ekstep.org/telemetry/edata/ext", - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/oe_end.json b/schemas/1.0/oe_end.json deleted file mode 100644 index 2902a70fab..0000000000 --- a/schemas/1.0/oe_end.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks", - "ext" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length":{ - "type": "number", - "minimum": 0 - } - } - }, - "ext": { - "id": "http://api.ekstep.org/telemetry/edata/ext", - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/oe_interact.json b/schemas/1.0/oe_interact.json deleted file mode 100644 index 210a4d6b96..0000000000 --- a/schemas/1.0/oe_interact.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_interact", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "extype", - "id", - "uri" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "TOUCH", - "DRAG", - "DROP", - "PINCH", - "ZOOM", - "SHAKE", - "ROTATE", - "SPEAK", - "LISTEN", - "WRITE", - "DRAW", - "START", - "END", - "CHOOSE", - "ACTIVATE", - "OTHER" - ] - }, - "extype": { - "type": "string" - }, - "id": { - "type": "string" - }, - "uri": { - "type": "string" - } - } - }, - "ext": { - "id": "http://api.ekstep.org/telemetry/edata/ext", - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/oe_interrupt.json b/schemas/1.0/oe_interrupt.json deleted file mode 100644 index c909070c8c..0000000000 --- a/schemas/1.0/oe_interrupt.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_interrupt", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER" - ] - }, - "id": { - "type": "string" - } - } - }, - "ext": { - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/oe_learn.json b/schemas/1.0/oe_learn.json deleted file mode 100644 index 348d0d2e73..0000000000 --- a/schemas/1.0/oe_learn.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_learn", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_LEARN" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "topics" - ], - "properties": { - "topics": { - "id": "http://api.ekstep.org/telemetry/edata/eks/topics", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "mc", - "skill", - "methods" - ], - "properties": { - "mc": { - "type": "string" - }, - "skill": { - "type": "string" - }, - "methods": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": [ - "PLAY", - "ANSWER", - "WRITE", - "SPEAK", - "OTHER" - ] - } - } - } - } - } - } - }, - "ext": { - "id": "http://api.ekstep.org/telemetry/edata/ext", - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/oe_level_set.json b/schemas/1.0/oe_level_set.json deleted file mode 100644 index e575d00d24..0000000000 --- a/schemas/1.0/oe_level_set.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_level_set", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_LEVEL_SET" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "category", - "current", - "max" - ], - "properties": { - "category": { - "id": "http://api.ekstep.org/telemetry/edata/eks/category", - "type": "string", - "minLength": 1 - }, - "current": { - "id": "http://api.ekstep.org/telemetry/edata/eks/current", - "type": "string", - "minLength": 1 - }, - "max": { - "id": "http://api.ekstep.org/telemetry/edata/eks/max", - "type": "string", - "minLength": 1 - } - } - }, - "ext": { - "id": "http://api.ekstep.org/telemetry/edata/ext", - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/oe_misc.json b/schemas/1.0/oe_misc.json deleted file mode 100644 index 46d6a95f0a..0000000000 --- a/schemas/1.0/oe_misc.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_misc", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_MISC" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "ext" - ], - "properties": { - "ext": { - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/1.0/oe_start.json b/schemas/1.0/oe_start.json deleted file mode 100644 index ec26c93c57..0000000000 --- a/schemas/1.0/oe_start.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_start", - "type": "object", - "required": [ - "eid", - "ts", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/1.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks", - "ext" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false - }, - "ext": { - "id": "http://api.ekstep.org/telemetry/edata/ext", - "type": "object" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.0/cdata.json b/schemas/2.0/cdata.json deleted file mode 100644 index 4c7261b025..0000000000 --- a/schemas/2.0/cdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/cdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/cdata/id", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ "type", "id"] - } -} diff --git a/schemas/2.0/common.json b/schemas/2.0/common.json deleted file mode 100644 index 9abb81be4f..0000000000 --- a/schemas/2.0/common.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": ["2.0"] - }, - "gdata": { - "$ref": "http://localhost:7070/schemas/2.0/gdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 1 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 1 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "cdata": { - "$ref": "http://localhost:7070/schemas/2.0/cdata.json" - } - } -} diff --git a/schemas/2.0/common_empty_sid_uid.json b/schemas/2.0/common_empty_sid_uid.json deleted file mode 100644 index 7b3c65966a..0000000000 --- a/schemas/2.0/common_empty_sid_uid.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "$ref": "http://localhost:7070/schemas/2.0/gdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "cdata": { - "$ref": "http://localhost:7070/schemas/2.0/cdata.json" - } - } -} diff --git a/schemas/2.0/event.json b/schemas/2.0/event.json deleted file mode 100644 index 2bdbae4479..0000000000 --- a/schemas/2.0/event.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/event", - "type": "object", - "oneOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/ge_genie_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_genie_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_session_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_session_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_launch_game.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_game_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_profile_set.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_partner_data.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_register_partner.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_view_progress.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_genie_update.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_game_update.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_api_call.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_service_api_call.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_genie_resume.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_interact.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_interrupt.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_error.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_feedback.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_assess.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_level_set.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_learn.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_earn.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_error.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_interact.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_interrupt.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_item_response.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_navigate.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_feedback.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_create_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_update_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_transfer.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/ge_delete_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.0/oe_xapi.json" - } - ] -} diff --git a/schemas/2.0/gdata.json b/schemas/2.0/gdata.json deleted file mode 100644 index 1b603a1b7b..0000000000 --- a/schemas/2.0/gdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/schemas/2.0/ge_api_call.json b/schemas/2.0/ge_api_call.json deleted file mode 100644 index a16544fa84..0000000000 --- a/schemas/2.0/ge_api_call.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_api_call", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_API_CALL" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "api", - "msgid", - "isize", - "osize", - "length", - "err" - ], - "properties": { - "api": { - "type": "string", - "minLength": 1 - }, - "msgid": { - "type": "string", - "minLength": 1 - }, - "isize": { - "type": "number", - "minimum": 0 - }, - "osize": { - "type": "number", - "minimum": 0 - }, - "length": { - "type": "number", - "minimum": 0 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_create_profile.json b/schemas/2.0/ge_create_profile.json deleted file mode 100644 index 89272cf4ab..0000000000 --- a/schemas/2.0/ge_create_profile.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_CREATE_PROFILE" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "loc": { - "type": "string" - }, - "uid": { - "type": "string" - }, - "handle": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "age": { - "type": "number" - }, - "standard": { - "type": "number" - }, - "language": { - "type": "string" - }, - "day": { - "type": "number" - }, - "month": { - "type": "number" - }, - "board": { - "type": "string" - }, - "medium": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_create_user.json b/schemas/2.0/ge_create_user.json deleted file mode 100644 index 2f0391439c..0000000000 --- a/schemas/2.0/ge_create_user.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_CREATE_USER" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "loc": { - "type": "string" - }, - "uid": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_delete_profile.json b/schemas/2.0/ge_delete_profile.json deleted file mode 100644 index 2969259e48..0000000000 --- a/schemas/2.0/ge_delete_profile.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_delete_profile", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_DELETE_PROFILE" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "uid": { - "type": "string" - }, - "duration": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_error.json b/schemas/2.0/ge_error.json deleted file mode 100644 index d97651605c..0000000000 --- a/schemas/2.0/ge_error.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id", - "err", - "eventId", - "data" - ], - "properties": { - "err": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "SYSTEM","GENIE","GAME","GENIESERVICES" - ] - }, - "id": { - "type": "string" - }, - "eventId": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_export.json b/schemas/2.0/ge_export.json deleted file mode 100644 index 79ad905c30..0000000000 --- a/schemas/2.0/ge_export.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_EXPORT" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "telemetryEvents": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_feedback.json b/schemas/2.0/ge_feedback.json deleted file mode 100644 index 05b438fb1b..0000000000 --- a/schemas/2.0/ge_feedback.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_FEEDBACK" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "rating", - "context", - "comments" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/type", - "type": "string", - "enum": [ - "FLAG", - "SURVEY", - "RATING" - ] - }, - "rating": { - "id": "http://api.ekstep.org/telemetry/edata/eks/rating", - "type": "number" - }, - "context": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id", - "stageid" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context/id", - "type": "string" - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context/stageid", - "type": "string" - } - } - }, - "comments": { - "id": "http://api.ekstep.org/telemetry/edata/eks/comments", - "type": "string" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_game_end.json b/schemas/2.0/ge_game_end.json deleted file mode 100644 index 8b132c99af..0000000000 --- a/schemas/2.0/ge_game_end.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_game_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GAME_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "gid", - "length", - "tmsize", - "err" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "length": { - "type": "number", - "minimum": 0 - }, - "tmsize": { - "type": "number", - "minimum": 0 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_game_update.json b/schemas/2.0/ge_game_update.json deleted file mode 100644 index deabbb1215..0000000000 --- a/schemas/2.0/ge_game_update.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_game_update", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GAME_UPDATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "mode", - "ver", - "size", - "err", - "gid" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "mode":{ - "enum": [ - "MDATA", - "WIFI", - "SDCARD", - "BT", - "OTHER" - ] - }, - "ver": { - "type": "string", - "minLength": 1 - }, - "size": { - "type": "number", - "minimum": -1 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_genie_end.json b/schemas/2.0/ge_genie_end.json deleted file mode 100644 index 7b25a9f634..0000000000 --- a/schemas/2.0/ge_genie_end.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_genie_resume.json b/schemas/2.0/ge_genie_resume.json deleted file mode 100644 index 638e7fd260..0000000000 --- a/schemas/2.0/ge_genie_resume.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_resume", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_RESUME" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "loc" - ], - "properties": { - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_genie_start.json b/schemas/2.0/ge_genie_start.json deleted file mode 100644 index aa89ad4a6f..0000000000 --- a/schemas/2.0/ge_genie_start.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "dspec", - "loc" - ], - "properties": { - "dspec": { - "id": "http://api.ekstep.org/telemetry/edata/dspec", - "type": "object", - "additionalProperties": false, - "required": [ - "os", - "make", - "dname", - "dlocname", - "mem", - "id", - "idisk", - "edisk", - "scrn", - "camera", - "cpu", - "sims", - "cap" - ], - "properties": { - "os": { - "type": "string", - "minLength": 1 - }, - "dname": { - "type": "string" - }, - "dlocname": { - "type": "string" - }, - "make": { - "type": "string", - "minLength": 1 - }, - "id": { - "type": "string" - }, - "idisk": { - "type": "number", - "minimum": -1 - }, - "edisk": { - "type": "number", - "minimum": -1 - }, - "mem": { - "type": "number", - "minimum": -1 - }, - "sims": { - "type": "number", - "minimum": -1 - }, - "scrn": { - "type": "number", - "minimum": -1 - }, - "camera": { - "type": "string" - }, - "cpu": { - "type": "string" - }, - "cap": { - "type": "array", - "items": { - "type": "string" - } - }, - "mdata": { - "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/id", - "type": "string" - } - } - } - } - }, - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_genie_update.json b/schemas/2.0/ge_genie_update.json deleted file mode 100644 index fb8c249ffd..0000000000 --- a/schemas/2.0/ge_genie_update.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_update", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_UPDATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "properties": { - "mode": { - "enum": [ - "MDATA", - "WIFI" - ] - }, - "ver": { - "type": "string", - "minLength": 1 - }, - "size": { - "type": "number", - "minimum": -1 - }, - "err": { - "type": "string" - }, - "referrer": { - "id": "http://api.ekstep.org/telemetry/edata/eks/referrer", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_import.json b/schemas/2.0/ge_import.json deleted file mode 100644 index 3bc7ea0e5e..0000000000 --- a/schemas/2.0/ge_import.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_IMPORT" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "did": { - "type": "string" - }, - "telemetryEvents": { - "type": "number" - }, - "userProfiles": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_interact.json b/schemas/2.0/ge_interact.json deleted file mode 100644 index 3cfd851489..0000000000 --- a/schemas/2.0/ge_interact.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_interact", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "stageid", - "type", - "subtype", - "extype", - "pos", - "values", - "id", - "tid", - "uri" - ], - "properties": { - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/stageid", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string", - "enum": [ - "TOUCH", - "SHOW", - "HIDE", - "DRAG", - "DROP", - "PINCH", - "ZOOM", - "SHAKE", - "ROTATE", - "SPEAK", - "LISTEN", - "WRITE", - "DRAW", - "START", - "END", - "CHOOSE", - "ACTIVATE", - "OTHER" - ] - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "extype": { - "id": "http://api.ekstep.org/telemetry/edata/extype", - "type": "string" - }, - "pos": { - "id": "http://api.ekstep.org/telemetry/edata/pos", - "type": "array" - }, - "values": { - "id": "http://api.ekstep.org/telemetry/edata/values", - "type": "array", - "items": { - "type": "object" - } - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/id", - "type": "string" - }, - "tid": { - "id": "http://api.ekstep.org/telemetry/edata/tid", - "type": "string" - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/uri", - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_interrupt.json b/schemas/2.0/ge_interrupt.json deleted file mode 100644 index 4d2943cadc..0000000000 --- a/schemas/2.0/ge_interrupt.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_interrupt", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND" - ] - }, - "id": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_launch_game.json b/schemas/2.0/ge_launch_game.json deleted file mode 100644 index 338c5844e5..0000000000 --- a/schemas/2.0/ge_launch_game.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_launch_game", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_LAUNCH_GAME" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "gid", - "tmschm", - "err" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "tmschm": { - "enum": [ - "NONE", - "INTENT", - "FILE" - ] - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_misc.json b/schemas/2.0/ge_misc.json deleted file mode 100644 index ef37342a20..0000000000 --- a/schemas/2.0/ge_misc.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_misc", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_MISC" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false - } - } - } - ] -} diff --git a/schemas/2.0/ge_partner_data.json b/schemas/2.0/ge_partner_data.json deleted file mode 100644 index ea0a5541c9..0000000000 --- a/schemas/2.0/ge_partner_data.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_PARTNER_DATA" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "partnerid": { - "type": "string" - }, - "publickeyid": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_profile_set.json b/schemas/2.0/ge_profile_set.json deleted file mode 100644 index 552270f10e..0000000000 --- a/schemas/2.0/ge_profile_set.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_profile_set", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_PROFILE_SET" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "ueksid", - "attrs" - ], - "properties": { - "ueksid": { - "type": "string", - "minLength": 1 - }, - "attrs": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_register_partner.json b/schemas/2.0/ge_register_partner.json deleted file mode 100644 index d6a74b92b9..0000000000 --- a/schemas/2.0/ge_register_partner.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_REGISTER_PARTNER" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "partnerid": { - "type": "string" - }, - "publickey": { - "type": "string" - }, - "publickeyid": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_service_api_call.json b/schemas/2.0/ge_service_api_call.json deleted file mode 100644 index a316c5cd52..0000000000 --- a/schemas/2.0/ge_service_api_call.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SERVICE_API_CALL" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.0" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "message": { - "type": "string" - }, - "service": { - "type": "string" - }, - "result": { - "type": "object", - "additionalProperties": true - }, - "method": { - "type": "string" - }, - "errorMessages": { - "type": "array", - "items": { - "type": "string" - } - }, - "error": { - "type": "string" - }, - "status": { - "type": "string" - }, - "mode": { - "type": "string", - "enum": [ - "WIFI", - "MDATA", - "LOCAL", - "" - ] - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_session_end.json b/schemas/2.0/ge_session_end.json deleted file mode 100644 index b7ddec5fa4..0000000000 --- a/schemas/2.0/ge_session_end.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_session_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SESSION_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_session_start.json b/schemas/2.0/ge_session_start.json deleted file mode 100644 index 15a92d1bd3..0000000000 --- a/schemas/2.0/ge_session_start.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_session_start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SESSION_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "loc" - ], - "properties": { - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/ge_start_partner_session.json b/schemas/2.0/ge_start_partner_session.json deleted file mode 100644 index 4a43f0d066..0000000000 --- a/schemas/2.0/ge_start_partner_session.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_START_PARTNER_SESSION" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": {} - } - } - } - } -} diff --git a/schemas/2.0/ge_stop_partner_session.json b/schemas/2.0/ge_stop_partner_session.json deleted file mode 100644 index c35b669642..0000000000 --- a/schemas/2.0/ge_stop_partner_session.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_STOP_PARTNER_SESSION" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "length": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_transfer.json b/schemas/2.0/ge_transfer.json deleted file mode 100644 index 1aafc0a6d7..0000000000 --- a/schemas/2.0/ge_transfer.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_transfer", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_TRANSFER" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "direction": { - "enum": [ - "EXPORT", - "IMPORT" - ] - }, - "datatype": { - "enum": [ - "TELEMETRY", - "CONTENT", - "PROFILE", - "EXPLODEDCONTENT" - ] - }, - "count": { - "type": "number" - }, - "size": { - "type": "number" - }, - "contents": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": true, - "properties": { - "identifier": { - "type": "string" - }, - "origin": { - "type": "string", - "minLength": 1 - }, - "transferId": { - "type": "string", - "minLength": 1 - }, - "transferCount": { - "type": "number" - }, - "pkgVersion": { - "type": "number" - } - } - } - } - }, - "required": [ - "direction", - "datatype", - "count" - ] - } - } - } - } -} diff --git a/schemas/2.0/ge_update_profile.json b/schemas/2.0/ge_update_profile.json deleted file mode 100644 index 6fa1c81aef..0000000000 --- a/schemas/2.0/ge_update_profile.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_UPDATE_PROFILE" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "uid": { - "type": "string" - }, - "handle": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "age": { - "type": "number" - }, - "standard": { - "type": "number" - }, - "language": { - "type": "string" - }, - "day": { - "type": "number" - }, - "month": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.0/ge_view_progress.json b/schemas/2.0/ge_view_progress.json deleted file mode 100644 index f1edff46ad..0000000000 --- a/schemas/2.0/ge_view_progress.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_view_progress", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_VIEW_PROGRESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "ueksid" - ], - "properties": { - "ueksid": { - "type": "string", - "minLength": 1 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_assess.json b/schemas/2.0/oe_assess.json deleted file mode 100644 index cb682a3f94..0000000000 --- a/schemas/2.0/oe_assess.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/oe_assess", - "type": "object", - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ASSESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "qid", - "pass", - "score", - "resvalues", - "length", - "exlength", - "params", - "uri", - "qindex" - ], - "properties": { - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qid", - "type": "string" - }, - "pass": { - "id": "http://api.ekstep.org/telemetry/edata/eks/pass", - "enum": [ - "Yes", - "No" - ] - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/eks/score", - "type": "number", - "minimum": 0 - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - }, - "length": { - "id": "http://api.ekstep.org/telemetry/edata/eks/length", - "type": "number", - "minimum": 0 - }, - "exlength": { - "id": "http://api.ekstep.org/telemetry/edata/eks/exlength", - "type": "number", - "minimum": 0 - }, - "params": { - "id": "http://api.ekstep.org/telemetry/edata/eks/params", - "type": "array", - "items": { - "type": "object" - } - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/eks/uri", - "type": "string" - }, - "qindex": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qindex", - "type": "number", - "minimum": 0 - }, - "qtitle": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle", - "type": "string" - }, - "qdesc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle", - "type": "string" - }, - "maxscore": { - "id": "http://api.ekstep.org/telemetry/edata/eks/maxscore", - "type": "number", - "minimum": 0 - }, - "mc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mc", - "type": "array", - "items": { - "type": "string" - } - }, - "mmc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mmc", - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - ], - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ] -} diff --git a/schemas/2.0/oe_earn.json b/schemas/2.0/oe_earn.json deleted file mode 100644 index 06a78b5e2c..0000000000 --- a/schemas/2.0/oe_earn.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_earn", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_EARN" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "required": [ - "type", - "points" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "MONEY","GEMS","POINTS" - ] - }, - "points": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_end.json b/schemas/2.0/oe_end.json deleted file mode 100644 index 6a087bec32..0000000000 --- a/schemas/2.0/oe_end.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length":{ - "type": "number", - "minimum": 0 - }, - "stageid":{ - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_error.json b/schemas/2.0/oe_error.json deleted file mode 100644 index e96ec14568..0000000000 --- a/schemas/2.0/oe_error.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_error", - "type": "object", - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "env", - "type", - "stageid", - "objecttype", - "objectid", - "err", - "action", - "data", - "severity" - ], - "properties": { - "env": { - "id": "http://api.ekstep.org/telemetry/edata/eks/env", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/type", - "enum": [ - "SYSTEM", - "API", - "PLUGIN", - "CONTENT", - "ASSET", - "OTHER" - ] - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/stageid", - "type": "string" - }, - "objecttype": { - "id": "http://api.ekstep.org/telemetry/edata/objecttype", - "type": "string" - }, - "objectid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/objectid", - "type": "string" - }, - "err": { - "id": "http://api.ekstep.org/telemetry/edata/eks/err", - "type": "string" - }, - "action": { - "id": "http://api.ekstep.org/telemetry/edata/eks/action", - "type": "string" - }, - "data": { - "id": "http://api.ekstep.org/telemetry/edata/eks/data", - "type": "string" - }, - "severity": { - "id": "http://api.ekstep.org/telemetry/edata/eks/severity", - "type": "string" - } - } - } - } - } - } - } - ], - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata" - ] -} diff --git a/schemas/2.0/oe_feedback.json b/schemas/2.0/oe_feedback.json deleted file mode 100644 index 26e42c1386..0000000000 --- a/schemas/2.0/oe_feedback.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_FEEDBACK" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "rating", - "context", - "comments", - "resvalues" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/type", - "type": "string", - "enum": [ - "FLAG", "SURVEY", "RATING" - ] - }, - "rating": { - "id": "http://api.ekstep.org/telemetry/edata/eks/rating", - "type": "number" - }, - "context": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context", - "type": "string" - }, - "comments": { - "id": "http://api.ekstep.org/telemetry/edata/eks/comments", - "type": "string" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_interact.json b/schemas/2.0/oe_interact.json deleted file mode 100644 index d81f749957..0000000000 --- a/schemas/2.0/oe_interact.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_interact", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "stageid", - "type", - "subtype", - "extype", - "pos", - "values", - "id", - "tid", - "uri" - ], - "properties": { - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/stageid", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string", - "enum": [ - "TOUCH", - "SHOW", - "HIDE", - "DRAG", - "DROP", - "PINCH", - "ZOOM", - "SHAKE", - "ROTATE", - "SPEAK", - "LISTEN", - "WRITE", - "DRAW", - "START", - "END", - "CHOOSE", - "ACTIVATE", - "OTHER", - "SCROLL", - "HEARTBEAT" - ] - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "extype": { - "id": "http://api.ekstep.org/telemetry/edata/extype", - "type": "string" - }, - "pos": { - "id": "http://api.ekstep.org/telemetry/edata/pos", - "type": "array" - }, - "values": { - "id": "http://api.ekstep.org/telemetry/edata/values", - "type": "array", - "items": { - "type": "object" - } - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/id", - "type": "string" - }, - "tid": { - "id": "http://api.ekstep.org/telemetry/edata/tid", - "type": "string" - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/uri", - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_interrupt.json b/schemas/2.0/oe_interrupt.json deleted file mode 100644 index 08fc8306a6..0000000000 --- a/schemas/2.0/oe_interrupt.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_interrupt", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "stageid" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND","RESUME" - ] - }, - "stageid": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_item_response.json b/schemas/2.0/oe_item_response.json deleted file mode 100644 index 8b472bc8fa..0000000000 --- a/schemas/2.0/oe_item_response.json +++ /dev/null @@ -1,79 +0,0 @@ - -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ITEM_RESPONSE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "qid", - "type", - "state", - "resvalues" - ], - "properties": { - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/qid", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string", - "enum": [ - "CHOOSE","DRAG","SELECT","MATCH","INPUT","SPEAK","WRITE" - ] - }, - "state": { - "id": "http://api.ekstep.org/telemetry/edata/state", - "type": "string", - "enum": [ - "SELECTED", "UNSELECTED" - ] - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_learn.json b/schemas/2.0/oe_learn.json deleted file mode 100644 index 31f8fc7ce7..0000000000 --- a/schemas/2.0/oe_learn.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_learn", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_LEARN" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "topics" - ], - "properties": { - "topics": { - "id": "http://api.ekstep.org/telemetry/edata/eks/topics", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "mc", - "methods" - ], - "properties": { - "mc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mc", - "type": "string" - }, - "methods": { - "id": "http://api.ekstep.org/telemetry/edata/eks/methods", - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": [ - "PLAY", - "ANSWER", - "WRITE", - "SPEAK", - "OTHER" - ] - } - } - } - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_level_set.json b/schemas/2.0/oe_level_set.json deleted file mode 100644 index f30bf741d2..0000000000 --- a/schemas/2.0/oe_level_set.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_level_set", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_LEVEL_SET" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "category", - "current", - "max" - ], - "properties": { - "category": { - "id": "http://api.ekstep.org/telemetry/edata/eks/category", - "type": "string", - "minLength": 1 - }, - "current": { - "id": "http://api.ekstep.org/telemetry/edata/eks/current", - "type": "string", - "minLength": 1 - }, - "max": { - "id": "http://api.ekstep.org/telemetry/edata/eks/max", - "type": "string", - "minLength": 1 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_navigate.json b/schemas/2.0/oe_navigate.json deleted file mode 100644 index 74cb9a1b36..0000000000 --- a/schemas/2.0/oe_navigate.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_NAVIGATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "itype", - "stageid", - "stageto" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "itype": { - "id": "http://api.ekstep.org/telemetry/edata/itype", - "type": "string" - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/stageid", - "type": "string" - }, - "stageto": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_start.json b/schemas/2.0/oe_start.json deleted file mode 100644 index c2ac2b24e6..0000000000 --- a/schemas/2.0/oe_start.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "properties": { - "mode": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mode", - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.0/oe_xapi.json b/schemas/2.0/oe_xapi.json deleted file mode 100644 index 8b27a4ea2a..0000000000 --- a/schemas/2.0/oe_xapi.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_xapi", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_XAPI" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "xapi" - ], - "properties": { - "xapi": { - "id": "http://api.ekstep.org/telemetry/edata/xapi", - "type": "object" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/cdata.json b/schemas/2.1/cdata.json deleted file mode 100644 index 4c7261b025..0000000000 --- a/schemas/2.1/cdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/cdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/cdata/id", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ "type", "id"] - } -} diff --git a/schemas/2.1/common.json b/schemas/2.1/common.json deleted file mode 100644 index 9a54ea88cc..0000000000 --- a/schemas/2.1/common.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.1" - ] - }, - "gdata": { - "$ref": "http://localhost:7070/schemas/2.1/gdata.json" - }, - "pdata": { - "$ref": "http://localhost:7070/schemas/2.1/pdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 1 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 1 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "etags": { - "$ref": "http://localhost:7070/schemas/2.1/etags.json" - }, - "cdata": { - "$ref": "http://localhost:7070/schemas/2.1/cdata.json" - }, - "channel": { - "id": "http://api.ekstep.org/telemetry/channel", - "type": "string", - "minLength": 1 - } - } -} \ No newline at end of file diff --git a/schemas/2.1/common_empty_sid_uid.json b/schemas/2.1/common_empty_sid_uid.json deleted file mode 100644 index 7f5fea1735..0000000000 --- a/schemas/2.1/common_empty_sid_uid.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.1" - ] - }, - "pdata": { - "$ref": "http://localhost:7070/schemas/2.1/pdata.json" - }, - "gdata": { - "$ref": "http://localhost:7070/schemas/2.1/gdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "etags": { - "$ref": "http://localhost:7070/schemas/2.1/etags.json" - }, - "cdata": { - "$ref": "http://localhost:7070/schemas/2.1/cdata.json" - }, - "channel": { - "id": "http://api.ekstep.org/telemetry/2.1/channel", - "type": "string", - "minLength": 1 - } - } -} diff --git a/schemas/2.1/etags.json b/schemas/2.1/etags.json deleted file mode 100644 index e00c02e041..0000000000 --- a/schemas/2.1/etags.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "object", - "properties": { - "app": { - "id": "http://api.ekstep.org/telemetry/etags/app", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/app/items", - "type": "string" - } - }, - "partner": { - "id": "http://api.ekstep.org/telemetry/etags/partner", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/partner/items", - "type": "string" - } - }, - "dims": { - "id": "http://api.ekstep.org/telemetry/etags/dims", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/dims/items", - "type": "string" - } - } - }, - "additionalProperties": false -} diff --git a/schemas/2.1/event.json b/schemas/2.1/event.json deleted file mode 100644 index 0cf8cc2f3c..0000000000 --- a/schemas/2.1/event.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/event", - "type": "object", - "oneOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/ge_genie_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_genie_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_session_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_session_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_launch_game.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_game_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_profile_set.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_partner_data.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_register_partner.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_start_partner_session.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_stop_partner_session.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_view_progress.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_genie_update.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_game_update.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_api_call.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_service_api_call.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_genie_resume.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_interact.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_interrupt.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_error.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_feedback.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_assess.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_level_set.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_learn.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_earn.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_error.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_interact.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_interrupt.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_item_response.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_navigate.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_feedback.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_create_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_update_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_transfer.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_delete_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_summary.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/oe_xapi.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.1/ge_create_user.json" - } - ] -} diff --git a/schemas/2.1/gdata.json b/schemas/2.1/gdata.json deleted file mode 100644 index 1b603a1b7b..0000000000 --- a/schemas/2.1/gdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/schemas/2.1/ge_api_call.json b/schemas/2.1/ge_api_call.json deleted file mode 100644 index b3b7c75d6f..0000000000 --- a/schemas/2.1/ge_api_call.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_api_call", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_API_CALL" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "api", - "msgid", - "isize", - "osize", - "length", - "err" - ], - "properties": { - "api": { - "type": "string", - "minLength": 1 - }, - "msgid": { - "type": "string", - "minLength": 1 - }, - "isize": { - "type": "number", - "minimum": 0 - }, - "osize": { - "type": "number", - "minimum": 0 - }, - "length": { - "type": "number", - "minimum": 0 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_create_profile.json b/schemas/2.1/ge_create_profile.json deleted file mode 100644 index 809f90122f..0000000000 --- a/schemas/2.1/ge_create_profile.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_CREATE_PROFILE" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "loc": { - "type": "string" - }, - "uid": { - "type": "string" - }, - "handle": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "age": { - "type": "number" - }, - "standard": { - "type": "number" - }, - "language": { - "type": "string" - }, - "day": { - "type": "number" - }, - "month": { - "type": "number" - }, - "board": { - "type": "string" - }, - "medium": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_create_user.json b/schemas/2.1/ge_create_user.json deleted file mode 100644 index e9ae8936b0..0000000000 --- a/schemas/2.1/ge_create_user.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_CREATE_USER" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "loc": { - "type": "string" - }, - "uid": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_delete_profile.json b/schemas/2.1/ge_delete_profile.json deleted file mode 100644 index 398d932319..0000000000 --- a/schemas/2.1/ge_delete_profile.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_delete_profile", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_DELETE_PROFILE" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "uid": { - "type": "string" - }, - "duration": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_error.json b/schemas/2.1/ge_error.json deleted file mode 100644 index fbb84e8621..0000000000 --- a/schemas/2.1/ge_error.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id", - "err", - "eventId", - "data" - ], - "properties": { - "err": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "SYSTEM","GENIE","GAME","GENIESERVICES" - ] - }, - "id": { - "type": "string" - }, - "eventId": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_export.json b/schemas/2.1/ge_export.json deleted file mode 100644 index 29cd66a90b..0000000000 --- a/schemas/2.1/ge_export.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_EXPORT" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "telemetryEvents": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_feedback.json b/schemas/2.1/ge_feedback.json deleted file mode 100644 index 3517193ec3..0000000000 --- a/schemas/2.1/ge_feedback.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_FEEDBACK" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "rating", - "context", - "comments" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/type", - "type": "string", - "enum": [ - "FLAG", - "SURVEY", - "RATING" - ] - }, - "rating": { - "id": "http://api.ekstep.org/telemetry/edata/eks/rating", - "type": "number" - }, - "context": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id", - "stageid" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context/id", - "type": "string" - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context/stageid", - "type": "string" - } - } - }, - "comments": { - "id": "http://api.ekstep.org/telemetry/edata/eks/comments", - "type": "string" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_game_end.json b/schemas/2.1/ge_game_end.json deleted file mode 100644 index 1a9383989f..0000000000 --- a/schemas/2.1/ge_game_end.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_game_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GAME_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "gid", - "length", - "tmsize", - "err" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "length": { - "type": "number", - "minimum": 0 - }, - "tmsize": { - "type": "number", - "minimum": 0 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_game_update.json b/schemas/2.1/ge_game_update.json deleted file mode 100644 index f42c239e27..0000000000 --- a/schemas/2.1/ge_game_update.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_game_update", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GAME_UPDATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "mode", - "ver", - "size", - "err", - "gid" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "mode":{ - "enum": [ - "MDATA", - "WIFI", - "SDCARD", - "BT", - "OTHER" - ] - }, - "ver": { - "type": "string", - "minLength": 1 - }, - "size": { - "type": "number", - "minimum": -1 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_genie_end.json b/schemas/2.1/ge_genie_end.json deleted file mode 100644 index bab84d4551..0000000000 --- a/schemas/2.1/ge_genie_end.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_genie_resume.json b/schemas/2.1/ge_genie_resume.json deleted file mode 100644 index 60b2e4cdc6..0000000000 --- a/schemas/2.1/ge_genie_resume.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_resume", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_RESUME" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "loc" - ], - "properties": { - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_genie_start.json b/schemas/2.1/ge_genie_start.json deleted file mode 100644 index eb49966c1f..0000000000 --- a/schemas/2.1/ge_genie_start.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "dspec", - "loc" - ], - "properties": { - "dspec": { - "id": "http://api.ekstep.org/telemetry/edata/dspec", - "type": "object", - "additionalProperties": false, - "required": [ - "os", - "make", - "dname", - "dlocname", - "mem", - "id", - "idisk", - "edisk", - "scrn", - "camera", - "cpu", - "sims", - "cap" - ], - "properties": { - "os": { - "type": "string", - "minLength": 1 - }, - "dname": { - "type": "string" - }, - "dlocname": { - "type": "string" - }, - "make": { - "type": "string", - "minLength": 1 - }, - "id": { - "type": "string" - }, - "idisk": { - "type": "number", - "minimum": -1 - }, - "edisk": { - "type": "number", - "minimum": -1 - }, - "mem": { - "type": "number", - "minimum": -1 - }, - "sims": { - "type": "number", - "minimum": -1 - }, - "scrn": { - "type": "number", - "minimum": -1 - }, - "camera": { - "type": "string" - }, - "cpu": { - "type": "string" - }, - "cap": { - "type": "array", - "items": { - "type": "string" - } - }, - "mdata": { - "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/id", - "type": "string" - } - } - } - } - }, - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_genie_update.json b/schemas/2.1/ge_genie_update.json deleted file mode 100644 index 7e081f0f80..0000000000 --- a/schemas/2.1/ge_genie_update.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_genie_update", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GENIE_UPDATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "properties": { - "mode": { - "enum": [ - "MDATA", - "WIFI" - ] - }, - "ver": { - "type": "string", - "minLength": 1 - }, - "size": { - "type": "number", - "minimum": -1 - }, - "err": { - "type": "string" - }, - "referrer": { - "id": "http://api.ekstep.org/telemetry/edata/eks/referrer", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_import.json b/schemas/2.1/ge_import.json deleted file mode 100644 index 64e7038707..0000000000 --- a/schemas/2.1/ge_import.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_IMPORT" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "did": { - "type": "string" - }, - "telemetryEvents": { - "type": "number" - }, - "userProfiles": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_interact.json b/schemas/2.1/ge_interact.json deleted file mode 100644 index fdc0058d0a..0000000000 --- a/schemas/2.1/ge_interact.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_interact", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "stageid", - "type", - "subtype", - "extype", - "pos", - "values", - "id", - "tid", - "uri" - ], - "properties": { - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/stageid", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string", - "enum": [ - "TOUCH", - "SHOW", - "HIDE", - "DRAG", - "DROP", - "PINCH", - "ZOOM", - "SHAKE", - "ROTATE", - "SPEAK", - "LISTEN", - "WRITE", - "DRAW", - "START", - "END", - "CHOOSE", - "ACTIVATE", - "OTHER" - ] - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "extype": { - "id": "http://api.ekstep.org/telemetry/edata/extype", - "type": "string" - }, - "pos": { - "id": "http://api.ekstep.org/telemetry/edata/pos", - "type": "array" - }, - "values": { - "id": "http://api.ekstep.org/telemetry/edata/values", - "type": "array", - "items": { - "type": "object" - } - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/id", - "type": "string" - }, - "tid": { - "id": "http://api.ekstep.org/telemetry/edata/tid", - "type": "string" - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/uri", - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_interrupt.json b/schemas/2.1/ge_interrupt.json deleted file mode 100644 index 175f72bf2c..0000000000 --- a/schemas/2.1/ge_interrupt.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_interrupt", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND" - ] - }, - "id": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_launch_game.json b/schemas/2.1/ge_launch_game.json deleted file mode 100644 index afdc6da457..0000000000 --- a/schemas/2.1/ge_launch_game.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_launch_game", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_LAUNCH_GAME" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "gid", - "tmschm", - "err" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "tmschm": { - "enum": [ - "NONE", - "INTENT", - "FILE" - ] - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_misc.json b/schemas/2.1/ge_misc.json deleted file mode 100644 index ef470b6a7a..0000000000 --- a/schemas/2.1/ge_misc.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_misc", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_MISC" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false - } - } - } - ] -} diff --git a/schemas/2.1/ge_partner_data.json b/schemas/2.1/ge_partner_data.json deleted file mode 100644 index 381e19748f..0000000000 --- a/schemas/2.1/ge_partner_data.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_PARTNER_DATA" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.1" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "partnerid": { - "type": "string" - }, - "publickeyid": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_profile_set.json b/schemas/2.1/ge_profile_set.json deleted file mode 100644 index 9b0afad194..0000000000 --- a/schemas/2.1/ge_profile_set.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_profile_set", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_PROFILE_SET" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "ueksid", - "attrs" - ], - "properties": { - "ueksid": { - "type": "string", - "minLength": 1 - }, - "attrs": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_register_partner.json b/schemas/2.1/ge_register_partner.json deleted file mode 100644 index 859390fc69..0000000000 --- a/schemas/2.1/ge_register_partner.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_REGISTER_PARTNER" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.1" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "partnerid": { - "type": "string" - }, - "publickey": { - "type": "string" - }, - "publickeyid": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_service_api_call.json b/schemas/2.1/ge_service_api_call.json deleted file mode 100644 index 14201f1ec0..0000000000 --- a/schemas/2.1/ge_service_api_call.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SERVICE_API_CALL" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.1" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "message": { - "type": "string" - }, - "service": { - "type": "string" - }, - "result": { - "type": "object", - "additionalProperties": true - }, - "method": { - "type": "string" - }, - "errorMessages": { - "type": "array", - "items": { - "type": "string" - } - }, - "error": { - "type": "string" - }, - "status": { - "type": "string" - }, - "mode": { - "type": "string", - "enum": [ - "WIFI", - "MDATA", - "LOCAL", - "" - ] - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_session_end.json b/schemas/2.1/ge_session_end.json deleted file mode 100644 index 494b43163a..0000000000 --- a/schemas/2.1/ge_session_end.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_session_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SESSION_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_session_start.json b/schemas/2.1/ge_session_start.json deleted file mode 100644 index 0c46fdc93e..0000000000 --- a/schemas/2.1/ge_session_start.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_session_start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SESSION_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "loc" - ], - "properties": { - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/ge_start_partner_session.json b/schemas/2.1/ge_start_partner_session.json deleted file mode 100644 index 8712755d73..0000000000 --- a/schemas/2.1/ge_start_partner_session.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_START_PARTNER_SESSION" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": {} - } - } - } - } -} diff --git a/schemas/2.1/ge_stop_partner_session.json b/schemas/2.1/ge_stop_partner_session.json deleted file mode 100644 index 1b993ae8a6..0000000000 --- a/schemas/2.1/ge_stop_partner_session.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_STOP_PARTNER_SESSION" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "length": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_transfer.json b/schemas/2.1/ge_transfer.json deleted file mode 100644 index e9b16a75a2..0000000000 --- a/schemas/2.1/ge_transfer.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_transfer", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_TRANSFER" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "direction": { - "enum": [ - "EXPORT", - "IMPORT" - ] - }, - "datatype": { - "enum": [ - "TELEMETRY", - "CONTENT", - "PROFILE", - "EXPLODEDCONTENT" - ] - }, - "count": { - "type": "number" - }, - "size": { - "type": "number" - }, - "contents": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": true, - "properties": { - "identifier": { - "type": "string" - }, - "origin": { - "type": "string", - "minLength": 1 - }, - "transferId": { - "type": "string", - "minLength": 1 - }, - "transferCount": { - "type": "number" - }, - "pkgVersion": { - "type": "number" - } - } - } - } - }, - "required": [ - "direction", - "datatype", - "count" - ] - } - } - } - } -} diff --git a/schemas/2.1/ge_update_profile.json b/schemas/2.1/ge_update_profile.json deleted file mode 100644 index 0f4b713aaa..0000000000 --- a/schemas/2.1/ge_update_profile.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_UPDATE_PROFILE" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "minLength": 1 - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "uid": { - "type": "string" - }, - "handle": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "age": { - "type": "number" - }, - "standard": { - "type": "number" - }, - "language": { - "type": "string" - }, - "day": { - "type": "number" - }, - "month": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.1/ge_view_progress.json b/schemas/2.1/ge_view_progress.json deleted file mode 100644 index 25e05c7d48..0000000000 --- a/schemas/2.1/ge_view_progress.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_view_progress", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_VIEW_PROGRESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "ueksid" - ], - "properties": { - "ueksid": { - "type": "string", - "minLength": 1 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_assess.json b/schemas/2.1/oe_assess.json deleted file mode 100644 index b5b21c07ba..0000000000 --- a/schemas/2.1/oe_assess.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/oe_assess", - "type": "object", - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ASSESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "qid", - "pass", - "score", - "resvalues", - "length", - "exlength", - "params", - "uri", - "qindex" - ], - "properties": { - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qid", - "type": "string" - }, - "pass": { - "id": "http://api.ekstep.org/telemetry/edata/eks/pass", - "enum": [ - "Yes", - "No" - ] - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/eks/score", - "type": "number", - "minimum": 0 - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - }, - "length": { - "id": "http://api.ekstep.org/telemetry/edata/eks/length", - "type": "number", - "minimum": 0 - }, - "exlength": { - "id": "http://api.ekstep.org/telemetry/edata/eks/exlength", - "type": "number", - "minimum": 0 - }, - "params": { - "id": "http://api.ekstep.org/telemetry/edata/eks/params", - "type": "array", - "items": { - "type": "object" - } - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/eks/uri", - "type": "string" - }, - "qindex": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qindex", - "type": "number", - "minimum": 0 - }, - "qtitle": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle", - "type": "string" - }, - "qdesc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle", - "type": "string" - }, - "maxscore": { - "id": "http://api.ekstep.org/telemetry/edata/eks/maxscore", - "type": "number", - "minimum": 0 - }, - "mc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mc", - "type": "array", - "items": { - "type": "string" - } - }, - "mmc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mmc", - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - ], - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata", - "pdata", - "channel" - ] -} diff --git a/schemas/2.1/oe_earn.json b/schemas/2.1/oe_earn.json deleted file mode 100644 index d4dd750d55..0000000000 --- a/schemas/2.1/oe_earn.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_earn", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_EARN" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "required": [ - "type", - "points" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "MONEY","GEMS","POINTS" - ] - }, - "points": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_end.json b/schemas/2.1/oe_end.json deleted file mode 100644 index e9d99c1747..0000000000 --- a/schemas/2.1/oe_end.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length":{ - "type": "number", - "minimum": 0 - }, - "progress":{ - "type": "number", - "minimum": 0 - }, - "stageid":{ - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_error.json b/schemas/2.1/oe_error.json deleted file mode 100644 index e21cd17893..0000000000 --- a/schemas/2.1/oe_error.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_error", - "type": "object", - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "env", - "type", - "stageid", - "objecttype", - "objectid", - "err", - "action", - "data", - "severity" - ], - "properties": { - "env": { - "id": "http://api.ekstep.org/telemetry/edata/eks/env", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/type", - "enum": [ - "SYSTEM", - "API", - "PLUGIN", - "CONTENT", - "ASSET", - "OTHER" - ] - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/stageid", - "type": "string" - }, - "objecttype": { - "id": "http://api.ekstep.org/telemetry/edata/objecttype", - "type": "string" - }, - "objectid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/objectid", - "type": "string" - }, - "err": { - "id": "http://api.ekstep.org/telemetry/edata/eks/err", - "type": "string" - }, - "action": { - "id": "http://api.ekstep.org/telemetry/edata/eks/action", - "type": "string" - }, - "data": { - "id": "http://api.ekstep.org/telemetry/edata/eks/data", - "type": "string" - }, - "severity": { - "id": "http://api.ekstep.org/telemetry/edata/eks/severity", - "type": "string" - } - } - } - } - } - } - } - ], - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata", - "pdata", - "channel" - ] -} diff --git a/schemas/2.1/oe_feedback.json b/schemas/2.1/oe_feedback.json deleted file mode 100644 index bb19958ed8..0000000000 --- a/schemas/2.1/oe_feedback.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_FEEDBACK" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "rating", - "context", - "comments", - "resvalues" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/type", - "type": "string", - "enum": [ - "FLAG", "SURVEY", "RATING" - ] - }, - "rating": { - "id": "http://api.ekstep.org/telemetry/edata/eks/rating", - "type": "number" - }, - "context": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context", - "type": "string" - }, - "comments": { - "id": "http://api.ekstep.org/telemetry/edata/eks/comments", - "type": "string" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_interact.json b/schemas/2.1/oe_interact.json deleted file mode 100644 index e6faba5fa2..0000000000 --- a/schemas/2.1/oe_interact.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_interact", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "stageid", - "type", - "subtype", - "extype", - "pos", - "values", - "id", - "tid", - "uri" - ], - "properties": { - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/stageid", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string", - "enum": [ - "TOUCH", - "SHOW", - "HIDE", - "DRAG", - "DROP", - "PINCH", - "ZOOM", - "SHAKE", - "ROTATE", - "SPEAK", - "LISTEN", - "WRITE", - "DRAW", - "START", - "END", - "CHOOSE", - "ACTIVATE", - "OTHER", - "SCROLL", - "HEARTBEAT" - ] - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "extype": { - "id": "http://api.ekstep.org/telemetry/edata/extype", - "type": "string" - }, - "pos": { - "id": "http://api.ekstep.org/telemetry/edata/pos", - "type": "array" - }, - "values": { - "id": "http://api.ekstep.org/telemetry/edata/values", - "type": "array", - "items": { - "type": "object" - } - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/id", - "type": "string" - }, - "tid": { - "id": "http://api.ekstep.org/telemetry/edata/tid", - "type": "string" - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/uri", - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_interrupt.json b/schemas/2.1/oe_interrupt.json deleted file mode 100644 index 048ce25afa..0000000000 --- a/schemas/2.1/oe_interrupt.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_interrupt", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "stageid" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND","RESUME" - ] - }, - "stageid": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_item_response.json b/schemas/2.1/oe_item_response.json deleted file mode 100644 index 426a608495..0000000000 --- a/schemas/2.1/oe_item_response.json +++ /dev/null @@ -1,81 +0,0 @@ - -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ITEM_RESPONSE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "qid", - "type", - "state", - "resvalues" - ], - "properties": { - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/qid", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string", - "enum": [ - "CHOOSE","DRAG","SELECT","MATCH","INPUT","SPEAK","WRITE" - ] - }, - "state": { - "id": "http://api.ekstep.org/telemetry/edata/state", - "type": "string", - "enum": [ - "SELECTED", "UNSELECTED" - ] - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_learn.json b/schemas/2.1/oe_learn.json deleted file mode 100644 index d92668ac28..0000000000 --- a/schemas/2.1/oe_learn.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_learn", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_LEARN" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "topics" - ], - "properties": { - "topics": { - "id": "http://api.ekstep.org/telemetry/edata/eks/topics", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "mc", - "methods" - ], - "properties": { - "mc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mc", - "type": "string" - }, - "methods": { - "id": "http://api.ekstep.org/telemetry/edata/eks/methods", - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": [ - "PLAY", - "ANSWER", - "WRITE", - "SPEAK", - "OTHER" - ] - } - } - } - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_level_set.json b/schemas/2.1/oe_level_set.json deleted file mode 100644 index ef160fa029..0000000000 --- a/schemas/2.1/oe_level_set.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_level_set", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_LEVEL_SET" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "category", - "current", - "max" - ], - "properties": { - "category": { - "id": "http://api.ekstep.org/telemetry/edata/eks/category", - "type": "string", - "minLength": 1 - }, - "current": { - "id": "http://api.ekstep.org/telemetry/edata/eks/current", - "type": "string", - "minLength": 1 - }, - "max": { - "id": "http://api.ekstep.org/telemetry/edata/eks/max", - "type": "string", - "minLength": 1 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_navigate.json b/schemas/2.1/oe_navigate.json deleted file mode 100644 index ccfdeb8960..0000000000 --- a/schemas/2.1/oe_navigate.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_NAVIGATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "itype", - "stageid", - "stageto" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "itype": { - "id": "http://api.ekstep.org/telemetry/edata/itype", - "type": "string" - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/stageid", - "type": "string" - }, - "stageto": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_start.json b/schemas/2.1/oe_start.json deleted file mode 100644 index 461743157c..0000000000 --- a/schemas/2.1/oe_start.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "properties": { - "mode": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mode", - "type": "string" - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/stageid", - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_summary.json b/schemas/2.1/oe_summary.json deleted file mode 100644 index 77349bc296..0000000000 --- a/schemas/2.1/oe_summary.json +++ /dev/null @@ -1,210 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_earn", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_SUMMARY" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "required": [ - "timespent" - ], - "properties": { - "timespent": { - "id": "http://api.ekstep.org/telemetry/edata/eks/timespent", - "type": "number", - "minimum": 0 - }, - "interrupts": { - "id": "http://api.ekstep.org/telemetry/edata/eks/interrupts", - "type": "number", - "minimum": 0 - }, - "screensummary": { - "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary", - "type": "array", - "items": { - "type": "object" - }, - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/id", - "type": "string" - }, - "timespent": { - "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/timespent", - "type": "number", - "minimum": 0 - }, - "count": { - "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/count", - "type": "number", - "minimum": 0 - } - } - }, - "eventsummary": { - "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary", - "type": "array", - "items": { - "type": "object" - }, - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary/id", - "type": "string" - }, - "count": { - "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary/count", - "type": "number", - "minimum": 0 - } - } - }, - "items":{ - "id": "http://api.ekstep.org/telemetry/edata/eks/items", - "type": "array", - "items": { - "type": "object" - }, - "properties":{ - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/qid", - "type" : "string" - }, - "pass": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/pass", - "type" : "string" - }, - "maxscore": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/maxscore", - "type" : "number", - "minimum": 0 - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/score", - "type" : "string" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/resvalues", - "type" : "array", - "items": { - "type": "object" - } - }, - "length": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/length", - "type" : "number", - "minimum": 0 - }, - "qindex" : { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/qindex", - "type" : "number", - "minimum": 0 - }, - "qdesc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/qdesc", - "type" : "string" - }, - "qtitle": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/qtitle", - "type" : "string" - }, - "mmc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/mmc", - "type" : "array", - "items": { - "type": "string" - } - }, - "mc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/mc", - "type" : "array", - "items": { - "type": "string" - } - } - } - }, - "itemsummary": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary", - "type": "array", - "items": { - "type": "object" - }, - "properties": { - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/qid", - "type": "string" - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/score", - "type": "number", - "minimum": 0 - }, - "attempts": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/attempts", - "type": "number", - "minimum": 0 - }, - "timespent": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/timespent", - "type": "number", - "minimum": 0 - } - } - }, - "interactevents": { - "id": "http://api.ekstep.org/telemetry/edata/eks/interactevents", - "type": "number", - "minimum": 0 - }, - "interacteventspermin": { - "id": "http://api.ekstep.org/telemetry/edata/eks/interacteventspermin", - "type": "number", - "minimum": 0 - }, - "progress": { - "id": "http://api.ekstep.org/telemetry/edata/eks/progress", - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/oe_xapi.json b/schemas/2.1/oe_xapi.json deleted file mode 100644 index 4652d9bab6..0000000000 --- a/schemas/2.1/oe_xapi.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_xapi", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.1/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_XAPI" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "xapi" - ], - "properties": { - "xapi": { - "id": "http://api.ekstep.org/telemetry/edata/eks/xapi", - "type": "object" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.1/pdata.json b/schemas/2.1/pdata.json deleted file mode 100644 index cd509bc58b..0000000000 --- a/schemas/2.1/pdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/pdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/pdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/schemas/2.2/cdata.json b/schemas/2.2/cdata.json deleted file mode 100644 index 4c7261b025..0000000000 --- a/schemas/2.2/cdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/cdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/cdata/id", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ "type", "id"] - } -} diff --git a/schemas/2.2/common.json b/schemas/2.2/common.json deleted file mode 100644 index 8c311a231e..0000000000 --- a/schemas/2.2/common.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "$ref": "http://localhost:7070/schemas/2.2/gdata.json" - }, - "pdata": { - "$ref": "http://localhost:7070/schemas/2.2/pdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 1 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 1 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "etags": { - "$ref": "http://localhost:7070/schemas/2.2/etags.json" - }, - "cdata": { - "$ref": "http://localhost:7070/schemas/2.2/cdata.json" - }, - "channel": { - "id": "http://api.ekstep.org/telemetry/channel", - "type": "string", - "minLength": 1 - } - } -} \ No newline at end of file diff --git a/schemas/2.2/common_empty_sid_uid.json b/schemas/2.2/common_empty_sid_uid.json deleted file mode 100644 index 6ba8549c7e..0000000000 --- a/schemas/2.2/common_empty_sid_uid.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "pdata": { - "$ref": "http://localhost:7070/schemas/2.2/pdata.json" - }, - "gdata": { - "$ref": "http://localhost:7070/schemas/2.2/gdata.json" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "etags": { - "$ref": "http://localhost:7070/schemas/2.2/etags.json" - }, - "cdata": { - "$ref": "http://localhost:7070/schemas/2.2/cdata.json" - }, - "channel": { - "id": "http://api.ekstep.org/telemetry/2.2/channel", - "type": "string", - "minLength": 1 - } - } -} diff --git a/schemas/2.2/etags.json b/schemas/2.2/etags.json deleted file mode 100644 index e00c02e041..0000000000 --- a/schemas/2.2/etags.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "object", - "properties": { - "app": { - "id": "http://api.ekstep.org/telemetry/etags/app", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/app/items", - "type": "string" - } - }, - "partner": { - "id": "http://api.ekstep.org/telemetry/etags/partner", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/partner/items", - "type": "string" - } - }, - "dims": { - "id": "http://api.ekstep.org/telemetry/etags/dims", - "type": "array", - "items": { - "id": "http://api.ekstep.org/telemetry/etags/dims/items", - "type": "string" - } - } - }, - "additionalProperties": false -} diff --git a/schemas/2.2/event.json b/schemas/2.2/event.json deleted file mode 100644 index a63c71e16e..0000000000 --- a/schemas/2.2/event.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/event", - "type": "object", - "oneOf": [{ - "$ref": "http://localhost:7070/schemas/2.2/ge_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_resume.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_update.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_api_call.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_service_api_call.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_session_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_session_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_game_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_profile_set.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_partner_data.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_register_partner.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_start_partner_session.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_stop_partner_session.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_view_progress.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_game_update.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_interact.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_interrupt.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_error.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_feedback.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_assess.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_level_set.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_start.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_learn.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_earn.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_error.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_interact.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_interrupt.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_end.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_item_response.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_navigate.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_feedback.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_create_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_update_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_transfer.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_delete_profile.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_summary.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/oe_xapi.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_create_user.json" - }, - { - "$ref": "http://localhost:7070/schemas/2.2/ge_launch_game.json" - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/gdata.json b/schemas/2.2/gdata.json deleted file mode 100644 index 1b603a1b7b..0000000000 --- a/schemas/2.2/gdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/schemas/2.2/ge_api_call.json b/schemas/2.2/ge_api_call.json deleted file mode 100644 index 2de5d7f6c3..0000000000 --- a/schemas/2.2/ge_api_call.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_api_call", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_API_CALL" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "api", - "msgid", - "isize", - "osize", - "length", - "err" - ], - "properties": { - "api": { - "type": "string", - "minLength": 1 - }, - "msgid": { - "type": "string", - "minLength": 1 - }, - "isize": { - "type": "number", - "minimum": 0 - }, - "osize": { - "type": "number", - "minimum": 0 - }, - "length": { - "type": "number", - "minimum": 0 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.2/ge_create_profile.json b/schemas/2.2/ge_create_profile.json deleted file mode 100644 index 0b93ef9987..0000000000 --- a/schemas/2.2/ge_create_profile.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_CREATE_PROFILE" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "loc": { - "type": "string" - }, - "uid": { - "type": "string" - }, - "handle": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "age": { - "type": "number" - }, - "standard": { - "type": "number" - }, - "language": { - "type": "string" - }, - "day": { - "type": "number" - }, - "month": { - "type": "number" - }, - "board": { - "type": "string" - }, - "medium": { - "type": "string" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/2.2/ge_create_user.json b/schemas/2.2/ge_create_user.json deleted file mode 100644 index 952f537aa8..0000000000 --- a/schemas/2.2/ge_create_user.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_CREATE_USER" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "loc": { - "type": "string" - }, - "uid": { - "type": "string" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/2.2/ge_delete_profile.json b/schemas/2.2/ge_delete_profile.json deleted file mode 100644 index a87c0ea40b..0000000000 --- a/schemas/2.2/ge_delete_profile.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_delete_profile", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_DELETE_PROFILE" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum":[ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "uid": { - "type": "string" - }, - "duration": { - "type": "number" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/2.2/ge_end.json b/schemas/2.2/ge_end.json deleted file mode 100644 index 9f0a942821..0000000000 --- a/schemas/2.2/ge_end.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.2/ge_error.json b/schemas/2.2/ge_error.json deleted file mode 100644 index 646a22e29a..0000000000 --- a/schemas/2.2/ge_error.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id", - "err", - "eventId", - "data" - ], - "properties": { - "err": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "SYSTEM", - "GENIE", - "GAME", - "GENIESERVICES" - ] - }, - "id": { - "type": "string" - }, - "eventId": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/ge_feedback.json b/schemas/2.2/ge_feedback.json deleted file mode 100644 index 068273a041..0000000000 --- a/schemas/2.2/ge_feedback.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_FEEDBACK" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "rating", - "context", - "comments" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/type", - "type": "string", - "enum": [ - "FLAG", - "SURVEY", - "RATING" - ] - }, - "rating": { - "id": "http://api.ekstep.org/telemetry/edata/eks/rating", - "type": "number" - }, - "context": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id", - "stageid" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context/id", - "type": "string" - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context/stageid", - "type": "string" - } - } - }, - "comments": { - "id": "http://api.ekstep.org/telemetry/edata/eks/comments", - "type": "string" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/ge_game_end.json b/schemas/2.2/ge_game_end.json deleted file mode 100644 index d7f6d30243..0000000000 --- a/schemas/2.2/ge_game_end.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_game_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GAME_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "gid", - "length", - "tmsize", - "err" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "length": { - "type": "number", - "minimum": 0 - }, - "tmsize": { - "type": "number", - "minimum": 0 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.2/ge_game_update.json b/schemas/2.2/ge_game_update.json deleted file mode 100644 index 57428bb22c..0000000000 --- a/schemas/2.2/ge_game_update.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_game_update", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_GAME_UPDATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "mode", - "ver", - "size", - "err", - "gid" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "mode": { - "enum": [ - "MDATA", - "WIFI", - "SDCARD", - "BT", - "OTHER" - ] - }, - "ver": { - "type": "string", - "minLength": 1 - }, - "size": { - "type": "number", - "minimum": -1 - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/ge_interact.json b/schemas/2.2/ge_interact.json deleted file mode 100644 index ac2f99645d..0000000000 --- a/schemas/2.2/ge_interact.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_interact", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "stageid", - "type", - "subtype", - "extype", - "pos", - "values", - "id", - "tid", - "uri" - ], - "properties": { - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/stageid", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string", - "enum": [ - "TOUCH", - "SHOW", - "HIDE", - "DRAG", - "DROP", - "PINCH", - "ZOOM", - "SHAKE", - "ROTATE", - "SPEAK", - "LISTEN", - "WRITE", - "DRAW", - "START", - "END", - "CHOOSE", - "ACTIVATE", - "OTHER" - ] - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "extype": { - "id": "http://api.ekstep.org/telemetry/edata/extype", - "type": "string" - }, - "pos": { - "id": "http://api.ekstep.org/telemetry/edata/pos", - "type": "array" - }, - "values": { - "id": "http://api.ekstep.org/telemetry/edata/values", - "type": "array", - "items": { - "type": "object" - } - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/id", - "type": "string" - }, - "tid": { - "id": "http://api.ekstep.org/telemetry/edata/tid", - "type": "string" - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/uri", - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/ge_interrupt.json b/schemas/2.2/ge_interrupt.json deleted file mode 100644 index 3638e71ebb..0000000000 --- a/schemas/2.2/ge_interrupt.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_interrupt", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "IDLE", - "SLEEP", - "CALL", - "SWITCH", - "LOCK", - "OTHER", - "BACKGROUND" - ] - }, - "id": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/ge_launch_game.json b/schemas/2.2/ge_launch_game.json deleted file mode 100644 index fc7247655b..0000000000 --- a/schemas/2.2/ge_launch_game.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_launch_game", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_LAUNCH_GAME" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "gid", - "tmschm", - "err" - ], - "properties": { - "gid": { - "type": "string", - "minLength": 1 - }, - "tmschm": { - "enum": [ - "NONE", - "INTENT", - "FILE" - ] - }, - "err": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/ge_partner_data.json b/schemas/2.2/ge_partner_data.json deleted file mode 100644 index d0e26c4175..0000000000 --- a/schemas/2.2/ge_partner_data.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_PARTNER_DATA" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "partnerid": { - "type": "string" - }, - "publickeyid": { - "type": "string" - }, - "data": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.2/ge_profile_set.json b/schemas/2.2/ge_profile_set.json deleted file mode 100644 index 5461c28c28..0000000000 --- a/schemas/2.2/ge_profile_set.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_profile_set", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_PROFILE_SET" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "ueksid", - "attrs" - ], - "properties": { - "ueksid": { - "type": "string", - "minLength": 1 - }, - "attrs": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/ge_register_partner.json b/schemas/2.2/ge_register_partner.json deleted file mode 100644 index 9c2e9919d7..0000000000 --- a/schemas/2.2/ge_register_partner.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_REGISTER_PARTNER" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "partnerid": { - "type": "string" - }, - "publickey": { - "type": "string" - }, - "publickeyid": { - "type": "string" - } - } - } - } - } - } -} diff --git a/schemas/2.2/ge_resume.json b/schemas/2.2/ge_resume.json deleted file mode 100644 index c259d3f252..0000000000 --- a/schemas/2.2/ge_resume.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_resume", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_RESUME" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "loc" - ], - "properties": { - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.2/ge_service_api_call.json b/schemas/2.2/ge_service_api_call.json deleted file mode 100644 index 7c34d134b6..0000000000 --- a/schemas/2.2/ge_service_api_call.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SERVICE_API_CALL" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "message": { - "type": "string" - }, - "service": { - "type": "string" - }, - "result": { - "type": "object", - "additionalProperties": true - }, - "method": { - "type": "string" - }, - "errorMessages": { - "type": "array", - "items": { - "type": "string" - } - }, - "error": { - "type": "string" - }, - "status": { - "type": "string" - }, - "mode": { - "type": "string", - "enum": [ - "WIFI", - "MDATA", - "LOCAL", - "" - ] - } - } - } - } - } - } -} diff --git a/schemas/2.2/ge_session_end.json b/schemas/2.2/ge_session_end.json deleted file mode 100644 index b2238ff328..0000000000 --- a/schemas/2.2/ge_session_end.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_session_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SESSION_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.2/ge_session_start.json b/schemas/2.2/ge_session_start.json deleted file mode 100644 index 085f3f7779..0000000000 --- a/schemas/2.2/ge_session_start.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_session_start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_SESSION_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "loc" - ], - "properties": { - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.2/ge_start.json b/schemas/2.2/ge_start.json deleted file mode 100644 index b81a6bcd9b..0000000000 --- a/schemas/2.2/ge_start.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "dspec", - "loc" - ], - "properties": { - "dspec": { - "id": "http://api.ekstep.org/telemetry/edata/dspec", - "type": "object", - "additionalProperties": false, - "required": [ - "os", - "make", - "dname", - "dlocname", - "mem", - "id", - "idisk", - "edisk", - "scrn", - "camera", - "cpu", - "sims", - "cap" - ], - "properties": { - "os": { - "type": "string", - "minLength": 1 - }, - "dname": { - "type": "string" - }, - "dlocname": { - "type": "string" - }, - "make": { - "type": "string", - "minLength": 1 - }, - "id": { - "type": "string" - }, - "idisk": { - "type": "number", - "minimum": -1 - }, - "edisk": { - "type": "number", - "minimum": -1 - }, - "mem": { - "type": "number", - "minimum": -1 - }, - "sims": { - "type": "number", - "minimum": -1 - }, - "scrn": { - "type": "number", - "minimum": -1 - }, - "camera": { - "type": "string" - }, - "cpu": { - "type": "string" - }, - "cap": { - "type": "array", - "items": { - "type": "string" - } - }, - "mdata": { - "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/id", - "type": "string" - } - } - } - } - }, - "loc": { - "type": "string" - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.2/ge_start_partner_session.json b/schemas/2.2/ge_start_partner_session.json deleted file mode 100644 index a73b128e73..0000000000 --- a/schemas/2.2/ge_start_partner_session.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_START_PARTNER_SESSION" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": {} - } - } - } - } -} diff --git a/schemas/2.2/ge_stop_partner_session.json b/schemas/2.2/ge_stop_partner_session.json deleted file mode 100644 index 9830bab8cd..0000000000 --- a/schemas/2.2/ge_stop_partner_session.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_STOP_PARTNER_SESSION" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "length": { - "type": "number" - } - } - } - } - } - } -} diff --git a/schemas/2.2/ge_transfer.json b/schemas/2.2/ge_transfer.json deleted file mode 100644 index a3d7d6008e..0000000000 --- a/schemas/2.2/ge_transfer.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_transfer", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_TRANSFER" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "direction": { - "enum": [ - "EXPORT", - "IMPORT" - ] - }, - "datatype": { - "enum": [ - "TELEMETRY", - "CONTENT", - "PROFILE", - "EXPLODEDCONTENT" - ] - }, - "count": { - "type": "number" - }, - "size": { - "type": "number" - }, - "contents": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": true, - "properties": { - "identifier": { - "type": "string" - }, - "origin": { - "type": "string", - "minLength": 1 - }, - "transferId": { - "type": "string", - "minLength": 1 - }, - "transferCount": { - "type": "number" - }, - "pkgVersion": { - "type": "number" - } - } - } - } - }, - "required": [ - "direction", - "datatype", - "count" - ] - } - } - } - } -} \ No newline at end of file diff --git a/schemas/2.2/ge_update.json b/schemas/2.2/ge_update.json deleted file mode 100644 index 75a0050dbc..0000000000 --- a/schemas/2.2/ge_update.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_update", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_UPDATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "properties": { - "mode": { - "enum": [ - "MDATA", - "WIFI" - ] - }, - "ver": { - "type": "string", - "minLength": 1 - }, - "size": { - "type": "number", - "minimum": -1 - }, - "err": { - "type": "string" - }, - "referrer": { - "id": "http://api.ekstep.org/telemetry/edata/eks/referrer", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} diff --git a/schemas/2.2/ge_update_profile.json b/schemas/2.2/ge_update_profile.json deleted file mode 100644 index 7db13c06c2..0000000000 --- a/schemas/2.2/ge_update_profile.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_UPDATE_PROFILE" - ] - }, - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "2.2" - ] - }, - "gdata": { - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/gdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/gdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "id", - "ver" - ] - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/sid", - "type": "string", - "minLength": 0 - }, - "uid": { - "id": "http://api.ekstep.org/telemetry/uid", - "type": "string", - "minLength": 0 - }, - "did": { - "id": "http://api.ekstep.org/telemetry/did", - "type": "string", - "minLength": 1 - }, - "tags": { - "type": "array", - "items": { - "type": "object" - } - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": true, - "properties": { - "uid": { - "type": "string" - }, - "handle": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "age": { - "type": "number" - }, - "standard": { - "type": "number" - }, - "language": { - "type": "string" - }, - "day": { - "type": "number" - }, - "month": { - "type": "number" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/2.2/ge_view_progress.json b/schemas/2.2/ge_view_progress.json deleted file mode 100644 index deb1c29dae..0000000000 --- a/schemas/2.2/ge_view_progress.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/ge_view_progress", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "GE_VIEW_PROGRESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "ueksid" - ], - "properties": { - "ueksid": { - "type": "string", - "minLength": 1 - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_assess.json b/schemas/2.2/oe_assess.json deleted file mode 100644 index 33d0313327..0000000000 --- a/schemas/2.2/oe_assess.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/oe_assess", - "type": "object", - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ASSESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "qid", - "pass", - "score", - "resvalues", - "length", - "exlength", - "params", - "uri", - "qindex" - ], - "properties": { - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qid", - "type": "string" - }, - "pass": { - "id": "http://api.ekstep.org/telemetry/edata/eks/pass", - "enum": [ - "Yes", - "No" - ] - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/eks/score", - "type": "number", - "minimum": 0 - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - }, - "length": { - "id": "http://api.ekstep.org/telemetry/edata/eks/length", - "type": "number", - "minimum": 0 - }, - "exlength": { - "id": "http://api.ekstep.org/telemetry/edata/eks/exlength", - "type": "number", - "minimum": 0 - }, - "params": { - "id": "http://api.ekstep.org/telemetry/edata/eks/params", - "type": "array", - "items": { - "type": "object" - } - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/eks/uri", - "type": "string" - }, - "qindex": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qindex", - "type": "number", - "minimum": 0 - }, - "qtitle": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle", - "type": "string" - }, - "qdesc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle", - "type": "string" - }, - "maxscore": { - "id": "http://api.ekstep.org/telemetry/edata/eks/maxscore", - "type": "number", - "minimum": 0 - }, - "mc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mc", - "type": "array", - "items": { - "type": "string" - } - }, - "mmc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mmc", - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - ], - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata", - "pdata", - "channel" - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_earn.json b/schemas/2.2/oe_earn.json deleted file mode 100644 index e1bb751be7..0000000000 --- a/schemas/2.2/oe_earn.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_earn", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_EARN" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "required": [ - "type", - "points" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "MONEY", - "GEMS", - "POINTS" - ] - }, - "points": { - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_end.json b/schemas/2.2/oe_end.json deleted file mode 100644 index c2a9cdbe8f..0000000000 --- a/schemas/2.2/oe_end.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "length" - ], - "properties": { - "length": { - "type": "number", - "minimum": 0 - }, - "progress": { - "type": "number", - "minimum": 0 - }, - "stageid": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_error.json b/schemas/2.2/oe_error.json deleted file mode 100644 index 7ddab3828e..0000000000 --- a/schemas/2.2/oe_error.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_error", - "type": "object", - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "env", - "type", - "stageid", - "objecttype", - "objectid", - "err", - "action", - "data", - "severity" - ], - "properties": { - "env": { - "id": "http://api.ekstep.org/telemetry/edata/eks/env", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/type", - "enum": [ - "SYSTEM", - "API", - "PLUGIN", - "CONTENT", - "ASSET", - "OTHER" - ] - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/stageid", - "type": "string" - }, - "objecttype": { - "id": "http://api.ekstep.org/telemetry/edata/objecttype", - "type": "string" - }, - "objectid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/objectid", - "type": "string" - }, - "err": { - "id": "http://api.ekstep.org/telemetry/edata/eks/err", - "type": "string" - }, - "action": { - "id": "http://api.ekstep.org/telemetry/edata/eks/action", - "type": "string" - }, - "data": { - "id": "http://api.ekstep.org/telemetry/edata/eks/data", - "type": "string" - }, - "severity": { - "id": "http://api.ekstep.org/telemetry/edata/eks/severity", - "type": "string" - } - } - } - } - } - } - } - ], - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "edata", - "pdata", - "channel" - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_feedback.json b/schemas/2.2/oe_feedback.json deleted file mode 100644 index 5898d2dfb3..0000000000 --- a/schemas/2.2/oe_feedback.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_FEEDBACK" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "rating", - "context", - "comments", - "resvalues" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/eks/type", - "type": "string", - "enum": [ - "FLAG", - "SURVEY", - "RATING" - ] - }, - "rating": { - "id": "http://api.ekstep.org/telemetry/edata/eks/rating", - "type": "number" - }, - "context": { - "id": "http://api.ekstep.org/telemetry/edata/eks/context", - "type": "string" - }, - "comments": { - "id": "http://api.ekstep.org/telemetry/edata/eks/comments", - "type": "string" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_interact.json b/schemas/2.2/oe_interact.json deleted file mode 100644 index 82bb414abb..0000000000 --- a/schemas/2.2/oe_interact.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_interact", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "stageid", - "type", - "subtype", - "extype", - "pos", - "values", - "id", - "tid", - "uri" - ], - "properties": { - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/stageid", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string", - "enum": [ - "TOUCH", - "SHOW", - "HIDE", - "DRAG", - "DROP", - "PINCH", - "ZOOM", - "SHAKE", - "ROTATE", - "SPEAK", - "LISTEN", - "WRITE", - "DRAW", - "START", - "END", - "CHOOSE", - "ACTIVATE", - "OTHER", - "SCROLL", - "HEARTBEAT" - ] - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "extype": { - "id": "http://api.ekstep.org/telemetry/edata/extype", - "type": "string" - }, - "pos": { - "id": "http://api.ekstep.org/telemetry/edata/pos", - "type": "array" - }, - "values": { - "id": "http://api.ekstep.org/telemetry/edata/values", - "type": "array", - "items": { - "type": "object" - } - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/id", - "type": "string" - }, - "tid": { - "id": "http://api.ekstep.org/telemetry/edata/tid", - "type": "string" - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/uri", - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_interrupt.json b/schemas/2.2/oe_interrupt.json deleted file mode 100644 index fe3c8b296b..0000000000 --- a/schemas/2.2/oe_interrupt.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_interrupt", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "stageid" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "IDLE", - "SLEEP", - "CALL", - "SWITCH", - "LOCK", - "OTHER", - "BACKGROUND", - "RESUME" - ] - }, - "stageid": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_item_response.json b/schemas/2.2/oe_item_response.json deleted file mode 100644 index 3efd05b799..0000000000 --- a/schemas/2.2/oe_item_response.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_ITEM_RESPONSE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "qid", - "type", - "state", - "resvalues" - ], - "properties": { - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/qid", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string", - "enum": [ - "CHOOSE", - "DRAG", - "SELECT", - "MATCH", - "INPUT", - "SPEAK", - "WRITE" - ] - }, - "state": { - "id": "http://api.ekstep.org/telemetry/edata/state", - "type": "string", - "enum": [ - "SELECTED", - "UNSELECTED" - ] - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/res", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_learn.json b/schemas/2.2/oe_learn.json deleted file mode 100644 index 10922475f8..0000000000 --- a/schemas/2.2/oe_learn.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_learn", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_LEARN" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "topics" - ], - "properties": { - "topics": { - "id": "http://api.ekstep.org/telemetry/edata/eks/topics", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "mc", - "methods" - ], - "properties": { - "mc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mc", - "type": "string" - }, - "methods": { - "id": "http://api.ekstep.org/telemetry/edata/eks/methods", - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": [ - "PLAY", - "ANSWER", - "WRITE", - "SPEAK", - "OTHER" - ] - } - } - } - } - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_level_set.json b/schemas/2.2/oe_level_set.json deleted file mode 100644 index 3a9dc1c3da..0000000000 --- a/schemas/2.2/oe_level_set.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_level_set", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_LEVEL_SET" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "category", - "current", - "max" - ], - "properties": { - "category": { - "id": "http://api.ekstep.org/telemetry/edata/eks/category", - "type": "string", - "minLength": 1 - }, - "current": { - "id": "http://api.ekstep.org/telemetry/edata/eks/current", - "type": "string", - "minLength": 1 - }, - "max": { - "id": "http://api.ekstep.org/telemetry/edata/eks/max", - "type": "string", - "minLength": 1 - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_navigate.json b/schemas/2.2/oe_navigate.json deleted file mode 100644 index 0330d70194..0000000000 --- a/schemas/2.2/oe_navigate.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_NAVIGATE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "itype", - "stageid", - "stageto" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "itype": { - "id": "http://api.ekstep.org/telemetry/edata/itype", - "type": "string" - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/stageid", - "type": "string" - }, - "stageto": { - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_start.json b/schemas/2.2/oe_start.json deleted file mode 100644 index 599487c742..0000000000 --- a/schemas/2.2/oe_start.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "properties": { - "mode": { - "id": "http://api.ekstep.org/telemetry/edata/eks/mode", - "type": "string" - }, - "stageid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/stageid", - "type": "string" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_summary.json b/schemas/2.2/oe_summary.json deleted file mode 100644 index 08c86a17bc..0000000000 --- a/schemas/2.2/oe_summary.json +++ /dev/null @@ -1,210 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_earn", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata", - "pdata", - "channel" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_SUMMARY" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "required": [ - "timespent" - ], - "properties": { - "timespent": { - "id": "http://api.ekstep.org/telemetry/edata/eks/timespent", - "type": "number", - "minimum": 0 - }, - "interrupts": { - "id": "http://api.ekstep.org/telemetry/edata/eks/interrupts", - "type": "number", - "minimum": 0 - }, - "screensummary": { - "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary", - "type": "array", - "items": { - "type": "object" - }, - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/id", - "type": "string" - }, - "timespent": { - "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/timespent", - "type": "number", - "minimum": 0 - }, - "count": { - "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/count", - "type": "number", - "minimum": 0 - } - } - }, - "eventsummary": { - "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary", - "type": "array", - "items": { - "type": "object" - }, - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary/id", - "type": "string" - }, - "count": { - "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary/count", - "type": "number", - "minimum": 0 - } - } - }, - "items": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items", - "type": "array", - "items": { - "type": "object" - }, - "properties": { - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/qid", - "type": "string" - }, - "pass": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/pass", - "type": "string" - }, - "maxscore": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/maxscore", - "type": "number", - "minimum": 0 - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/score", - "type": "string" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/resvalues", - "type": "array", - "items": { - "type": "object" - } - }, - "length": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/length", - "type": "number", - "minimum": 0 - }, - "qindex": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/qindex", - "type": "number", - "minimum": 0 - }, - "qdesc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/qdesc", - "type": "string" - }, - "qtitle": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/qtitle", - "type": "string" - }, - "mmc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/mmc", - "type": "array", - "items": { - "type": "string" - } - }, - "mc": { - "id": "http://api.ekstep.org/telemetry/edata/eks/items/mc", - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "itemsummary": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary", - "type": "array", - "items": { - "type": "object" - }, - "properties": { - "qid": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/qid", - "type": "string" - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/score", - "type": "number", - "minimum": 0 - }, - "attempts": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/attempts", - "type": "number", - "minimum": 0 - }, - "timespent": { - "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/timespent", - "type": "number", - "minimum": 0 - } - } - }, - "interactevents": { - "id": "http://api.ekstep.org/telemetry/edata/eks/interactevents", - "type": "number", - "minimum": 0 - }, - "interacteventspermin": { - "id": "http://api.ekstep.org/telemetry/edata/eks/interacteventspermin", - "type": "number", - "minimum": 0 - }, - "progress": { - "id": "http://api.ekstep.org/telemetry/edata/eks/progress", - "type": "number", - "minimum": 0 - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/oe_xapi.json b/schemas/2.2/oe_xapi.json deleted file mode 100644 index 04c1e9e546..0000000000 --- a/schemas/2.2/oe_xapi.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/oe_xapi", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "gdata", - "sid", - "uid", - "did", - "mid", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/2.2/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "OE_XAPI" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "eks" - ], - "properties": { - "eks": { - "id": "http://api.ekstep.org/telemetry/edata/eks", - "type": "object", - "additionalProperties": false, - "required": [ - "xapi" - ], - "properties": { - "xapi": { - "id": "http://api.ekstep.org/telemetry/edata/eks/xapi", - "type": "object" - } - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/2.2/pdata.json b/schemas/2.2/pdata.json deleted file mode 100644 index cd509bc58b..0000000000 --- a/schemas/2.2/pdata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/pdata/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/pdata/ver", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["id","ver"] -} diff --git a/schemas/3.0/actor.json b/schemas/3.0/actor.json deleted file mode 100644 index cd87b3fb22..0000000000 --- a/schemas/3.0/actor.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/actor/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/actor/type", - "type": "string" - } - }, - "required": [ "type", "id"] -} \ No newline at end of file diff --git a/schemas/3.0/assess.json b/schemas/3.0/assess.json deleted file mode 100644 index dd5e066ccb..0000000000 --- a/schemas/3.0/assess.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/assess", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "ASSESS" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "item", - "pass", - "score", - "resvalues", - "duration" - ], - "properties": { - "item": { - "$ref": "http://localhost:7070/schemas/3.0/question.json" - }, - "index": { - "id": "http://api.ekstep.org/telemetry/edata/index", - "type": "number" - }, - "pass": { - "id": "http://api.ekstep.org/telemetry/edata/pass", - "type": "string" - }, - "score": { - "id": "http://api.ekstep.org/telemetry/edata/score", - "type": "number" - }, - "resvalues": { - "id": "http://api.ekstep.org/telemetry/edata/resvalues", - "type": "array", - "items": { - "type": "object" - } - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/audit.json b/schemas/3.0/audit.json deleted file mode 100644 index bf8fea1b04..0000000000 --- a/schemas/3.0/audit.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/audit", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "AUDIT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "props": { - "id": "http://api.ekstep.org/telemetry/edata/props", - "type": "array", - "items": { - "type": "string" - } - }, - "state": { - "id": "http://api.ekstep.org/telemetry/edata/state", - "type": "string" - }, - "prevstate": { - "id": "http://api.ekstep.org/telemetry/edata/prevstate", - "type": "string" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/cdata.json b/schemas/3.0/cdata.json deleted file mode 100644 index fd82b2b534..0000000000 --- a/schemas/3.0/cdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/cdata/type", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/cdata/id", - "type": "string" - } - }, - "additionalProperties": false, - "required": [ "type", "id"] - } -} \ No newline at end of file diff --git a/schemas/3.0/common.json b/schemas/3.0/common.json deleted file mode 100644 index 4228704830..0000000000 --- a/schemas/3.0/common.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "type": "object", - "properties": { - "ets": { - "id": "http://api.ekstep.org/telemetry/ets", - "type": "number", - "format": "date-time" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/ver", - "type": "string", - "enum": [ - "3.0", - "3.1" - ] - }, - "mid": { - "id": "http://api.ekstep.org/telemetry/mid", - "type": "string", - "minLength": 1 - }, - "actor": { - "$ref": "http://localhost:7070/schemas/3.0/actor.json" - }, - "context": { - "$ref": "http://localhost:7070/schemas/3.0/context.json" - }, - "object": { - "$ref": "http://localhost:7070/schemas/3.0/object.json" - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - } - } -} \ No newline at end of file diff --git a/schemas/3.0/context.json b/schemas/3.0/context.json deleted file mode 100644 index 95f3f2cc7c..0000000000 --- a/schemas/3.0/context.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": "object", - "properties": { - "channel": { - "id": "http://api.ekstep.org/telemetry/context/channel", - "type": "string", - "minLength": 1 - }, - "pdata": { - "$ref": "http://localhost:7070/schemas/3.0/pdata.json" - }, - "env": { - "id": "http://api.ekstep.org/telemetry/context/env", - "type": "string" - }, - "sid": { - "id": "http://api.ekstep.org/telemetry/context/sid", - "type": "string" - }, - "did": { - "id": "http://api.ekstep.org/telemetry/context/did", - "type": "string" - }, - "cdata": { - "$ref": "http://localhost:7070/schemas/3.0/cdata.json" - } - }, - "required": [ - "channel", - "env", - "pdata" - ] -} \ No newline at end of file diff --git a/schemas/3.0/dspec.json b/schemas/3.0/dspec.json deleted file mode 100644 index cc9ee16701..0000000000 --- a/schemas/3.0/dspec.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "type": "object", - "properties": { - "os": { - "id": "http://api.ekstep.org/telemetry/dspec/os", - "type": "string" - }, - "make": { - "id": "http://api.ekstep.org/telemetry/dspec/make", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/dspec/id", - "type": "string" - }, - "mem": { - "id": "http://api.ekstep.org/telemetry/dspec/mem", - "type": "number", - "minimum": -1 - }, - "idisk": { - "id": "http://api.ekstep.org/telemetry/dspec/idisk", - "type": "number", - "minimum": -1 - }, - "edisk": { - "id": "http://api.ekstep.org/telemetry/dspec/edisk", - "type": "number", - "minimum": -1 - }, - "scrn": { - "id": "http://api.ekstep.org/telemetry/dspec/scrn", - "type": "number", - "minimum": -1 - }, - "camera": { - "id": "http://api.ekstep.org/telemetry/dspec/camera", - "type": "string" - }, - "cpu": { - "id": "http://api.ekstep.org/telemetry/dspec/cpu", - "type": "string" - }, - "sims": { - "id": "http://api.ekstep.org/telemetry/dspec/sims", - "type": "number", - "minimum": -1 - }, - "cap": { - "id": "http://api.ekstep.org/telemetry/dspec/cap", - "type": "array", - "items": { - "type": "string" - } - } - } -} \ No newline at end of file diff --git a/schemas/3.0/end.json b/schemas/3.0/end.json deleted file mode 100644 index 2805c2f288..0000000000 --- a/schemas/3.0/end.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/end", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "END" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "mode": { - "id": "http://api.ekstep.org/telemetry/edata/mode", - "type": "string" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "summary": { - "id": "http://api.ekstep.org/telemetry/edata/summary", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/envelope.json b/schemas/3.0/envelope.json deleted file mode 100644 index 76ecbf8fde..0000000000 --- a/schemas/3.0/envelope.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "id":"http://api.ekstep.org/telemetry/envelope", - "type":"object", - "required":[ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf":[ - { - "$ref":"http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties":{ - "eid":{ - "id":"http://api.ekstep.org/telemetry/mid", - "type":"string", - "minLength": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/error.json b/schemas/3.0/error.json deleted file mode 100644 index 27cff116b7..0000000000 --- a/schemas/3.0/error.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/error", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "ERROR" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "err", - "errtype", - "stacktrace" - ], - "properties": { - "err": { - "id": "http://api.ekstep.org/telemetry/edata/err", - "type": "string" - }, - "errtype": { - "id": "http://api.ekstep.org/telemetry/edata/errtype", - "type": "string" - }, - "stacktrace": { - "id": "http://api.ekstep.org/telemetry/edata/stacktrace", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "object": { - "$ref": "http://localhost:7070/schemas/3.0/inlineobject.json" - }, - "plugin": { - "$ref": "http://localhost:7070/schemas/3.0/plugin.json" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/event.json b/schemas/3.0/event.json deleted file mode 100644 index 656882247a..0000000000 --- a/schemas/3.0/event.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://api.ekstep.org/telemetry/event", - "type": "object", - "oneOf": [{ - "$ref": "http://localhost:7070/schemas/3.0/start.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/end.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/impression.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/interact.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/assess.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/response.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/interrupt.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/feedback.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/share.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/audit.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/error.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/heartbeat.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/log.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/search.json" - }, - { - "$ref": "http://localhost:7070/schemas/3.0/exdata.json" - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/exdata.json b/schemas/3.0/exdata.json deleted file mode 100644 index 2d3ed99b40..0000000000 --- a/schemas/3.0/exdata.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/exdata", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "EXDATA" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "data": { - "id": "http://api.ekstep.org/telemetry/edata/data", - "type": "string" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/feedback.json b/schemas/3.0/feedback.json deleted file mode 100644 index c453c17ecf..0000000000 --- a/schemas/3.0/feedback.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/feedback", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "FEEDBACK" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "properties": { - "rating": { - "id": "http://api.ekstep.org/telemetry/edata/rating", - "type": "number" - }, - "comments": { - "id": "http://api.ekstep.org/telemetry/edata/comments", - "type": "string" - }, - "commentid": { - "id": "http://api.ekstep.org/telemetry/edata/commentid", - "type": "string" - }, - "commenttxt": { - "id": "http://api.ekstep.org/telemetry/edata/commenttxt", - "type": "string" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/heartbeat.json b/schemas/3.0/heartbeat.json deleted file mode 100644 index 20fa50cd73..0000000000 --- a/schemas/3.0/heartbeat.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/heartbeat", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "HEARTBEAT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object" - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/impression.json b/schemas/3.0/impression.json deleted file mode 100644 index 7f1b8623be..0000000000 --- a/schemas/3.0/impression.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/impression", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "IMPRESSION" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "pageid", - "uri" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/edata/uri", - "type": "string" - }, - "visits": { - "id": "http://api.ekstep.org/telemetry/edata/visits", - "type": "array", - "items": { - "type": "object", - "properties": { - "objid": { - "id": "http://api.ekstep.org/telemetry/edata/visits/objid", - "type": "string" - }, - "objtype": { - "id": "http://api.ekstep.org/telemetry/edata/visits/objtype", - "type": "string" - }, - "objver": { - "id": "http://api.ekstep.org/telemetry/edata/visits/objver", - "type": "string" - }, - "section": { - "id": "http://api.ekstep.org/telemetry/edata/visits/section", - "type": "string" - }, - "index": { - "id": "http://api.ekstep.org/telemetry/edata/visits/index", - "type": "number" - } - }, - "additionalProperties": false, - "required": [ - "objid", - "objtype" - ] - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/inlineobject.json b/schemas/3.0/inlineobject.json deleted file mode 100644 index 69c5e6abd3..0000000000 --- a/schemas/3.0/inlineobject.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "type", - "ver" - ], - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/inlineobject/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/inlineobject/type", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/inlineobject/ver", - "type": "string" - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/inlineobject/subtype", - "type": "string" - }, - "name": { - "id": "http://api.ekstep.org/telemetry/inlineobject/name", - "type": "string" - }, - "code": { - "id": "http://api.ekstep.org/telemetry/inlineobject/code", - "type": "string" - }, - "parent": { - "$ref": "http://localhost:7070/schemas/3.0/parent.json" - } - } -} \ No newline at end of file diff --git a/schemas/3.0/interact.json b/schemas/3.0/interact.json deleted file mode 100644 index 13e600d9d9..0000000000 --- a/schemas/3.0/interact.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/interact", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "INTERACT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "id" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "subtype": { - "id": "http://api.ekstep.org/telemetry/edata/subtype", - "type": "string" - }, - "id": { - "id": "http://api.ekstep.org/telemetry/edata/id", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "target": { - "$ref": "http://localhost:7070/schemas/3.0/target.json" - }, - "plugin": { - "$ref": "http://localhost:7070/schemas/3.0/plugin.json" - }, - "extra": { - "id": "http://api.ekstep.org/telemetry/edata/extra", - "type": "object" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/interrupt.json b/schemas/3.0/interrupt.json deleted file mode 100644 index 6f8835ba82..0000000000 --- a/schemas/3.0/interrupt.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/interrupt", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "INTERRUPT" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/items.json b/schemas/3.0/items.json deleted file mode 100644 index 937cd76555..0000000000 --- a/schemas/3.0/items.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/items", - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/items/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/items/type", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/items/ver", - "type": "string" - }, - "params": { - "id": "http://api.ekstep.org/telemetry/items/params", - "type": "array", - "items": { - "type": "object" - } - }, - "origin": { - "id": "http://api.ekstep.org/telemetry/items/origin", - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/items/origin/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/items/origin/type", - "type": "string" - } - } - }, - "to": { - "id": "http://api.ekstep.org/telemetry/items/to", - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/items/to/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/items/to/type", - "type": "string" - } - } - } - } - } -} \ No newline at end of file diff --git a/schemas/3.0/log.json b/schemas/3.0/log.json deleted file mode 100644 index e29b1a03bc..0000000000 --- a/schemas/3.0/log.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/log", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "LOG" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type", - "level", - "message" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "level": { - "id": "http://api.ekstep.org/telemetry/edata/level", - "type": "string" - }, - "message": { - "id": "http://api.ekstep.org/telemetry/edata/message", - "type": "string" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - }, - "params": { - "id": "http://api.ekstep.org/telemetry/edata/params", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/object.json b/schemas/3.0/object.json deleted file mode 100644 index 4adffacb39..0000000000 --- a/schemas/3.0/object.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/object/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/object/type", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/object/ver", - "type": "string" - }, - "rollup": { - "$ref": "http://localhost:7070/schemas/3.0/rollup.json" - } - } -} \ No newline at end of file diff --git a/schemas/3.0/parent.json b/schemas/3.0/parent.json deleted file mode 100644 index 07d4262af7..0000000000 --- a/schemas/3.0/parent.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/parent/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/parent/type", - "type": "string" - } - } -} \ No newline at end of file diff --git a/schemas/3.0/pdata.json b/schemas/3.0/pdata.json deleted file mode 100644 index 18f9ba13b6..0000000000 --- a/schemas/3.0/pdata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/pdata/id", - "type": "string" - }, - "pid": { - "id": "http://api.ekstep.org/telemetry/pdata/pid", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/pdata/ver", - "type": "string" - } - }, - "required": ["id"] -} \ No newline at end of file diff --git a/schemas/3.0/plugin.json b/schemas/3.0/plugin.json deleted file mode 100644 index cfd7d80ba9..0000000000 --- a/schemas/3.0/plugin.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "ver" - ], - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/plugin/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/plugin/ver", - "type": "string" - }, - "category": { - "id": "http://api.ekstep.org/telemetry/plugin/category", - "type": "string" - } - } -} \ No newline at end of file diff --git a/schemas/3.0/question.json b/schemas/3.0/question.json deleted file mode 100644 index 89bd24f052..0000000000 --- a/schemas/3.0/question.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "type": "object", - "additionalProperties": false, - "required": [ - "id" - ], - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/question/id", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/question/type", - "type": "string" - }, - "maxscore": { - "id": "http://api.ekstep.org/telemetry/question/maxscore", - "type": "number" - }, - "exlength": { - "id": "http://api.ekstep.org/telemetry/question/exlength", - "type": "number" - }, - "params": { - "id": "http://api.ekstep.org/telemetry/question/params", - "type": "array", - "items": { - "type": "object" - } - }, - "uri": { - "id": "http://api.ekstep.org/telemetry/question/uri", - "type": "string" - }, - "desc": { - "id": "http://api.ekstep.org/telemetry/question/desc", - "type": "string" - }, - "title": { - "id": "http://api.ekstep.org/telemetry/question/title", - "type": "string" - }, - "mmc": { - "id": "http://api.ekstep.org/telemetry/question/mmc", - "type": "array", - "items": { - "type": "string" - } - }, - "mc": { - "id": "http://api.ekstep.org/telemetry/question/mc", - "type": "array", - "items": { - "type": "string" - } - } - } -} \ No newline at end of file diff --git a/schemas/3.0/response.json b/schemas/3.0/response.json deleted file mode 100644 index d2b3630e7f..0000000000 --- a/schemas/3.0/response.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "RESPONSE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "target", - "type", - "values" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/type", - "type": "string" - }, - "target": { - "$ref": "http://localhost:7070/schemas/3.0/target.json" - }, - "values": { - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/rollup.json b/schemas/3.0/rollup.json deleted file mode 100644 index bfd89209f2..0000000000 --- a/schemas/3.0/rollup.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "type": "object", - "properties": { - "l1": { - "id": "http://api.ekstep.org/telemetry/context/rollup/l1", - "type": "string" - }, - "l2": { - "id": "http://api.ekstep.org/telemetry/context/rollup/l2", - "type": "string" - }, - "l3": { - "id": "http://api.ekstep.org/telemetry/context/rollup/l3", - "type": "string" - }, - "l4": { - "id": "http://api.ekstep.org/telemetry/context/rollup/l4", - "type": "string" - } - } -} \ No newline at end of file diff --git a/schemas/3.0/search.json b/schemas/3.0/search.json deleted file mode 100644 index e5858bccf4..0000000000 --- a/schemas/3.0/search.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/search", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "SEARCH" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "query", - "size", - "topn" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "query": { - "id": "http://api.ekstep.org/telemetry/edata/query", - "type": "string" - }, - "filters": { - "id": "http://api.ekstep.org/telemetry/edata/filters", - "type": "object" - }, - "sort": { - "id": "http://api.ekstep.org/telemetry/edata/sort", - "type": "object" - }, - "correlationid": { - "id": "http://api.ekstep.org/telemetry/edata/correlationid", - "type": "string" - }, - "size": { - "id": "http://api.ekstep.org/telemetry/edata/size", - "type": "number" - }, - "topn": { - "id": "http://api.ekstep.org/telemetry/edata/topn", - "type": "array", - "items": { - "type": "object" - } - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/share.json b/schemas/3.0/share.json deleted file mode 100644 index 880df92ee5..0000000000 --- a/schemas/3.0/share.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/share", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "SHARE" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "items" - ], - "properties": { - "dir": { - "id": "http://api.ekstep.org/telemetry/edata/dir", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "items": { - "$ref": "http://localhost:7070/schemas/3.0/items.json" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/start.json b/schemas/3.0/start.json deleted file mode 100644 index d274b124a7..0000000000 --- a/schemas/3.0/start.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "id": "http://api.ekstep.org/telemetry/start", - "type": "object", - "required": [ - "eid", - "ets", - "ver", - "mid", - "actor", - "context", - "edata" - ], - "allOf": [ - { - "$ref": "http://localhost:7070/schemas/3.0/common.json" - }, - { - "properties": { - "eid": { - "id": "http://api.ekstep.org/telemetry/eid", - "enum": [ - "START" - ] - }, - "edata": { - "id": "http://api.ekstep.org/telemetry/edata", - "type": "object", - "additionalProperties": false, - "required": [ - "type" - ], - "properties": { - "type": { - "id": "http://api.ekstep.org/telemetry/edata/type", - "type": "string" - }, - "dspec": { - "$ref": "http://localhost:7070/schemas/3.0/dspec.json" - }, - "uaspec": { - "$ref": "http://localhost:7070/schemas/3.0/uaspec.json" - }, - "loc": { - "id": "http://api.ekstep.org/telemetry/edata/loc", - "type": "string" - }, - "mode": { - "id": "http://api.ekstep.org/telemetry/edata/mode", - "type": "string" - }, - "duration": { - "id": "http://api.ekstep.org/telemetry/edata/duration", - "type": "number" - }, - "pageid": { - "id": "http://api.ekstep.org/telemetry/edata/pageid", - "type": "string" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/schemas/3.0/target.json b/schemas/3.0/target.json deleted file mode 100644 index 363c7103b0..0000000000 --- a/schemas/3.0/target.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "ver", - "type" - ], - "properties": { - "id": { - "id": "http://api.ekstep.org/telemetry/target/id", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/target/ver", - "type": "string" - }, - "type": { - "id": "http://api.ekstep.org/telemetry/target/type", - "type": "string" - }, - "parent": { - "$ref": "http://localhost:7070/schemas/3.0/parent.json" - } - } -} \ No newline at end of file diff --git a/schemas/3.0/uaspec.json b/schemas/3.0/uaspec.json deleted file mode 100644 index cc7d9f53c6..0000000000 --- a/schemas/3.0/uaspec.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "type": "object", - "properties": { - "agent": { - "id": "http://api.ekstep.org/telemetry/uaspec/agent", - "type": "string" - }, - "ver": { - "id": "http://api.ekstep.org/telemetry/uaspec/ver", - "type": "string" - }, - "system": { - "id": "http://api.ekstep.org/telemetry/uaspec/system", - "type": "string" - }, - "platform": { - "id": "http://api.ekstep.org/telemetry/uaspec/platform", - "type": "string" - }, - "raw": { - "id": "http://api.ekstep.org/telemetry/uaspec/raw", - "type": "string" - } - } -} \ No newline at end of file