diff --git a/.example.env b/.example.env new file mode 100644 index 0000000..6d2cb72 --- /dev/null +++ b/.example.env @@ -0,0 +1,2 @@ +MONGO_URI=mongodb://user:password@localhost:27017/?authSource=admin +TEST_DB_NAME=test_db diff --git a/config/__init__.py b/config/__init__.py index bd475a3..ef06baa 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -1,7 +1,18 @@ -from .hier_config import HierConfig from .base_config import BaseConfig +from .deal_config import DealConfig +from .fm_config import DEFAULT_ROLE, FMConfig +from .hier_config import (DRILLDOWN_BUILDERS, HIERARCHY_BUILDERS, HierConfig, + write_hierarchy_to_gbm) +from .periods_config import PeriodsConfig __all__ = [ "BaseConfig", - "HierConfig" -] \ No newline at end of file + "HierConfig", + "PeriodsConfig", + "FMConfig", + "DealConfig", + "HIERARCHY_BUILDERS", + "DRILLDOWN_BUILDERS", + "write_hierarchy_to_gbm", + "DEFAULT_ROLE", +] diff --git a/config/base_config.py b/config/base_config.py index af3f8f4..f6680d1 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -5,7 +5,7 @@ # Reach out ot Waqas or Kuldeep for details. from aviso.settings import sec_context -from infra import AUDIT_COLL +from infra.constants import AUDIT_COLL from utils.date_utils import epoch # TODO: Do we still need to support the backup and email? diff --git a/config/deal_config.py b/config/deal_config.py new file mode 100644 index 0000000..72a896a --- /dev/null +++ b/config/deal_config.py @@ -0,0 +1,4864 @@ +import copy +import logging +from collections import defaultdict + +from config import BaseConfig +from infra.filters import fetch_filter, parse_filters, fetch_many_filters +from utils.common import cached_property +from aviso.settings import sec_context + +from utils.misc_utils import get_nested, get_node_depth + +logger = logging.getLogger("gnana.%s" % __name__) + + +DEFAULT_BEST_CASE_VALS = ["Best Case"] +DEFAULT_COMMIT_VALS = [ + "Commit", + "Committed", + "Forecasted", + "commit", + "Forecasted Upside", + "True", + "true", + True, +] +DEFAULT_PIPELINE_VALS = ["Pipeline", "pipeline"] +DEFAULT_RENEWAL_VALS = [ + "Renewal", + "renewal", + "RENEWAL", + "Renewals", + "Recurring", + "Resume", + "subscription renewal", + "support renewal", + "Existing Customer - Maintenance renewal", + "Existing customer - Subscription renewal", + "Existing customer - subscription renewal", + "Maintenance renewal", + "Existing Customer \u2013 maintenance renewal", + "Maintenance Renewal (MR)", + "Existing Business", + "delayed renewal", + "Contract renewal", + "contractual renewal", + "Customer", + "Support Renewal", + "EC renewal", +] +DEFAULT_MOST_LIKELY_VALS = ["Most Likely", "most likely"] +DEFAULT_DLF_VALS = ["True", "true", True] +PULL_IN_LIST = [ + "__fav__", + "OpportunityName", + "OpportunityOwner", + "win_prob", + "pullin_prob", + "CloseDate", + "Amount", + "Stage", + "ForecastCategory", + "__id__", + "SFDCObject" +] +DEFAULT_COVID_OPPMAP_MAP_TYPES = {"amount": "Amount", "count": "Count"} +DEFAULT_STANDARD_OPPMAP_MAP_TYPES = { + "amount": "Amount", + "count": "Count", + "quartiles": "Quartiles", +} +DEFAULT_DLF_FCST_COLL_SCHEMA = [ + "opp_id", + "is_deleted", + "period", + "close_period", + "drilldown_list", + "hierarchy_list", + "dlf.in_fcst", + "update_date", +] + +DEFAULT_DISPLAY_INSIGHTS_CARD = { + "amount": [], + "close_date": ["pushes", "suggested_push"], + "stage": ["sharp_decline", "stage_dur", "stage_age"], + "global": [ + "grouper", + "other_group", + "rare_group", + "score_history", + "close_date_exp", + ], + "score_explanation": [ + "upside_deal", + "deal_amount_reco", + "deal_speed", + "scenario", + "score_history_dip", + "primary_competitor", + "competitor_win_loss", + "recency", + "new_deal", + "custom", + "closing_soon_after_eoq", + "high_leverage_moments", + "risk_insights", + "winscore_projections_upper", + "winscore_projections_lower", + "winscore_insights" + ], + "field_level": [ + "stale_deal", + "cd_change", + "anomaly", + "amount_change", + "no_amount", + "recommit2", + "stale_commit", + "never_pushed", + "engagement_grade", + "yearold", + "closedate", + "past_closedate", + "dlf_bad", + "dlf_good", + + ], + "in_fcst": ["dlf_change"], +} +DEFAULT_OPPMAP_JUDGE_TYPE_OPTION_LABELS = { + "dlf": "DLF", + "commit": "Commit", + "most_likely": DEFAULT_MOST_LIKELY_VALS[0], +} +DEFAULT_MILESTONES_NEW = [ + { + "name": "Lead Conversion", + "color": "#f89685", + "items": ['Convert lead to 5% probability'] + }, + { + "name": "Account Engagement", + "color": "#2faadc", + "items": ["Develop customer interest in proceeding with conversations"] + }, + { + "name": "Qualification", + "color": "#107e4e", + "items": ['Completion of Disco call', 'BANT Qualification'] + }, + { + "name": "Identify pain points and Metrics", + "color": "#3d4689", + "items": ["Identification of Pain Points", "Identification of Metrics"] + }, + { + "name": "Identify Champion", + "color": "#46d62e", + "items": ['Champion Identification'] + }, + { + "name": "Identify your stakeholders", + "color": "#ffc22b", + "items": ['Identification of Economic Buyer', 'Identification of Decision Process', + 'Identification of Decision Criteria'] + }, + { + "name": "Approval from Executive board", + "color": "#e1a612", + "items": ['Schedule EB meeting', 'Business reviews'] + }, + { + "name": "Legal Approval", + "color": "#e03e28", + "items": ['Legal Approval', 'Ready for signatures'] + }, + { + "name": "Signatures", + "color": "#6e77c2", + "items": ["Signatures to be done by both the parties"] + }, + { + "name": "Final review", + "color": "#025b8d", + "items": ['Deal desk final review'] + } +] + +DEFAULT_MILESTONES_RENEWAL = [ + { + "name": "Renewal Generated", + "color": "#f89685", + "items": [] + }, + { + "name": "Schedule Meeting with Customer", + "color": "#2faadc", + "items": ["Setup meeting with customer to discuss renewal"] + }, + { + "name": "Verbal Agreement", + "color": "#107e4e", + "items": ['Get verbal agreement to renew'] + }, + { + "name": "Renewal Quote", + "color": "#3d4689", + "items": ["Meeting with EB/ Executive sponsor", "Begin negotiating propoal components", + "Complete the Business Case", + "Get the initial budget approved", "No churn/dollar churn amount agreement"] + } +] + +DEFAULT_MILESTONES = { + 'new': DEFAULT_MILESTONES_NEW, + 'renewal': DEFAULT_MILESTONES_RENEWAL +} + + + +def _convert_state(state): + if state == "True": + return True + elif state == "False": + return False + return state + + + +class DealConfig(BaseConfig): + config_name = "deals" + + @cached_property + def rollup_for_writeback(self): + return self.config.get('rollup_for_writeback', False) + + @cached_property + def is_recommended_actions_enabled(self): + return self.config.get('enable_recommended_actions_task', False) + + @cached_property + def dummy_tenant(self): + """ + tenant is using dummy data, is not hooked up to any gbm/etl + """ + return self.config.get("dummy_tenant", False) + + @cached_property + def hide_deal_grid_fields(self): + """ + hide deal grid fields + """ + return self.config.get("hide_deal_grid_fields", []) + + @cached_property + def bootstrapped(self): + """ + tenant has been bootstrapped and has initial data loaded into app + """ + return self.config.get("bootstrapped", True) + + # gateway schema + @cached_property + def gateway_schema(self): + return self.config.get("gateway_schema", {}) + + @cached_property + def is_aviso_tenant(self): + return self.config.get("aviso_tenant", False) + + @cached_property + def forecast_panel(self): + return self.config.get("forecast_panel", {}) + + @cached_property + def ci_top_deals_panel(self): + return self.config.get("ci_top_deals_panel", {}) + + @cached_property + def deal_alert_fields(self): + return self.config.get("deal_alert_fields", []) + + @cached_property + def deal_alert_on(self): + return self.config.get("deal_alert_on", False) + + @cached_property + def traffic_light_criteria(self): + return self.config.get("traffic_light_criteria", False) + + @cached_property + def disable_deal_details_tab(self): + return self.config.get('disable_deal_details_tab', False) + + @cached_property + def disable_deal_history_tab(self): + return self.config.get('disable_deal_history_tab', False) + + @cached_property + def deal_details_config(self): + """ + Fetch the tenant specific navigation config + """ + default_config = { + 'details': 'Details', + 'history': 'History', + 'relationships': 'Relationships', + 'interactions': 'Interactions', + 'deal_room': 'Deal Room', + 'to_do': 'TO DO\'s' + } + return self.config.get('deal_details_config', default_config) + + @cached_property + def make_field_history_public(self): + return self.config.get('make_field_history_public', []) + + @cached_property + def not_deals_tenant(self): + return self.config.get('not_deals_tenant', {}) + + @cached_property + def enable_email_tracking_nudge(self): + return self.config.get('enable_email_tracking_nudge', False) + + @cached_property + def special_pivot_month_closeout_day(self): + return self.config.get("special_pivot_month_closeout_day", 3) + + @cached_property + def crm_url(self): + return self.config.get('crm_url') + + @cached_property + def special_pivot_filters(self): + from infra.filters import fetch_all_filters + allowed_filters = {} + filters = fetch_all_filters(self, grid_only=True, is_pivot_special=True) + special_pivot_filters = self.config.get("special_pivot", {}).get("filters", []) + for filt_id, filt in filters.items(): + if filt_id in special_pivot_filters: + allowed_filters[filt_id] = filt + return allowed_filters + + @cached_property + def raw_crr_schema(self): + if self.persona_schemas: + # Getting the schema based on persona. If the user belongs to multiple personas, then merge the schemas + personas = sec_context.get_current_user_personas() + if not personas: + return self.config.get("CRR_schema") + if len(personas) == 1: + return self.persona_schemas.get(personas[0], self.config.get("CRR_schema")) + final_schema = {} + for persona in personas: + schema = self.persona_schemas.get( + persona, self.config.get("CRR_schema")) + for k, v in schema.items(): + if k not in final_schema: + final_schema[k] = v + else: + if type(v) == dict: + final_schema[k].update(v) + elif type(v) == list: + final_schema[k].extend(v) + else: + logger.error( + "This type of type is not supported in schemas of personas yet. please check" + ) + raise Exception( + "This type of type is not supported in schemas of personas yet. please check" + ) + + return final_schema + user_role = sec_context.get_current_user_role() + if user_role and self.crr_role_schemas.get(user_role, None): + return self.crr_role_schemas.get(user_role, {}) + + return self.config.get("CRR_schema") + + @cached_property + def get_persona_schemas(self): + if self.persona_schemas: + personas = sec_context.get_current_user_personas() + if not personas: + return self.config.get("schema") + if len(personas) == 1: + return self.persona_schemas.get(personas[0], self.config.get("schema")) + final_schema = {} + personas = list(set(personas)) + for persona in personas: + schema = self.persona_schemas.get( + persona, self.config.get("schema")) + for k, v in schema.items(): + if k not in final_schema: + final_schema[k] = v + else: + if type(v) == dict: + final_schema[k].update(v) + elif type(v) == list: + final_schema[k].extend(v) + else: + logger.error( + "This type of type is not supported in schemas of personas yet. please check" + ) + raise Exception( + "This type of type is not supported in schemas of personas yet. please check" + ) + + return final_schema + + @cached_property + def get_user_role_schemas(self): + user_role = sec_context.get_current_user_role() + if user_role and self.role_schemas.get(user_role): + return self.role_schemas.get(user_role, {}) + + @cached_property + def raw_schema(self): + person_schema = self.get_persona_schemas + if person_schema: + return person_schema + user_role_schema = self.get_user_role_schemas + if user_role_schema: + return user_role_schema + return self.config.get("schema") + + # + # Field Map + # + @cached_property + def field_map(self): + """ + mapping of standard aviso deal fields to their tenant specific field names + + Returns: + dict -- {'amount': 'as_of_Amount', ...} + """ + return self.config.get("field_map", {}) + + @cached_property + def add_poc_fields_to_indicator_report(self): + """ + mapping of standard aviso deal fields to their tenant specific field names + + Returns: + dict -- {'amount': 'as_of_Amount', ...} + """ + return self.config.get("add_poc_fields_to_indicator_report", False) + + @cached_property + def leading_indicator_ref_stages(self): + """ + mapping of standard aviso deal fields to their tenant specific field names + + Returns: + dict -- {'amount': 'as_of_Amount', ...} + """ + return self.config.get("leading_indicator_ref_stages", ["Validate", "Stakeholder Alignment"]) + + @cached_property + def leading_indicator_poc_timestamp_fields(self): + """ + mapping of standard aviso deal fields to their tenant specific field names + + Returns: + dict -- {'amount': 'as_of_Amount', ...} + """ + return self.config.get("leading_indicator_poc_timestamp_fields", ["POVStartDate", "POVEndDate"]) + + @cached_property + def leading_indicator_bva_fields(self): + """ + mapping of standard aviso deal fields to their tenant specific field names + + Returns: + dict -- {'amount': 'as_of_Amount', ...} + """ + return self.config.get("leading_indicator_bva_fields", + ["BVA_Presented_to_Customer", "BVA_Presented_to_Customer_transition_timestamp"]) + + @cached_property + def stage_transition_timestamp(self): + """ + mapping of standard aviso deal fields to their tenant specific field names + + Returns: + dict -- {'amount': 'as_of_Amount', ...} + """ + return self.config.get("stage_transition_timestamp", "Stage_transition_timestamp") + + @cached_property + # + # write back fields + # + def writeback_fields(self): + crm_writable_fields = [] + for i in self.config["schema"]["deal"]: + if "crm_writable" in i.keys(): + crm_writable_fields.append(self.config["schema"]["deal_fields"][i['field']]) + return crm_writable_fields + + # + # Deal Alert Extended + # + @cached_property + def deal_alerts_fields_extended(self): + """ + Deal Alert Config for extended fields. + """ + return self.config.get("deal_alerts_fields_extended", None) + + @cached_property + def custom_manager_fc_ranks(self): + return self.config.get("custom_manager_fc_ranks", {}) + + @cached_property + def custom_gvp_fc_ranks(self): + return self.config.get("custom_gvp_fc_ranks", {}) + + @cached_property + def sankey_for_lacework(self): + """ + Added sankey config for lacework to handle CS-8700 + This config will be used for only for lacework to load sankey even when deal results fails. + """ + return self.config.get("sankey_for_lacework", False) + + @cached_property + def high_leverage_moments(self): + """ + High leverage moments config is defined to populate high leverage deals insights. + high_leverage_moments = { + 'forecast_category_order': ['Pipeline', 'Upside', 'Commit', 'Closed'], + 'stage_order': ['1-Validate', '2-Qualify', '3-Compete', '4-Negotiate', '5-Selected', '6-End user PO Issued', '8-Closed Won'], + 'days': [7,14,21,28], + 'hlm_threshold': 0.3 + } + """ + return self.config.get("high_leverage_moments", {}) + + @cached_property + def report_custom_fields_dict(self): + """ + report_custom_fields for a tenant + + Returns: + dict -- field name + """ + return self.config.get("report_custom_fields", {}) + + @cached_property + def crr_amount_field(self): + return get_nested(self.config, ["field_map", "crr_amount"]) or 'forecast' + + @cached_property + def amount_field(self): + """ + name of amount field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "amount"]) + + @cached_property + def news_deal_limit(self): + """ + Fetches the news deal limit from the configuration. + + Returns: + int | None -- The deal limit if configured, else None. + """ + return get_nested(self.config, ["dashboard", "news", "deal_limit"]) + + @cached_property + def accountid_field(self): + """ + name of account id field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "accountid"]) + + @cached_property + def meddicscore_field(self): + """ + name of meddicscore field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "meddicscore"]) + + @cached_property + def amount_prev_wk_field(self): + """ + name of amount field for tenant + + Returns: + str -- field name + """ + amount_field = get_nested(self.config, ["field_map", "amount"]) + amount_prev_wk_field = get_nested(self.config, ["field_map", "amount_prev_wk"]) + if not amount_prev_wk_field and amount_field: + amount_prev_wk_field = amount_field + '_prev_wk' + return amount_prev_wk_field + + @cached_property + def crr_amount_prev_wk_field(self): + """ + name of amount field for tenant + + Returns: + str -- field name + """ + amount_field = get_nested(self.config, ["field_map", "crr_amount"]) + amount_prev_wk_field = get_nested(self.config, ["field_map", "crr_amount_prev_wk"]) + if not amount_prev_wk_field and amount_field: + amount_prev_wk_field = amount_field + '_prev_wk' + return amount_prev_wk_field + + @cached_property + def pivot_amount_fields(self): + """ + map of amount field for tenant based on pivot + + Returns: + dict -- pivot:amount_field_name + """ + return self.config.get("pivot_amount_fields", None) + + @cached_property + def close_date_field(self): + """ + name of close date field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "close_date"]) + + @cached_property + def monthly_close_period_enrichment(self): + return self.config.get("monthly_close_period_enrichment", False) + + @cached_property + def export_close_date_field(self): + """ + name of close date field for export (for jfrog) + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "export_close_date"]) + + @cached_property + def original_close_date_field(self): + """ + name of the close date field which is not modified by rev scheduling code, etc + + Returns: + str -- field name + """ + return self.config.get("original_close_date_field", get_nested(self.config, ["field_map", "close_date"])) + + @cached_property + def crr_groupby_field(self): + """ + name of group by field for special pivot + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "CRR_BAND_DESCR"]) + + @cached_property + def crr_ceo_fields(self): + return get_nested(self.config, ['crr_ceo_fields', []]) + + @cached_property + def stage_field(self): + """ + name of stage field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "stage"]) + + @cached_property + def type_field(self): + """ + name of stage field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "type"]) + + @cached_property + def stage_trans_field(self): + """ + name of stage field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "stage_trans"]) + + @cached_property + def get_additional_ai_forecast_diff_fields(self): + """ + return SFDCObject Config Key so as to get the CRM resource name of the deal. + + Returns: + str -- SDFCObject Config key + + This config also used in csv export(CS-19586) + """ + return self.config.get('additional_ai_forecast_diff_fields', []) + + @cached_property + def forecast_category_field(self): + """ + name of forecast category field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "forecast_category"]) + + @cached_property + def manager_forecast_category_field(self): + """ + name of manager forecast category field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "manager_forecast_category"]) + + @cached_property + def gvp_forecast_category_field(self): + """ + name of gvp forecast category field for tenant + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "gvp_forecast_category"]) + + @cached_property + def extend_deal_change_for(self): + return self.config.get("extend_deal_change_for", []) + + @cached_property + def oppmap_forecast_category_field(self): + """ + name of oppmap forecast category field for tenant + + Returns: + str -- field name + """ + return get_nested( + self.config, + ["oppmap", "forecast_category"], + get_nested(self.config, ["field_map", "forecast_category"]), + ) + + def oppmap_deal_type_grouping(self, type): + """ + New/Renewal type values for oppmap + + Returns: + str -- type value Ex: new + """ + return get_nested( + self.config, + ["oppmap", "deal_type_grouping"], + { + "new": ["New"], + "renewal": ["Renewal"], + "cross_Sell/upsell/extensions": [ + "Add-On", + "Add-On Business", + "Amendment", + "Existing Business", + "Upgrade", + "Upgrade or downgrade", + ], + }, + )[type] + + @cached_property + def oppmap_deal_type_options(self): + """ + deal type options for oppmap + + Returns: + str -- type value Ex: new + """ + return get_nested( + self.config, + ["oppmap", "oppmap_deal_type_options"], + [ + ("all", "ALL"), + ("new", "New"), + ("renewal", "Renewal"), + ("cross_Sell/upsell/extensions", "Cross Sell/Upsell/Extensions"), + ], + ) + + """ + The score cutoff where a deal is considered risky""" + + @cached_property + def opp_map_score_cutoff(self): + return get_nested(self.config, ["opp_map", "score_cutoff"]) + + @cached_property + def owner_field(self): + """ + Owner Id of the deal + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "owner_id"]) + + @cached_property + def owner_name_field(self): + """ + Owner Name of Deal + + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "opp_owner"]) + + @cached_property + def owner_id_fields(self): + """ + mapping of owner id fields to drilldown if drilldowns, else None + + Returns: + list -- list of tuples of (owner id field, drilldown) + """ + default_owner_id_fields = [(self.owner_field, None)] + owner_id_fields = get_nested( + self.config, ["owner_id_fields"], default_owner_id_fields + ) + if isinstance(owner_id_fields, dict): + logger.warning( + "using old config format for owner_id_fields please switch") + owner_id_fields = [(k, v) for k, v in owner_id_fields.items()] + return owner_id_fields + + @cached_property + def user_data_name(self): + return self.config.get("UserData", "User") + + @cached_property + def user_email_fld(self): + return self.config.get("User_email_fld", "Email") + + @cached_property + def user_level_persona_fields(self): + if self.persona_schemas: + # When persona schemas are enabled, we show user_level fields based on the fields of that particular persona. + # So including the user_level_fields config in the schema itself + personas = sec_context.get_current_user_personas() + ret_val = {} + for persona in personas: + for k, v in ( + self.persona_schemas.get(persona, {}) + .get("user_level_fields", {}) + .items() + ): + if k not in ret_val: + ret_val[k] = v + else: + if type(v) == list: + ret_val[k].extend(v) + elif type(v) == dict: + ret_val[k].update(v) + else: + ret_val[k] = v + return ret_val if ret_val else {} + + @cached_property + def user_level_role_fields(self): + user_role = sec_context.get_current_user_role() + if user_role and self.role_schemas.get(user_role): + return self.role_schemas.get(user_role).get("user_level_fields", {}) + + @cached_property + def user_level_fields(self): + user_level_persona_fields = self.user_level_persona_fields + user_level_role_fields = self.user_level_role_fields + user_level_fields = copy.deepcopy(self.config.get("user_level_fields", {})) + if user_level_persona_fields: + for fld in user_level_persona_fields['fields']: + if fld not in user_level_fields['fields']: + user_level_fields['fields'].append(fld) + for fld in user_level_persona_fields['dlf_fields']: + if fld not in user_level_fields['dlf_fields']: + user_level_fields['dlf_fields'].append(fld) + + if user_level_role_fields: + for fld in user_level_role_fields['fields']: + if fld not in user_level_fields['fields']: + user_level_fields['fields'].append(fld) + for fld in user_level_role_fields['dlf_fields']: + if fld not in user_level_fields['dlf_fields']: + user_level_fields['dlf_fields'].append(fld) + logger.info("final user level fields {}".format(user_level_fields)) + + return user_level_fields + + @cached_property + def pivot_special_user_level_fields(self): + if self.persona_schemas: + # When persona schemas are enabled, we show user_level fields based on the fields of that particular persona. + # So including the user_level_fields config in the schema itself + personas = sec_context.get_current_user_personas() + ret_val = {} + for persona in personas: + for k, v in ( + self.persona_schemas.get(persona, {}) + .get("pivot_special_user_level_fields", {}) + .items() + ): + if k not in ret_val: + ret_val[k] = v + else: + if type(v) == list: + ret_val[k].extend(v) + elif type(v) == dict: + ret_val[k].update(v) + else: + ret_val[k] = v + return ret_val if ret_val else self.config.get("pivot_special_user_level_fields", {}) + + user_role = sec_context.get_current_user_role() + if user_role and self.role_schemas.get(user_role): + return self.role_schemas.get(user_role).get("pivot_special_user_level_fields", + self.config.get("pivot_special_user_level_fields", {})) + + return self.config.get("pivot_special_user_level_fields", {}) + + @cached_property + def user_name_fld(self): + return self.config.get("User_name_fld", "Name") + + @cached_property + def restrict_lead_deals(self): + """ + restrict_lead_deals - True if lead deals are supposed to be excluded from the reports_db + lead deal identification - starts with 00Q + """ + return self.config.get('restrict_lead_deals', False) + + # + # Field Values + # + + @cached_property + def best_case_values(self): + """ + values of forecast category field that make a deal be considered in best case + optional: falls back to DEFAULT_BEST_CASE_VALS + Returns: + list -- [best case values] + """ + return get_nested( + self.config, ["field_values", "best_case"], DEFAULT_BEST_CASE_VALS + ) + + @cached_property + def commit_values(self): + """ + values of forecast category field that make a deal be considered in commit + optional: falls back to DEFAULT_COMMIT_VALS + + Returns: + list -- [commit values] + """ + return get_nested(self.config, ["field_values", "commit"], DEFAULT_COMMIT_VALS) + + @cached_property + def pipeline_values(self): + """ + values of forecast category field that make a deal be considered in pipeline + optional: falls back to DEFAULT_PIPELINE_VALS + + Returns: + list -- [pipeline values] + """ + return get_nested( + self.config, ["field_values", "pipeline"], DEFAULT_PIPELINE_VALS + ) + + @cached_property + def renewal_values(self): + """ + Values of renewal type deal. + optional: falls back to DEFAULT_RENEWAL_VALS + + Returns: + list -- [renewal values] + """ + return get_nested( + self.config, ["field_values", "renewal"], DEFAULT_RENEWAL_VALS + ) + + @cached_property + def most_likely_values(self): + """ + values of forecast category field that make a deal be considered most likely + optional: falls back to DEFAULT_MOST_LIKELY_VALS + + Returns: + list -- [most likely values] + """ + return get_nested( + self.config, ["field_values", + "most_likely"], DEFAULT_MOST_LIKELY_VALS + ) + + @cached_property + def dlf_values(self): + """ + values of forecast category field that make a deal be considered in dlf + optional: falls back to DEFAULT_DLF_VALS + + Returns: + list -- [dlf values] + """ + return get_nested(self.config, ["field_values", "dlf"], DEFAULT_DLF_VALS) + + # + # Total Fields + # + @cached_property + def special_pivot_total_fields(self): + """ + deal amount fields to compute totals for in deals grid + totals are unfiltered + + Returns: + list -- [(label, deal amount field, mongo operation)] + """ + tot_fields = [] + defualt_totals = [('forecast', "$sum"), ("crr_in_fcst", "$sum")] + label_map = { + v: k for k, v in self.raw_crr_schema.get("deal_fields", {}).items() + } + for field_dtls in get_nested( + self.config, ["totals", "crr_total_fields"], defualt_totals + ): + try: + field, op = field_dtls + except ValueError: + (field,) = field_dtls + op = "$sum" + label = label_map.get(field, field) + if 'ACT_CRR_value' in field: + label = "ACT_CRR" + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + tot_fields.append((label, field, op)) + return tot_fields + + @cached_property + def special_pivot_subtotal_fields(self): + """ + deal amount fields to compute subtotals for in deals grid + subtotals are filtered + + Returns: + list -- [(label, deal amount field, mongo operation)] + """ + tot_fields = [] + default_subtotals = [('forecast', "$sum"), ("crr_in_fcst", "$sum")] + label_map = { + v: k for k, v in self.raw_crr_schema.get("deal_fields", {}).items() + } + for field_dtls in get_nested( + self.config, ["totals", "crr_subtotal_fields"], default_subtotals + ): + try: + field, op = field_dtls + except ValueError: + (field,) = field_dtls + op = "$sum" + label = label_map.get(field, field) + if 'ACT_CRR_value' in field: + label = "ACT_CRR" + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + tot_fields.append((label, field, op)) + return tot_fields + + @cached_property + def same_total_subtotals(self): + """ + True if tenant requires totals and subtotals to be equal + CS-13349 - AppAnnie + """ + return self.config.get('same_total_subtotals', False) + + @cached_property + def total_fields(self): + """ + deal amount fields to compute totals for in deals grid + totals are unfiltered + + Returns: + list -- [(label, deal amount field, mongo operation)] + """ + tot_fields = [] + defualt_totals = [(self.amount_field, "$sum"), ("in_fcst", "$sum")] + label_map = { + v: k for k, v in self.raw_schema.get("deal_fields", {}).items() + } + for field_dtls in get_nested( + self.config, ["totals", "total_fields"], defualt_totals + ): + try: + field, op = field_dtls + except ValueError: + (field,) = field_dtls + op = "$sum" + label = label_map.get(field, field) + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + tot_fields.append((label, field, op)) + return tot_fields + + @cached_property + def subtotal_fields(self): + """ + deal amount fields to compute subtotals for in deals grid + subtotals are filtered + + Returns: + list -- [(label, deal amount field, mongo operation)] + """ + tot_fields = [] + default_subtotals = [(self.amount_field, "$sum"), ("in_fcst", "$sum")] + label_map = { + v: k for k, v in self.raw_schema.get("deal_fields", {}).items() + } + for field_dtls in get_nested( + self.config, ["totals", "subtotal_fields"], default_subtotals + ): + try: + field, op = field_dtls + except ValueError: + (field,) = field_dtls + op = "$sum" + label = label_map.get(field, field) + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + tot_fields.append((label, field, op)) + return tot_fields + + # + # Display Schema + # + def get_field_format(self, field, pivot=None): + try: + if pivot in self.config.get('special_pivot', []): + return next( + x["fmt"] for x in self.account_fields_config if x["field"] == field + ) + else: + return next( + x["fmt"] for x in self.deal_fields_config if x["field"] == field + ) + except StopIteration: + return None + + def get_field_label(self, field, pivot=None): + try: + if pivot in self.config.get('special_pivot', []): + return next( + x["label"] for x in self.account_fields_config if x["field"] == field + ) + else: + return next( + x["label"] for x in self.deal_fields_config if x["field"] == field + ) + except StopIteration: + return field + + @cached_property + def account_fields_config(self): + """ + all accounts fields available to UI, with description of how to format + label them + + Returns: + list -- [{'field': field, 'fmt': fmt, 'label': label}] + """ + return self.raw_crr_schema.get("deal", []) + + @cached_property + def deal_fields_config(self): + """ + all deal fields available to UI, with description of how to format + label them + + Returns: + list -- [{'field': field, 'fmt': fmt, 'label': label}] + """ + return self.raw_schema.get("deal", []) + + @cached_property + def trimmed_deal_fields_config(self): + fields = {} + for field_config in self.deal_fields_config: + fields.update({field_config.get('field'): field_config}) + return fields.values() + + @cached_property + def special_amounts(self): + """ + Special Amount fields which will be useful to add special amount fields and + + Returns: + list -- {'amount_field': "amount_label"} + """ + return self.raw_schema.get("special_amounts", {}) + + @cached_property + def secondary_deal_fields(self): + """ + secondary deal fields that dont appear in deals grid + + Returns: + set -- {fields} + """ + return { + field_dtls["field"] + for field_dtls in self.deal_fields_config + if field_dtls.get("secondary") + } + + def deal_ue_fields(self, pivot_schema="schema"): + """ + user editable deal fields + + Returns: + set -- {fields} + """ + return [ + field_dtls["field"] + for field_dtls in get_nested(self.config, [pivot_schema, "deal"], []) + if field_dtls.get("user_edit") + ] + + def pivot_secondary_deal_fields(self, pivot_schema="schema"): + """ + pivot secondary deal fields that dont appear in deals grid + + Returns: + set -- {fields} + """ + return { + field_dtls["field"] + for field_dtls in get_nested(self.config, [pivot_schema, "deal"], []) + if field_dtls.get("secondary") + } + + @cached_property + def primary_account_fields(self): + """ + primary account fields that appears in accounts grid + + Returns: + set -- {fields} + """ + return { + field_dtls["field"] + for field_dtls in self.account_fields_config + if field_dtls.get("primary") + } + + @cached_property + def primary_deal_fields(self): + """ + primary deal fields that appears in deals grid + + Returns: + set -- {fields} + """ + return { + field_dtls["field"] + for field_dtls in self.deal_fields_config + if field_dtls.get("primary") + } + + @cached_property + def gateway_deal_fields(self): + """ + secondary deal fields that dont appear in deals grid + + Returns: + set -- {fields} + """ + return { + field_dtls["field"] + for field_dtls in self.deal_fields_config + if field_dtls.get("gateway") + } + + @cached_property + def special_gateway_deal_fields(self): + """ + secondary deal fields that dont appear in deals grid + + Returns: + set -- {fields} + """ + return get_nested( + self.config, ["gateway_schema", "special_gateway_deal_fields"], {} + ) + + @cached_property + def gateway_dlf_expanded(self): + """ + boolean to activate expanded dlf fields with node level information + + Returns: + boolean -- + """ + return get_nested( + self.config, ["gateway_schema", "gateway_dlf_expanded"], False + ) + + @cached_property + def gateway_dlf_expanded_field(self): + """ + expanded dlf field with node level information in {label: field} format + + Returns: + dict -- dlf fields with node level information + default - {'dlfs': 'dlfs'} + """ + return get_nested( + self.config, ["gateway_schema", "gateway_dlf_expanded_field"], {'dlfs': 'dlfs'} + ) + + @cached_property + def special_deal_fields(self): + """ + deal fields that get called out in deal card + + Returns: + list -- [(deal field, fe key, label, format)] + """ + return self.raw_schema.get("special_deal_fields", []) + + def pivot_special_deal_fields(self, schema): + """ + deal fields that get called out in deal card + + Returns: + list -- [(deal field, fe key, label, format)] + """ + return get_nested(self.config, [schema, "special_deal_fields"], []) + + @cached_property + def default_hidden_fields(self): + """ + deal fields that get called out in deal card + + Returns: + list -- [(deal field, fe key, label, format)] + """ + return self.raw_schema.get("default_hidden_fields", []) + + @cached_property + def custom_layout_fields(self): + """ + deal fields that get called out in deal card + + Returns: + list -- [(deal field, fe key, label, format)] + """ + return self.raw_schema.get("deal") + + @cached_property + def deal_fields(self): + """ + deal fields to display in deals grid mapped to their tenant specific field names + + Returns: + dict -- {'OpportunityName': 'opp_name'} + """ + + return self._deal_fields() + + @cached_property + def formula_driven_fields(self): + """ + fields that are formula driven and formula + + Returns: + dictionary where key is field_name and value have formula and source + for e.g.: + {"ACVProb": {"formula": "a * b" + "source": {"a": "Amount", + "b": "Probability" }}} + """ + return self.config.get("formula_driven_fields", {}) + + def _deal_fields(self, gateway_call=False, pivot_schema=None, formula_driven_fields=[], segment=None): + d_fields = { + "alert": "alert", + "dealalert": "dealalert", + } # HACK: get alert into deal ... + + secondary_deal_fields = self.secondary_deal_fields + if pivot_schema: + schema = self.config.get(pivot_schema) + secondary_deal_fields = self.pivot_secondary_deal_fields( + pivot_schema=pivot_schema + ) + else: + schema = self.raw_schema + + for label, field in schema.get("deal_fields", {}).items(): + if label in secondary_deal_fields: + if (gateway_call and label in self.gateway_deal_fields) or label in formula_driven_fields: + pass + else: + continue + if field in self.dlf_fields: + field = ".".join(["dlf", field]) + d_fields[label] = field + if gateway_call: + d_fields.update(self.special_gateway_deal_fields) + # logger.info("deal fields %s" % d_fields) + if "segment_schema" in self.config: + segment_schema = self.config.get('segment_schema', {}) + if segment in segment_schema: + deal_fields = segment_schema[segment]["deal_fields"] + for key, value in deal_fields.items(): + d_fields[key] = value + + return d_fields + + @cached_property + def all_account_fields(self): + return self.raw_crr_schema.get("deal_fields", {}) + + def gateway_fields_from_schema(self, schema='schema'): + schema = self.config.get(schema) + deal_fields_config = schema.get("deal", []) + + gateway_deal_fields = [] + for field_dtls in deal_fields_config: + if field_dtls.get('gateway'): + field = field_dtls['field'] + if field in self.dlf_fields: + field = ".".join(["dlf", field]) + gateway_deal_fields.append([field_dtls['label'], field, field_dtls['fmt']]) + return gateway_deal_fields + + @cached_property + def all_deal_fields(self): + d_fields = { + "alert": "alert", + "dealalert": "dealalert", + } # HACK: get alert into deal ... + for label, field in self.raw_schema.get("deal_fields", {}).items(): + if field in self.dlf_fields: + field = ".".join(["dlf", field]) + d_fields[label] = field + return d_fields + + @cached_property + def filter_priority(self): + return self.config.get("filter_priority", []) + + @cached_property + def card_deal_fields(self): + """ + deal fields to display in deal card mapped to their tenant specific field names + optional: falls back to standard deal_fields + + Returns: + dict -- {'OpportunityName': 'opp_name'} + """ + d_fields = { + "alert": "alert", + "dealalert": "dealalert", + } # HACK: get alert into deal ... + for label, field in self.raw_schema.get( + "card_deal_fields", self.raw_schema.get("deal_fields", {}) + ).items(): + if field in self.dlf_fields: + field = ".".join(["dlf", field]) + if field not in self.raw_schema.get( + "excluded_from_deal_card", ["__comment__"] + ): + d_fields[label] = field + return d_fields + + def card_deal_fields_config(self): + """ + all deal card fields available to UI be it combined/irrespective od deal grid, with description of how to format + label them + + Returns: + list -- [{'field': field, 'fmt': fmt, 'label': label}] + """ + return self.raw_schema.get('deal_card', []) if self.raw_schema.get('deal_card', []) else self.deal_fields_config + + def pivot_deal_fields_config(self, pivot_schema="schema"): + """ + all pivot deal fields available to UI, with description of how to format + label them + + Returns: + list -- [{'field': field, 'fmt': fmt, 'label': label}] + """ + return get_nested(self.config, [pivot_schema, "deal"], []) + + def pivot_card_deal_fields(self, pivot_schema="schema"): + """ + deal fields to display in deal card mapped to their tenant specific field names + optional: falls back to standard deal_fields + + Returns: + dict -- {'OpportunityName': 'opp_name'} + """ + d_fields = { + "alert": "alert", + "dealalert": "dealalert", + } # HACK: get alert into deal ... + for label, field in get_nested( + self.config, + [pivot_schema, "card_deal_fields"], + get_nested(self.config, [pivot_schema, "deal_fields"], {}), + ).items(): + if field in self.dlf_fields: + field = ".".join(["dlf", field]) + if field not in get_nested( + self.config, [pivot_schema, "excluded_from_deal_card"], [ + "__comment__"] + ): + d_fields[label] = field + return d_fields + + def crr_card_fields(self): + return self.config.get('crr_card_fields', {}) + + def crr_card_special_fields(self): + return self.config.get('crr_card_special_fields', {}) + + def crr_card_graph_fields(self): + return self.config.get('crr_card_graph_fields', {}) + + @cached_property + def export_hierarchy_fields(self): + return self.config.get('export_hierarchy_fields', True) + + def export_deal_fields(self, schema="schema", special_pivot=False): + """ + deal fields to display in deals export to their tenant specific field names + optional: falls back to standard deal_fields + + Returns: + list -- [(label, key, fmt) ... ] + """ + d_fields = [] + pivot = schema.split('_')[0] + id = '__id__' if pivot in self.config.get('not_deals_tenant', {}).get('special_pivot', []) else 'opp_id' + if schema != 'schema' and schema in self.config: + pivot_schema = self.config.get(schema) + export_fields = pivot_schema.get("export_deal_fields", []) + if export_fields: + fields_order = [x[0] for x in export_fields] + export_fields = {label: field for label, field in export_fields} + else: + fields_order = [x["field"] for x in pivot_schema.get("deal", [])] + export_fields = pivot_schema.get("deal_fields", {}) + else: + export_fields = self.raw_schema.get("export_deal_fields", []) + if export_fields: + fields_order = [x[0] for x in export_fields] + export_fields = {label: field for label, field in export_fields} + else: + fields_order = [x["field"] for x in self.deal_fields_config] + export_fields = self.raw_schema.get("deal_fields", {}) + + for standard_field, db_field in export_fields.items(): + if standard_field[:2] == "__" and standard_field != '__comment__': + continue + label = self.get_field_label(standard_field, pivot=pivot) + fmt = self.get_field_format(standard_field, pivot=pivot) + if db_field in self.dlf_fields: + db_field = ".".join(["dlf", db_field]) + d_fields.extend( + [ + (label + " Status", standard_field, db_field, "dlf"), + ] + ) + if self.dlf_mode.get(db_field.split(".")[-1], None) != "N": + d_fields.append( + (label, standard_field, db_field, "dlf_amount")) + else: + d_fields.append((label, standard_field, db_field, fmt)) + + field_indices = {x: i for (i, x) in enumerate(fields_order)} + # Sort based on standard field name. + ordered_fields = sorted( + [fld for fld in d_fields if fld[1] in fields_order], + key=lambda x: field_indices[x[1]], + ) + all_fields = ordered_fields + [ + fld for fld in d_fields if fld[1] not in fields_order + ] + if special_pivot: + hierarchy_field_config = ('Hierarchy', '__segs', "list") + return [("Id", id, "str"), hierarchy_field_config] + [(label, db_field, fmt) for + (label, standard_field, db_field, fmt) in all_fields] + if self.export_hierarchy_fields: + hierarchy_field_config = ('Hierarchy', 'drilldown_list', "list") + return [("Id", id, "str"), hierarchy_field_config] + [(label, db_field, fmt) for + (label, standard_field, db_field, fmt) in all_fields] + return [("Id", id, "str")] + [(label, db_field, fmt) for + (label, standard_field, db_field, fmt) in all_fields] + + def export_pdf_deal_fields(self): + export_pdf_fields = self.raw_schema.get("export_pdf_deal_fields", []) + if not export_pdf_fields: + return [] + export_fields = [label for label, _, _ in self.export_deal_fields()] + return [label for label in export_pdf_fields if label in export_fields] + + @cached_property + def reload_post_writeback(self): + if 'reload_post_writeback' not in self.config.get('schema', {}): + return None + return self.config.get('schema').get('reload_post_writeback') + + @cached_property + def export_deal_fields_format(self): + if 'export_deal_fields_format' not in self.config.get('schema', {}): + return None + return self.config.get('schema').get('export_deal_fields_format') + + @cached_property + def pull_in_deal_fields(self): + """ + deal fields to display in pull in deals grid to their tenant specific field names + optional: falls back to standard deal_fields + + Returns: + dict -- {'OpportunityName': 'opp_name'} + """ + d_fields = {"__fav__": "__fav__"} + deal_fields_map = self.raw_schema.get("deal_fields", {}) + for label in PULL_IN_LIST: + d_fields[label] = deal_fields_map.get(label, label) + return d_fields + + @cached_property + def pull_in_fields_order(self): + """ + Force pull in deals columns to have an order + + Returns: + dict -- {'label': labels_of_order, + 'fields': fields_used_in_deals} + """ + return { + "label": PULL_IN_LIST, + "fields": [self.pull_in_deal_fields[l] for l in PULL_IN_LIST], + } + + @cached_property + def opp_template(self): + """ + template to make link to source crm system opportunity + + Returns: + str -- url stub + """ + try: + return sec_context.details.get_config("forecast", "tenant", {}).get( + "opportunity_link", "https://salesforce.com/{oppid}" + ) + except AttributeError: + return "https://salesforce.com/{oppid}" + + # + # Filters + # + @cached_property + def open_filter_criteria(self): + """ + mongo db filter criteria for open deals + + Returns: + dict -- {mongo db criteria} + """ + return fetch_filter([self._open_filter_id], self, db=self.db) + + @cached_property + def open_filter_raw(self): + """ + aviso filter syntax criteria for open deal + + Returns: + list -- [{'op': 'has', 'key': 'amt'}] + """ + return fetch_filter([self._open_filter_id], self, filter_type="raw", db=self.db) + + def open_filter(self, deal): + """ + check if a deal is open or not + + Arguments: + deal {dict} -- deal record + + Returns: + bool -- True if open, False if closed + """ + return self._py_open_func(deal, None, None) + + def won_filter(self, deal): + """ + check if a deal is won or not + + Arguments: + deal {dict} -- deal record + + Returns: + bool -- True if open, False if closed + """ + return self._py_won_func(deal, None, None) + + def lost_filter(self, deal): + """ + check if a deal is open or not + + Arguments: + deal {dict} -- deal record + + Returns: + bool -- True if open, False if closed + """ + return self._py_lost_func(deal, None, None) + + @cached_property + def opp_name_field(self): + """ + name of opportunity name field for tenant + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "opp_name"]) + + @cached_property + def _open_filter_id(self): + return get_nested(self.config, ["filters", "open_filter"]) + + @cached_property + def _favourites_filter_id(self): + return get_nested(self.config, ["filters", "favourites_filter"], 'favourites') + + @cached_property + def _py_open_func(self): + return fetch_filter( + [self._open_filter_id], self, filter_type="python", db=self.db + ) + + @cached_property + def _py_won_func(self): + return fetch_filter( + [self._won_filter_id], self, filter_type="python", db=self.db + ) + + @cached_property + def _py_lost_func(self): + return fetch_filter( + [self._lost_filter_id], self, filter_type="python", db=self.db + ) + + @cached_property + def _all_filter_criteria(self): + """ + mongo db filter criteria for won deals + + Returns: + dict -- {mongo db criteria} + """ + return fetch_filter([self._alldeals_filter_id], self, db=self.db) + + @cached_property + def _alldeals_filter_id(self): + return get_nested(self.config, ["filters", "all_filter"], 'All') + + @cached_property + def multiple_filter_apply_or(self): + return self.config.get("multiple_filter_apply_or", False) + + @cached_property + def ai_driven_deals_buckets(self): + default_expressions = {"Pullins": {"color": '#2bccff'}, + "Aviso AI- Predicted Wins": {"expr": "win_prob_threshold < win_prob < 1.0", + "color": '#800080'}} + return self.config.get("ai_driven_deals_buckets", default_expressions) + + @cached_property + def default_currency(self): + """ + Returns default currency set for the tenant to serve in notifications. + """ + td = sec_context.details + return td.get_config('forecast', 'tenant', {}).get('notifications_default_currency') or '$' + + # event-based nudge configs + @cached_property + def event_based_nudge_config(self): + """ + ... Set nudge test_mode=True/False for testing purpose + ... Set nudge enable=True/False to enable/disable event subscription + """ + config_params = { + "debug": False, + "eb_score_hist_dip_nudge": {'test_mode': True, 'test_email': 'amit.khachane@aviso.com', + 'enable': False}, + "eb_scenario_nudge": {'test_mode': True, 'test_email': 'amit.khachane@aviso.com', 'enable': False}, + "eb_dlf_nudge": {'test_mode': True, 'test_email': 'amit.khachane@aviso.com', 'enable': False}, + "eb_pulledin_nudge": {'test_mode': True, 'test_email': 'amit.khachane@aviso.com', 'enable': False, + "criteria": {'pulledin': True, 'terminal_fate': 'N/A'}}, + } + return get_nested( + self.config, ['nudge_config', 'event_based_nudge_config'], + config_params + ) + + # Ringcentral Nudges Filters + def nudge_filter_criteria(self, filter_id='Open Deals', root_node=None): + """ + mongo db filter criteria for nudge deals + Returns: + dict -- {mongo db criteria} + """ + # replicable_key = config.config.get('schema', {}).get('deal_fields', {}).get('Amount') + config = DealConfig() + if root_node is None: + logger.exception("Root node not found, passing empty criteria") + return {} + criteria = fetch_filter([filter_id], config, root_node=root_node) + return criteria + + # --x--Ringcentral Nudges Filters--x-- + + # + # DLF Config + # + @cached_property + def primary_dlf_field(self): + """ + main dlf field to use for other features like opp map and deal changes + + Returns: + str -- dlf field name + """ + try: + return next( + k for k, v in self.config.get("dlf", {}).items() if "primary" in v + ) + except StopIteration: + return None + + @cached_property + def analytics_dlf_field(self): + """ + dlf field to use in analytics field + + Returns: + str -- dlf field name + """ + try: + return next( + k + for k, v in self.config.get("dlf", {}).items() + if "use_in_pipeline_analytics" in v + ) + except StopIteration: + return None + + @cached_property + def dlf_mode(self): + """ + mapping of dlf field to dlf mode + 'N': no amount + 'O': optional amount + + Returns: + dict -- {'in_fcst': 'N'} + """ + return { + field: field_config["mode"] + for field, field_config in self.config.get("dlf", {}).items() + } + + @cached_property + def dlf(self): + """ + return all dlf + """ + return self.config.get("dlf", {}) + + @cached_property + def dlf_reports(self): + return self.config.get("dlf_reports", False) + + @cached_property + def top_deals_count(self): + """ + return how many deals should be shown in top_deals section in dashboard + """ + return self.config.get("top_deals_count", 30) + + @cached_property + def dlf_fields(self): + """ + all dlf fields + + Returns: + list -- [dlf field] + """ + return self.config.get("dlf", {}).keys() + + @cached_property + def oppmap_dlf_field(self): + return get_nested( + self.config, + ["oppmap", "dlf_field"], + 'in_fcst' + ) + + @cached_property + def dlf_mismatch_default(self): + """ + dlf_mismatch_default: True if we want to show mismatch wrt defaults + """ + return self.config.get("dlf_mismatch_default", False) + + @cached_property + def dlf_amount_field(self): + """ + mapping of dlf field to deal amount field used to back dlf + optional: falls back to tenants amount field + + Returns: + dict -- {'in_fcst': 'amount'} + """ + return { + field: field_config.get("amount_field", self.amount_field) + for field, field_config in self.config.get("dlf", {}).items() + } + + @cached_property + def change_dlf_with_writeback(self): + """ + mapping of field name with it's values. During writeback if there is change in field name + and value matches. DLF is toggled true + + Returns: + dict -- {'ManagerForecastCategory':['Commit','Best Case','Closed Won']} + """ + return self.config.get('change_dlf_with_writeback', []) + + @cached_property + def dlf_secondary_amount_field(self): + """ + mapping of dlf field to deal secondary amount field used to back dlf + + Returns: + dict -- {'in_fcst': 'amount'} + """ + return { + field: field_config.get("secondary_amount_field", "") + for field, field_config in self.config.get("dlf", {}).items() + } + + @cached_property + def multi_dlf(self): + return get_nested(self.config, ["multi_dlf"], False) + + @cached_property + def dlf_adorn_fields(self): + """ + mapping of dlf field to extra deal fields to adorn each on each dlf to see state at time forecast was made + + Returns: + dict -- {'in_fcst': {'win_prob': 'win_prob',}} + """ + return { + field: field_config.get( + "adorn_field", self._default_dlf_adorn_fields) + for field, field_config in self.config.get("dlf", {}).items() + } + + @cached_property + def dlf_crr_adorn_fields(self): + """ + mapping of dlf field to extra deal fields to adorn each on each dlf to see state at time forecast was made + + Returns: + dict -- {'in_fcst': {'win_prob': 'win_prob',}} + """ + return { + field: field_config.get( + "adorn_field", self._default_crr_dlf_adorn_fields) + for field, field_config in self.config.get("dlf", {}).items() + } + + @cached_property + def _default_dlf_adorn_fields(self): + return { + "score": "win_prob", + "stage": self.stage_field, + "forecastcategory": self.forecast_category_field, + "raw_amt": self.amount_field, + "closedate": self.close_date_field, + } + + @cached_property + def _default_crr_dlf_adorn_fields(self): + return { + "score": "win_prob", + "stage": self.stage_field, + "forecastcategory": self.forecast_category_field, + "raw_amt": self.crr_amount_field, + "closedate": self.close_date_field, + } + + def deals_dlf_rendered_config(self, node): + """ + dlf config rendered for consumption by front end + + Returns: + dict -- {dlf config} + """ + dlf_config = {} + for field, dtls in self.config.get("dlf", {}).items(): + mode = dtls["mode"] + dlf_config[field] = { + "has_amt": mode != "N", + "amt_editable": mode == "O", + "option_editable": True, + } + try: + if get_node_depth(node) >= dtls["hide_at_depth"]: + dlf_config[field]["hide"] = True + except: + pass + if "options" in dtls: + dlf_config[field]["options"] = dtls["options"] + # TODO: hide at depth grossness + return dlf_config + + @cached_property + def dlf_rendered_config(self): + """ + dlf config rendered for consumption by front end + + Returns: + dict -- {dlf config} + """ + dlf_config = {} + for field, dtls in self.config.get("dlf", {}).items(): + mode = dtls["mode"] + dlf_config[field] = { + "has_amt": mode != "N", + "amt_editable": mode == "O", + "option_editable": True, + } + if "options" in dtls: + dlf_config[field]["options"] = dtls["options"] + # TODO: hide at depth grossness + return dlf_config + + def dlf_locked_filter(self, deal, field): + """ + check if a deal is locked in or out of forecast + + Arguments: + deal {dict} -- deal record + field {str} -- dlf field name + + Returns: + bool -- True if locked in, False if locked out, None if not locked + """ + for state, filter_func in self._dlf_py_locked_func.get(field, {}).items(): + if filter_func(deal, None, None): + return state + + def ue_locked_filter(self, deal, field, pivot_schema='schema'): + + for state, filter_func in self._user_edit_dlf_py_locked_func(pivot_schema=pivot_schema).get(field, + {}).items(): + if filter_func(deal, None, None): + return state + return False + + def dlf_default_filter(self, deal, field): + """ + check if a deal if default to in our out of forecas + + Arguments: + deal {dict} -- deal record + field {str} -- dlf field name + + Returns: + bool -- True if default in, False if default out + """ + for state, filter_func in self._dlf_py_default_func.get(field, {}).items(): + if filter_func(deal, None, None): + return state + return self._dlf_default_values[field] + + @cached_property + def favourites_filter_criteria(self): + """ + mongo db filter criteria for favourite deals + + Returns: + dict -- {mongo db criteria} + """ + return fetch_filter([self._favourites_filter_id], self, db=self.db) + + @cached_property + def _dlf_default_values(self): + default_values = {} + for field, field_config in self.config.get("dlf").items(): + try: + default_values[field] = field_config["options"][0]["val"] + except (KeyError, IndexError): + default_values[field] = False + return default_values + + @cached_property + def _dlf_py_locked_func(self): + return { + field: { + _convert_state(state): fetch_filter( + filter_ids, self, filter_type="python", db=self.db + ) + for state, filter_ids in field_config.get( + "locked_filters", {} + ).items() + } + for field, field_config in self.config.get("dlf").items() + } + + def _user_edit_dlf_py_locked_func(self, pivot_schema=None): + deal_fields = self.pivot_deal_fields_config(pivot_schema=pivot_schema) + locked_filter_py = {} + for deal_field in deal_fields: + is_ue_field = deal_field.get('user_edit', False) + if is_ue_field: + locked_filters = deal_field.get('locked_filters', {}) + if locked_filters: + locked_filter_py.update({ + deal_field['field']: { + _convert_state(state): fetch_filter( + filter_ids, self, filter_type="python", db=self.db + ) + for state, filter_ids in locked_filters.items() + } + }) + + return locked_filter_py + + @cached_property + def _dlf_py_default_func(self): + return { + field: { + _convert_state(state): fetch_filter( + filter_ids, self, filter_type="python", db=self.db + ) + for state, filter_ids in field_config.get( + "default_filters", {} + ).items() + } + for field, field_config in self.config.get("dlf").items() + } + + # + # Dashboard Config + # Adaptive metrics config + @cached_property + def adaptive_metrics_categories(self): + cats = {} + am_categories = ( + self.config["dashboard"] + .get("adaptive_metrics_categories", {}) + .get("categories", {}) + ) + + for category in am_categories: + cats[category] = [] + + for (field_name, field_filter, field_tot_fields) in am_categories[category]: + + if "handler" in field_filter: + cats[category].append( + (field_name, field_filter, field_tot_fields)) + elif "get_ratio" in field_filter: + cats[category].append( + (field_name, field_filter, field_tot_fields)) + else: + try: + field = self.amount_field + label, op = field_tot_fields + if ( + "amount_fields" + in self.config["dashboard"]["adaptive_metrics_categories"] + ): + if ( + category + in self.config["dashboard"][ + "adaptive_metrics_categories" + ]["amount_fields"] + ): + if ( + field_name + in self.config["dashboard"][ + "adaptive_metrics_categories" + ]["amount_fields"][category] + ): + field = self.config["dashboard"][ + "adaptive_metrics_categories" + ]["amount_fields"][category][field_name] + + except ValueError: + (field,) = field_tot_fields + op = "$sum" + + cats[category].append( + ( + field_name, + parse_filters(field_filter, self), + [[label, field, op]], + ) + ) + + return cats + + @cached_property + def adaptive_metrics_additional_handler_filters(self): + am_handlers = ( + self.config["dashboard"] + .get("adaptive_metrics_categories", {}) + .get("additional_handler_filters", {}) + ) + for handler in am_handlers: + am_handlers[handler] = parse_filters(am_handlers[handler], self) + return am_handlers + + @cached_property + def adaptive_metrics_additinal_info(self): + cats = {} + am_additinal_info = ( + self.config["dashboard"] + .get("adaptive_metrics_categories", {}) + .get("additinal_info_categories", {}) + ) + + for category in am_additinal_info: + cats[category] = defaultdict(dict) + for (field_name, additinal_info_filter) in am_additinal_info[category]: + cats[category][field_name] = defaultdict(dict) + if "h" in additinal_info_filter: + cats[category][field_name]["past_count"] = additinal_info_filter[ + "h" + ] + if "f" in additinal_info_filter: + cats[category][field_name]["future_count"] = additinal_info_filter[ + "f" + ] + if "d" in additinal_info_filter: + cats[category][field_name][ + "difference_with" + ] = additinal_info_filter["d"] + if "tt" in additinal_info_filter: + cats[category][field_name]["tooltip"] = { + "type": "text", + "text": additinal_info_filter["tt"], + } + if "sl" in additinal_info_filter: + cats[category][field_name]["sublabel"] = additinal_info_filter["sl"] + + return cats + + @cached_property + def adaptive_metrics_additional_filters(self): + cats = {} + am_additinal_info = ( + self.config["dashboard"] + .get("adaptive_metrics_categories", {}) + .get("additional_filters", {}) + ) + + for category in am_additinal_info: + cats[category] = defaultdict(dict) + for (field_name, additinal_info_filter) in am_additinal_info[category]: + cats[category][field_name] = defaultdict(dict) + if "close_date" in additinal_info_filter: + cats[category][field_name]["close_date_in"] = additinal_info_filter[ + "close_date" + ] + if "created_date" in additinal_info_filter: + cats[category][field_name]["created_date"] = additinal_info_filter[ + "created_date" + ] + + return cats + + @cached_property + def adaptive_metrics_views_order(self): + order_info = ( + self.config["dashboard"] + .get("adaptive_metrics_categories", {}) + .get("views_order", []) + ) + return order_info + + @cached_property + def adaptive_metrics_cache_level(self): + cache_level = ( + self.config["dashboard"] + .get("adaptive_metrics_cache_level", 2) + ) + return cache_level + + @cached_property + def leaderboard_cache_level(self): + cache_level = ( + self.config["dashboard"] + .get("leaderboard_cache_level", 2) + ) + return cache_level + + @cached_property + def adaptive_metrics_views_format(self): + cats = {} + format_info = ( + self.config["dashboard"] + .get("adaptive_metrics_categories", {}) + .get("views_format", {}) + ) + for category in format_info: + cats[category] = defaultdict(dict) + if "fmt" in format_info[category]: + cats[category]["format"] = format_info[category]["fmt"] + return cats + + @cached_property + def adaptive_metrics_close_date_aware(self): + cats = {} + close_date_aware_info = ( + self.config["dashboard"] + .get("adaptive_metrics_categories", {}) + .get("close_date_aware", {}) + ) + for category in close_date_aware_info: + cats[category] = [] + for field_name in close_date_aware_info[category]: + cats[category].append(field_name) + return cats + + # Coaching Leaderboard config + @cached_property + def coaching_leaderboard_categories(self): + cats = {} + cl_categories = ( + self.config["dashboard"] + .get("coaching_leaderboard_categories", {}) + .get("categories", []) + ) + + # for category in cl_categories: + # cats[category] = [] + # + # for badge_name in cl_categories[category]: + # cats[category].append(badge_name) + + return cl_categories + + @cached_property + def tenant_diff_node(self): + cl_categories = ['lume.com', 'lumenbackup.com', 'netapp_pm.com', ] + + return cl_categories + + @cached_property + def tenant_diff_owner(self): + cl_categories = ['netapp.com'] + + return cl_categories + + @cached_property + def tenant_node_rename(self): + cl_categories = ['cisco.com'] + + return cl_categories + + @cached_property + def pqr_data(self): + """ + value for number of quarters to be considered for calculating time_threshold in deal_velocity badge in coaching + leaderboard. + """ + return self.config.get("dashboard", {}).get("pqr_data", {}) + + @cached_property + def deal_velocity_past_n_qtrs(self): + """ + value for number of quarters to be considered for calculating time_threshold in deal_velocity badge in coaching + leaderboard. + """ + return self.config.get("dashboard", {}).get("deal_velocity_past_n_qtrs", 4) + + # Deals Config + @cached_property + def deal_categories(self): + cats = [] + for cat_label, cat_filter, cat_tot_fields in get_nested( + self.config, ["dashboard", "deal_categories", "categories"], [] + ): + try: + label, op = cat_tot_fields + if self.config["dashboard"]["deal_categories"].get("amount_fields"): + field = self.config["dashboard"]["deal_categories"][ + "amount_fields" + ][cat_label] + else: + field = self.amount_field + except ValueError: + (field,) = cat_tot_fields + op = "$sum" + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + cats.append( + (cat_label, parse_filters( + cat_filter, self), [[label, field, op]]) + ) + return cats + + @cached_property + def totals_segmented_view(self): + """ + if this is set to true for segmented tenant, + subtotals and total will have same value on selection + of particular segment deals + """ + return self.config.get("totals_segmented_view", False) + + # + # Dashboard Category sort fields + # + @cached_property + def category_amount_fields(self): + cat_amt_fields = {} + if self.config["dashboard"]["deal_categories"].get("amount_fields"): + for cat_label, cat_filter, cat_tot_fields in get_nested( + self.config, ["dashboard", "deal_categories", "categories"], [] + ): + cat_amt_fields[cat_label] = self.config["dashboard"]["deal_categories"][ + "amount_fields" + ][cat_label] + return cat_amt_fields + else: + return None + + # + # covid Dashboard Config + # + + @cached_property + def covid_deal_categories(self): + cats = [] + for cat_label, cat_filter, cat_tot_fields in get_nested( + self.config, ["dashboard", + "covid_deal_categories", "categories"], [] + ): + try: + label, op = cat_tot_fields + field = self.amount_field + except ValueError: + (field,) = cat_tot_fields + op = "$sum" + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + cats.append( + (cat_label, parse_filters( + cat_filter, self), [[label, field, op]]) + ) + return cats + + @cached_property + def how_it_changed(self): + return self.config.get('new_home_page', {}).get('how_it_changed', []) + + @cached_property + def pipeline_quality_categories(self): + return self.config.get('new_home_page', {}).get('pipeline_quality_categories', []) + + @cached_property + def deals_stage_map_cached(self): + return self.config.get('deals_stage_map_cached', False) + + @cached_property + def pipe_dev_gbm_fields(self): + default_pipe_dev_gbm_fields = ['node', 'period', '__segs', 'forecast'] + return self.config.get('pipe_dev_gbm_fields', default_pipe_dev_gbm_fields) + + @cached_property + def deal_changes_categories(self): + cats = [] + for cat_label, cat_filter, cat_tot_fields in get_nested( + self.config, + ["dashboard", "deal_changes", "categories"], + self._default_deal_changes["categories"], + ): + try: + field, op = cat_tot_fields + except ValueError: + (field,) = cat_tot_fields + op = "$sum" + label = field + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + cats.append( + ( + cat_label, + parse_filters(cat_filter, self, hier_aware=False), + [[label, field, op]], + ) + ) + return cats + + @cached_property + def deal_changes_categories_won(self): + cats = [] + for cat_label, cat_filter, cat_tot_fields in get_nested( + self.config, + ["dashboard", "deal_changes", "categories"], + self._default_deal_changes["categories"], + ): + if cat_label == "Won": + try: + field, op = cat_tot_fields + except ValueError: + (field,) = cat_tot_fields + op = "$sum" + label = field + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + cats.append( + ( + cat_label, + parse_filters(cat_filter, self, hier_aware=False), + [[label, field, op]], + ) + ) + return cats + + @cached_property + def deal_changes_leaderboards(self): + lbs = get_nested( + self.config, + ["dashboard", "deal_changes", "leaderboards"], + self._default_deal_changes["leaderboards"], + ) + for lb, lb_dtls in lbs.items(): + if "schema" not in lb_dtls: + lb_dtls["schema"] = self.dashboard_deal_format + if "total_name" not in lb_dtls: + lb_dtls["total_name"] = "Amount" + if "arrow" not in lb_dtls: + lb_dtls["arrow"] = False + return lbs + + @cached_property + def deal_changes_leaderboards_label_performer_desc(self): + return get_nested( + self.config, ["dashboard", "deal_changes", "leaderboards", "desc"], "amount" + ) + + @cached_property + def deal_changes_pipe_field(self): + return get_nested( + self.config, ["dashboard", "deal_changes", + "pipe_field"], "tot_won_and_fcst" + ) + + @cached_property + def deal_changes_categories_order(self): + return get_nested( + self.config, + ["dashboard", "deal_changes", "categories_order"], + self._default_deal_changes["categories_order"], + ) + + @cached_property + def deal_changes_leaderboards_order(self): + return get_nested( + self.config, + ["dashboard", "deal_changes", "leaderboards_order"], + self._default_deal_changes["leaderboards_order"], + ) + + @cached_property + def deal_changes_default_key(self): + return get_nested( + self.config, + ["dashboard", "deal_changes", "default_key"], + self._default_deal_changes["default_key"], + ) + + @cached_property + def account_categories(self): + """ + for top account dashboard feature + the filters + labels to split dealts out by + + Returns: + list -- [(cat label, {cat filter}, [cat sum fields]) for each category] + """ + cats = [] + for cat_label, cat_filter, cat_tot_fields in get_nested( + self.config, ["dashboard", "accounts", "categories"] + ): + try: + field, op = cat_tot_fields + except ValueError: + (field,) = cat_tot_fields + op = "$sum" + label = field + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + cats.append( + ( + cat_label, + parse_filters(cat_filter, self, hier_aware=False), + [[label, field, op]], + ) + ) + return cats + + @cached_property + def account_fields_and_ops(self): + """ + mapping from account category label to deal fields and operations to perform on them for top accounts + + Returns: + dict -- {cat label: [(field label, db field name, db operation)]} + """ + cat_field_map = defaultdict(list) + for category, fields_dtls in get_nested( + self.config, ["dashboard", "accounts", "fields"], {} + ).items(): + for field_dtls in fields_dtls: + field, op, _, label, _ = field_dtls + if field in self.dlf_fields: + field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) + cat_field_map[category].append((field, field, op)) + return cat_field_map + + @cached_property + def account_schema(self): + """ + mapping from account category label to deal schema for category + + Returns: + dict -- {cat label: {deal schema}} + """ + cat_schemas = {} + for category, fields_dtls in get_nested( + self.config, ["dashboard", "accounts", "fields"], {} + ).items(): + cat_schemas[category] = [ + {"fmt": fmt, "label": label, "field": field, + "is_opp_name": is_opp_name} + for field, _, fmt, label, is_opp_name in fields_dtls + ] + return cat_schemas + + @cached_property + def deals_schema(self): + """ + mapping from deal category label to deal schema for category + + Returns: + dict -- {cat label: {deal schema}} + """ + cat_schemas = {} + for category, fields_dtls in get_nested( + self.config, ["dashboard", "deal_categories", "fields"], {} + ).items(): + cat_schemas[category] = [ + {"fmt": fmt, "label": label, "field": field} + for field, _, fmt, label in fields_dtls + ] + return cat_schemas + + @cached_property + def account_group_fields(self): + """ + deal fields to group by for top accounts feature + + Returns: + list -- [db deal fields] + """ + return get_nested(self.config, ["dashboard", "accounts", "group_fields"], []) + + @cached_property + def leaderboard_previous_values(self): + """ + previous values config for all leaderboards + + Returns: + dict -- {cat label: {prev config schema}} + """ + return get_nested(self.config, ["dashboard", "leaderboard_previous_values"], {}) + + @cached_property + def account_sort_fields(self): + """ + deal fields to sort by for top accounts feature + + Returns: + dict -- {cat label: [deal fields]} + """ + sorts = get_nested( + self.config, ["dashboard", "accounts", "sort_fields"], {}) + return { + cat: [ + (field, 1) if isinstance(field, str) else field + for field in fields + ] + for cat, fields in sorts.items() + } + + @cached_property + def deal_changes_categories_amounts(self): + return get_nested( + self.config, + ["dashboard", "deal_changes", "categories_amounts"], + self._default_deal_changes["categories_amounts"], + ) + + @cached_property + def show_winscore_in_dashboard_top_deals(self): + return get_nested( + self.config, + ["dashboard", 'show_winscore_in_dashboard_top_deals'], {}) + + @cached_property + def dashboard_deal_format(self): + # Need a way to make it easier to the correct subset of fields + forecast_cat_label = None + forecast_cat_field = None + for label, deal_field in self.raw_schema.get("deal_fields", {}).items(): + if deal_field == self.forecast_category_field: + forecast_cat_field = label + for x in self.deal_fields_config: + if x["field"] == label: + forecast_cat_label = x["label"] + break + break + return [ + {"fmt": "str", "label": "Opportunity Name", "field": "OpportunityName"}, + {"fmt": "str", "label": "Owner", "field": "OpportunityOwner"}, + {"fmt": "amount", "label": "Amount", "field": "Amount"}, + {"fmt": "excelDate", "label": "Close Date", "field": "CloseDate"}, + {"fmt": "str", "label": forecast_cat_label, "field": forecast_cat_field}, + {"field": "win_prob", "fmt": "prob", "label": "Aviso Score"}, + ] + + # TODO: BS How to change this to use covid fields + @cached_property + def covid_dashboard_deal_format(self): + # Need a way to make it easier to the correct subset of fields + forecast_cat_label = None + forecast_cat_field = None + for label, deal_field in self.raw_schema.get("deal_fields", {}).items(): + if deal_field == self.oppmap_forecast_category_field: + forecast_cat_field = label + for x in self.deal_fields_config: + if x["field"] == label: + forecast_cat_label = x["label"] + break + break + return [ + {"fmt": "str", "label": "Opportunity Name", "field": "OpportunityName"}, + { + "field": "__covid__", + "fmt": "", + "label": self.covid_labellings.get("aviso_column", "Covid"), + }, + {"fmt": "str", "label": "Owner", "field": "OpportunityOwner"}, + {"fmt": "str", "label": forecast_cat_label, "field": forecast_cat_field}, + {"fmt": "excelDate", "label": "Close Date", "field": "CloseDate"}, + {"fmt": "amount", "label": "Amount", "field": "Amount"}, + ] + + @cached_property + def covid_labellings(self): + return get_nested(self.config, ["dashboard", "covid_labellings"], {}) + + @cached_property + def activity_metrics_top_20_deals_filter(self): + # deals filter to find top 20 deals for activity metrics graph in dashboard + return get_nested(self.config, ['dashboard', 'activity_metrics', 'top_20_deals_filter'], {}) + + @cached_property + def week_on_week_filters(self): + default_filters = {'commit': {'ManagerForecastCategory': {'$in': ['Commit']}}, + 'open_pipeline': {'as_of_StageTrans': {'$nin': ['1', '99']}}} + return get_nested(self.config, ['week_on_week_filters'], default_filters) + + @cached_property + def segment_amount(self): + return get_nested(self.config, ["dashboard", "segment_amount"], {}) + + @cached_property + def activity_metrics_deal_filters(self): + default_filter = [[u'Commit Deals', + [{u'key': self.forecast_category_field, u'op': u'in', u'val': [u'Commit']}], + [u'amount', u'$sum']], + [u'Most Likely Deals', + [{u'key': self.forecast_category_field, u'op': u'in', u'val': [u'Most Likely']}], + [u'amount', u'$sum']], + [u'Best Case Deals', + [{u'key': self.forecast_category_field, u'op': u'in', u'val': [u'Best Case']}], + [u'amount', u'$sum']]] + cats = [] + for cat_label, cat_filter, cat_tot_fields in get_nested(self.config, ['new_home_page', + 'activity_metrics', + 'deal_filters'], default_filter): + try: + label, op = cat_tot_fields + field = self.amount_field + except ValueError: + field, = cat_tot_fields + op = '$sum' + if field in self.dlf_fields: + field = '.'.join(['dlf', field, '%(node)s', 'dlf_amt']) + cats.append((cat_label, parse_filters(cat_filter, self, hier_aware=False), [[label, field, op]])) + return cats + + @cached_property + def engagement_grade(self): + return get_nested(self.config, ['field_map', 'engagement_grade'], 'engagement_grade') + + @cached_property + def oppmap_labellings(self): + return get_nested(self.config, ["dashboard", "oppmap_labellings"], {}) + + @cached_property + def standard_oppmap_map_types(self): + oppmap_types = get_nested( + self.config, + ["oppmap", "standard_oppmap_map_types"], + DEFAULT_STANDARD_OPPMAP_MAP_TYPES, + ) + oppmap_type_tuples = [] + for map_type in oppmap_types: + oppmap_type_tuples.append((map_type, oppmap_types[map_type])) + return oppmap_type_tuples + + @cached_property + def covid_oppmap_map_types(self): + oppmap_types = get_nested( + self.config, + ["oppmap", "covid_oppmap_map_types"], + DEFAULT_COVID_OPPMAP_MAP_TYPES, + ) + oppmap_type_tuples = [] + for map_type in oppmap_types: + oppmap_type_tuples.append((map_type, oppmap_types[map_type])) + return oppmap_type_tuples + + @cached_property + def segment_field(self): + return self.config.get("segment_field", None) + + # TODO: make it fuller if possible + + @cached_property + def nudge_insight_facts(self): + """ + config notebook - https://jupyter.aviso.com/user/amitk/notebooks/amitk/Ticket%20Specific/AV-11394.ipynb + """ + return get_nested( + self.config, ['insight_config', 'nudge_insight_facts'], {}) + + @cached_property + def insight_config(self): + return self.config.get("insight_config", {}) + + @cached_property + def insight_task_config(self): + return self.config.get('insight_task_config', {}) + + @cached_property + def custom_stage_ranks(self): + return self.config.get("custom_stage_ranks", {}) + + @cached_property + def custom_fc_ranks(self): + return self.config.get("custom_fc_ranks", {}) + + @cached_property + def custom_fc_ranks_default(self): + dict_ = {"pipeline": 1, "upside": 2, "most likely": 3, "commit": 4} + return self.config.get("custom_fc_ranks", dict_) + + @cached_property + def close_date_pushes_flds(self): + CLOSEDATE_FIELD_MAP = { + "total_pushes": "close_date_total_pushes", + "months_pushed": "close_date_months_pushed", + } + return self.config.get("close_date_pushes_flds", CLOSEDATE_FIELD_MAP) + + @cached_property + def use_grouper_flag(self): + return self.config.get("use_grouper_flag", False) + + @cached_property + def weekly_report_dimensions(self): + return self.config.get("weekly_report_dimensions", []) + + @cached_property + def dimensions(self): + return self.config.get("dimensions", []) + + @cached_property + def fm_config(self): + return self.config.get("fm_config", {}) + + @cached_property + def custom_fc_ranks(self): + return self.config.get("custom_fc_ranks", {}) + + @cached_property + def bookingstimeline(self): + return self.config.get("bookingstimeline", False) + + @cached_property + def anomaly_config(self): + return self.config.get("anomaly_config", {}) + + @cached_property + def stale_nudge_enable(self): + return self.config.get("stale_nudge_enable", False) + + @cached_property + def crm_hygiene_fld(self): + return self.config.get("crm_hygiene_fld", []) + + @cached_property + def past_closedate_enabled(self): + return self.config.get("past_closedate_enabled", False) + + @cached_property + def close_date_thresh(self): + return self.config.get("close_date_thresh", 15) + + @cached_property + def frequent_fld_nudge(self): + return self.config.get("frequent_fld_nudge", False) + + @cached_property + def frequent_fld(self): + return self.config.get("frequent_fld", "NextStep") + + @cached_property + def frequency_eoq_time(self): + return self.config.get("frequency_eoq_time", 14) + + @cached_property + def pipeline_nudge(self): + return self.config.get("pipeline_nudge", False) + + @cached_property + def pipeline_fields(self): + fields = self.config.get("pipeline_fields", {}) + if fields: + plan_field = fields.get("plan_field") + booked_field = fields.get("booked_field") + top_field = fields.get("top_field") + rollup_field = fields.get("rollup_field") + + return { + "plan_field": plan_field, + "booked_field": booked_field, + "top_field": top_field, + "rollup_field": rollup_field, + } + else: + return {} + + @cached_property + def pipeline_ratio(self): + return self.config.get("pipeline_ratio", 3) + + @cached_property + def changed_deals_limit(self): + return self.config.get("changed_deals_limit", 200) + + @cached_property + def exclude_competitors(self): + return self.config.get("exclude_competitors", ["no competition"]) + + @cached_property + def late_stg_thresh(self): + return self.config.get("late_stg_thresh", 40.0) + + @cached_property + def close_date_update(self): + return self.config.get("close_date_update", False) + + @cached_property + def update_thresh(self): + return self.config.get("update_thresh", 30) + + @cached_property + def close_date_thresh(self): + return self.config.get("close_date_thresh", 30) + + @cached_property + def manager_only_cd_no_update(self): + return self.config.get("manager_only_cd_no_update", False) + + # @cached_property + # def at_risk_deals_enabled(self): + # return self.config.get("at_risk_deals_enabled", False) + + # @cached_property + # def low_pipeline_threshold(self): + # return self.config.get('low_pipeline_threshold', 15.0) + + @cached_property + def low_netxq_pipeline_nudge(self): + return self.config.get("low_netxq_pipeline_nudge", False) + + @cached_property + def commit_not_in_dlf(self): + return self.config.get("commit_not_in_dlf_nudge", False) + + @cached_property + def competitor_nudge_enabled(self): + return self.config.get("competitor_nudge_enabled", False) + + # AV-996 + @cached_property + def stale_nudge_thresh(self): + return self.config.get("stale_nudge_thresh", 40) + + @cached_property + def stale_nudge_config(self): + config_params = { + 'deal_filter_id': 'Filter_Closedate_Stage', + 'deal_cnt': None, + 'close_within_days': 10, + "send_to_mgr": True, + "send_to_rep": True, + 'filter_hierarchy': False, + 'nudge_heading': '', + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'stale_nudge_config'], + config_params + ) + + @cached_property + def at_risk_nudge_config(self): + config_params = { + 'deal_cnt': None, + 'send_to_rep': True, + 'senf_to_mgr': True, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + 'send_only_to': [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'at_risk_nudge_config'], + config_params + ) + + # @cached_property + # def score_history_dip_nudge(self): + # return self.config.get("score_history_dip_nudge", False) + + # @cached_property + # def manager_only_score_drop(self): + # return self.config.get("manager_only_score_drop", False) + + @cached_property + def score_hist_dip_nudge_config(self): + config_params = { + 'threshold': 10.0, + 'deal_filter_id': '', + 'deal_cnt': None, + "send_to_mgr": True, + "send_to_rep": True, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'score_hist_dip_nudge_config'], + config_params + ) + + # @cached_property + # def competitor_nudge_config(self): + # return self.config.get("competitor_nudge_config", {}) + @cached_property + def competitor_nudge_config(self): + config_params = { + 'competitor_field': 'Competitor', + 'group_by_field': 'Type', + 'deal_filter_id': 'Competitor Deals', + 'deal_cnt': None, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + 'send_only_to': [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'competitor_nudge_config'], + config_params + ) + + @cached_property + def past_closedate_nudge_config(self): + config_params = { + 'deal_filter_id': 'Filter_Closedate', + 'extra_param': '', + 'full_year_deals_view_cta': True, + 'deal_cnt': None, + "send_to_mgr": True, + "send_to_rep": True, + 'filter_hierarchy': False, + "nudge_heading": "", + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'past_closedate_nudge_config'], + config_params + ) + + @cached_property + def highamount_change_nudge_config(self): + config_params = { + 'deal_filter_id': '', + 'deal_cnt': None, + "send_to_mgr": True, + "send_to_rep": True, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'highamount_change_nudge_config'], + config_params + ) + + @cached_property + def tenant_agg_metrics_nudge_config(self): + config_params = { + 'deal_cnt': None, + 'deal_filter_id': 'Open Deals', + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'tenant_agg_metrics_nudge_config'], + config_params + ) + + @cached_property + def scenario_nudge_config(self): + config_params = { + 'threshold': 20, + 'list_of_stages': [], + 'projection_days': 7, + 'deal_filter_id': 'scenario_deals', + 'deal_cnt': None, + "send_to_mgr": True, + "send_to_rep": True, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'scenario_nudge_config'], + config_params + ) + + @cached_property + def upside_deal_stats_nudge_config(self): + config_params = { + 'industry_fld': 'Industry', + 'filter_types': ['Renewal', 'Renewals'], + 'threshold': 60, + 'win_prob_treshold': 40, + # additional + 'deal_filter_id': 'oppmap/amount/commit/upside', + 'deal_cnt': None, + "send_to_mgr": True, + "send_to_rep": True, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'upside_deal_stats_nudge_config'], + config_params + ) + + @cached_property + def pullin_deals_nudge_config(self): + config_params = { + 'deal_filter_id': '/pull-ins', + 'deal_cnt': None, + "send_to_mgr": True, + "send_to_rep": True, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'pullin_deals_nudge_config'], + config_params + ) + + @cached_property + def rep_closing_metrics_config(self): + config_params = { + 'threshold': 30.0, # Percentage threshold + 'deal_filter_id': 'Open Deals', + 'deal_cnt': None, + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'rep_closing_metrics_config'], + config_params + ) + + @cached_property + def cd_no_update_config(self): + config_params = { + 'recommended_stage': '', # default stage + 'deal_filter_id': 'cd_no_update', + 'deal_cnt': None, + 'send_to_managers_only': False, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + 'send_only_to': [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'cd_no_update_config'], + config_params + ) + + @cached_property + def non_commit_config(self): + config_params = { + 'notif_gap': 7, + 'deal_filter_id': 'non_commit_fast', + 'deal_cnt': None, + 'send_to_mgr': True, + 'send_to_rep': True, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'non_commit_config'], + config_params + ) + + @cached_property + def make_or_break_nudge_config(self): + config_params = { + 'deal_filter_id': 'Open Deals', + 'deal_cnt': None, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'make_or_break_nudge_config'], + config_params + ) + + @cached_property + def drop_in_engagement_grade_nudge_config(self): + config_params = { + 'deal_cnt': None, + 'deal_filter_id': '', + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'drop_in_engagement_grade_nudge_config'], + config_params + ) + + @cached_property + def anomaly_nudge_config(self): + config_params = { + 'deal_filter_id': '', + 'deal_cnt': None, + 'send_to_mgr': True, + 'send_to_rep': True, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + 'send_only_to': [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'anomaly_nudge_config'], + config_params + ) + + @cached_property + def outquater_pipline_nudge_config(self): + config_params = { + 'deal_filter_id': 'Open Deals', + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'outquater_pipline_nudge_config'], + config_params + ) + + @cached_property + def pipline_nudge_config(self): + config_params = { + 'deal_filter_id': 'Open Deals', + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'pipline_nudge_config'], + config_params + ) + + @cached_property + def low_pipeline_nudge_config(self): + config_params = { + 'nextq_coverage_ratio': 4, + 'deal_filter_id': 'nextq_open_deals', + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'low_pipeline_nudge_config'], + config_params + ) + + @cached_property + def late_stage_conversion_ratio_nudge_config(self): + config_params = { + 'upside_deal_filter': '/oppmap/amount/commit/upside', + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'late_stage_conversion_ratio_nudge_config'], + config_params + ) + + @cached_property + def upsell_recommendation_nudge_config(self): + config_params = { + 'filter_specific_nodes': False, + 'deal_filter_id': 'Open Deals', + 'allowed_specific_nodes': {}, + 'prohibited_roles': [], + 'prohibited_emails': [], + 'etl_line_item_name': 'OpportunityLineItem', + 'product_field': 'ProductName' + } + return get_nested( + self.config, ['nudge_config', 'upsell_recommendation_nudge_config'], + config_params + ) + + @cached_property + def past_deals_based_alert_nudge_config(self): + config_params = { + 'filter_specific_nodes': False, + 'deal_filter_id': 'Open Deals', + 'allowed_specific_nodes': {}, + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'past_deals_based_alert_nudge_config'], + config_params + ) + + # @cached_property + # def booking_accuracy_manager_only(self): + # return self.config.get("booking_accuracy_manager_only", False) + @cached_property + def deal_amount_recommendation_nudge_config(self): + config_params = { + 'deal_filter_id': '', + 'deal_cnt': None, + 'send_to_managers_only': False, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'deal_amount_recommendation_nudge_config'], + config_params + ) + + @cached_property + def booking_accuracy_nudge_config(self): + config_params = { + 'deal_filter_id': '', + 'deal_cnt': None, + 'gap': 7, + "send_to_mgr": True, + "send_to_rep": False, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'booking_accuracy_nudge_config'], + config_params + ) + + @cached_property + def discount_nudge_config(self): + config_params = { + 'deal_filter_id': '', + 'deal_cnt': None, + "send_to_mgr": True, + "send_to_rep": True, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'discount_nudge_config'], + config_params + ) + + @cached_property + def market_basket_nudge_config(self): + config_params = { + 'confidence': 50.0, + 'deal_filter_id': '', + 'deal_cnt': None, + 'send_to_managers': False, + 'filter_hierarchy': False, + 'allowed_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'market_basket_nudge_config'], + config_params + ) + + # forecast_dip_nudge + @cached_property + def forecast_dip_enabled(self): + return self.config.get("forecast_dip_enabled", False) + + @cached_property + def forecast_dip_thresh(self): + return self.config.get("forecast_dip_thresh", 10) + + @cached_property + def forecast_dip_nudge_config(self): + config_params = { + 'deal_filter_id': '', + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + 'notif_gap': 7, + "internal_heading": "", + 'send_only_to': [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'forecast_dip_nudge_config'], + config_params + ) + + @cached_property + def conversion_rate_nudge(self): + config_params = { + 'upside_deal_filter': '/oppmap/amount/commit/upside', + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + 'send_only_to': [], + 'prohibited_roles': [], + 'prohibited_emails': [], + 'commit_fld': 'commit' + } + return get_nested( + self.config, ['nudge_config', 'conversion_rate_nudge'], + config_params + ) + + @cached_property + def pace_value_dip_enabled(self): + return self.config.get("pace_value_dip_enabled", False) + + # @cached_property + # def pace_value_dip_thresh(self): + # return self.config.get("pace_value_dip_thresh", -5) + @cached_property + def pace_value_dip_nudge_config(self): + config_params = { + 'notif_gap': 7, + 'pace_value_dip_thresh': -5, + 'no_reps': None, + 'deal_filter_id': '', + 'filter_specific_nodes': False, + 'allowed_specific_nodes': {}, + "send_only_to": [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'pace_value_dip_nudge_config'], + config_params + ) + + @cached_property + def dlf_news_nudge_config(self): + config_params = { + 'up_thresh': .20, + 'down_thresh': .15, + 'no_reps': None, + 'deal_filter_id': '', + 'filter_hierarchy': False, + 'allowed_nodes': {}, + 'send_only_to': [], + 'prohibited_roles': [], + 'prohibited_emails': [] + } + return get_nested( + self.config, ['nudge_config', 'dlf_news_nudge_config'], + config_params + ) + + # --AV-996-- + + # Favorite Deals Nudge + @cached_property + def favorite_deals_nudge_config(self): + config_params = { + "since": 'yest', # bow, bom + "allowed_users": ["amit.khachane@aviso.com"], + "attributes": '', + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, ["nudge_config", + "favorite_deals_nudge_config"], config_params + ) + + @cached_property + def deal_delta_alert_config(self): + """Configurations for Deal Delta Alert Nudge (launchdarkly daily digest)""" + config_params = { + "since": 7, # bow, bom + "allowed_users": ["amit.khachane@aviso.com"], + "attributes": '', + "filter_id": 'Favorites', + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, ["nudge_config", "favorite_deals_nudge_config"], config_params + ) + + @cached_property + def meeting_nextsteps_nudge_config(self): + """Configurations for Pre-Meeting Nudge""" + config_params = { + "allowed_users": [], + "prohibited_roles": [], + "prohibited_emails": [], + "send_only_to": [], + "debug": True, + "save_notifications": True, + } + return get_nested( + self.config, ["nudge_config", "meeting_nextsteps_nudge_config"], config_params) + + @cached_property + def unclassified_meetings_nudge_config(self): + """Configurations for Unclassified Meetings Nudge""" + config_params = { + "targeted_roles": [], + "prohibited_usersids": [], + "send_only_to_usersids": [], + + } + return get_nested( + self.config, ["nudge_config", "unclassified_meetings_nudge_config"], config_params) + + # Jfrog Nudge + @cached_property + def pushed_out_deals_nudge_config(self): + config_params = { + "threshold": 1, + "deal_cnt": None, + "send_to_mgr": True, + "send_to_rep": True, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, ["nudge_config", + "pushed_out_deals_nudge"], config_params + ) + + @cached_property + def dashboard_updates_nudge_config(self): + """Configurations for Dashboard Updates Nudge""" + config_params = { + "targeted_roles": [], + "prohibited_usersids": [], + "send_only_to_usersids": [], + } + return get_nested( + self.config, ["nudge_config", "dashboard_updates_nudge_config"], config_params) + + # RingCentral Nudges + @cached_property + def best_case_nudge_config(self): + config_params = { + "threshold": 50, + "deal_cnt": None, + "send_to_mgr": True, + "send_to_rep": True, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, ["nudge_config", + "best_case_nudge_config"], config_params + ) + + @cached_property + def commit_nudge_config(self): + config_params = { + "threshold": 55, + "deal_cnt": None, + "send_to_mgr": True, + "send_to_rep": True, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, ["nudge_config", "commit_nudge_config"], config_params + ) + + @cached_property + def yearold_nudge_config(self): + config_params = { + "early_stages_thresh": [], + "send_to_mgr": True, + "send_to_rep": True, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + 'title_stages': "", + } + return get_nested( + self.config, ["nudge_config", + "yearold_nudge_config"], config_params + ) + + @cached_property + def yearold_rep_nudge_config(self): + config_params = { + "early_stages_thresh": [], + "deal_cnt": None, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + "title_stages": "", + } + return get_nested( + self.config, ["nudge_config", + "yearold_rep_nudge_config"], config_params + ) + + @cached_property + def stagnant_manager_nudge_config(self): + config_params = { + "threshold": 15, + "send_to_mgr": True, + "send_to_rep": True, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, + ["nudge_config", "stagnant_manager_nudge_config"], + config_params, + ) + + @cached_property + def stagnant_rep_nudge_config(self): + config_params = { + "threshold": 15, + "deal_cnt": None, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, ["nudge_config", + "stagnant_rep_nudge_config"], config_params + ) + + @cached_property + def highvalue_manager_nudge_config(self): + config_params = { + "stage_threshold": 50, + "threshold": 100000, + "send_to_mgr": True, + "send_to_rep": True, + "filter_hierarchy": False, + "filter_id": "highvalue_deals_100k", + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, + ["nudge_config", "highvalue_manager_nudge_config"], + config_params, + ) + + @cached_property + def highvalue_rep_nudge_config(self): + config_params = { + "stage_threshold": 50, + "threshold": 100000, + "deal_cnt": None, + "filter_hierarchy": False, + "filter_id": "highvalue_deals_100k", + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, ["nudge_config", + "highvalue_rep_nudge_config"], config_params + ) + + @cached_property + def closedate_manager_nudge_config(self): + config_params = { + "threshold": 15, + "send_to_mgr": True, + "send_to_rep": True, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, + ["nudge_config", "closedate_manager_nudge_config"], + config_params, + ) + + @cached_property + def closedate_rep_nudge_config(self): + config_params = { + "threshold": 15, + "deal_cnt": None, + "filter_hierarchy": False, + "nudge_heading": "", + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, ["nudge_config", + "closedate_rep_nudge_config"], config_params + ) + + @cached_property + def mismatch_manager_nudge_config(self): + config_params = { + "commit_stage_thresh": 55, + "bestcase_stage_thresh": 50, + "send_to_mgr": True, + "send_to_rep": True, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, + ["nudge_config", "mismatch_manager_nudge_config"], + config_params, + ) + + @cached_property + def mismatch_rep_nudge_config(self): + config_params = { + "commit_stage_thresh": 55, + "bestcase_stage_thresh": 50, + "deal_cnt": None, + "filter_hierarchy": False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, + ["nudge_config", "mismatch_rep_nudge_config"], + config_params, + ) + + @cached_property + def potential_manager_nudge_config(self): + config_params = { + 'arr_threshold': 150000, + 'tcv_threshold': 800000, + "send_to_mgr": True, + "send_to_rep": True, + 'filter_hierarchy': False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, + ['nudge_config', 'potential_manager_nudge_config'], + config_params, + ) + + @cached_property + def potential_rep_nudge_config(self): + config_params = { + 'arr_threshold': 150000, + 'tcv_threshold': 800000, + 'deal_cnt': None, + 'filter_hierarchy': False, + "allowed_nodes": {}, + "send_only_to": [], + "prohibited_roles": [], + "prohibited_emails": [], + } + return get_nested( + self.config, + ['nudge_config', 'potential_rep_nudge_config'], + config_params, + ) + + # --x--RingCentral Nudges--x-- + + @cached_property + def competitor_win_loss_config(self): + return self.config.get("competitor_win_loss_config", {}) + + @cached_property + def scenario_nudge_enabled(self): + return self.config.get("scenario_nudge_enabled", False) + + # @cached_property + # def manager_only_high_amount_change(self): + # return self.config.get("manager_only_high_amount_change", False) + + @cached_property + def high_risk_thresh(self): + return self.config.get("high_risk_thresh", 20) + + @cached_property + def cutoff_change(self): + return self.config.get("high_risk_amount_change_cutoff", 10.0) + + @cached_property + def high_risk_deals_enabled(self): + return self.config.get("high_risk_deals_enabled", False) + + @cached_property + def booking_accuracy_enable(self): + return self.config.get("booking_accuracy_enable", False) + + @cached_property + def tenant_aggregate_metrics(self): + return self.config.get("tenant_aggregate_metrics", False) + + @cached_property + def non_commit_enabled(self): + return self.config.get("non_commit_enabled", False) + + @cached_property + def non_commit_manager_only(self): + return self.config.get("non_commit_manager_only", False) + + @cached_property + def non_commit_dlf_enabled(self): + return self.config.get("non_commit_dlf_enabled", False) + + # groupby_fields_map for mobile api + @cached_property + def groupby_fields_map(self): + return self.config.get("groupby_fields_map", {}) + + # ADDITION OF CONFIGURATIONS TO RESTRICT EMAIL SENDING TO MANAGERS ONLY + @cached_property + def manager_only_past_closedate(self): + return self.config.get("manager_only_past_closedate", False) + + @cached_property + def discount_nudge_enabled(self): + return self.config.get("discount_nudge_enabled", False) + + @cached_property + def nudge_config(self): + return self.config.get("nudge_config", {}) + + @cached_property + def crm_hygiene_fld(self): + return self.config.get("crm_hygiene_fld", []) + + @cached_property + def won_filter_criteria(self): + """ + mongo db filter criteria for won deals + + Returns: + dict -- {mongo db criteria} + """ + return fetch_filter([self._won_filter_id], self, db=self.db) + + @cached_property + def _won_filter_id(self): + return get_nested(self.config, ["filters", "won_filter"]) + + def lost_filter_criteria(self): + + return fetch_filter([self._lost_filter_id], self, db=self.db) + + @cached_property + def _lost_filter_id(self): + return get_nested(self.config, ["filters", "lost_filter"]) + + def alldeal_filter_criteria(self): + + return fetch_filter([self._alldeal_filter_id], self, db=self.db) + + @cached_property + def _alldeal_filter_id(self): + return get_nested(self.config, ["filters", "all_filter"]) + + def pushout_deals_filter_criteria(self): + + return fetch_filter([self._pushout_deals_filter_id], self, db=self.db) + + @cached_property + def _pushout_deals_filter_id(self): + return get_nested(self.config, ["filters", "pushout_deals"]) + + def commit_filter_criteria(self): + + return fetch_filter([self._commit_filter_id], self, db=self.db) + + @cached_property + def _commit_filter_id(self): + return get_nested(self.config, ["filters", "Commit"], 'Commit') + + @cached_property + def created_date_field(self): + """ + name of created date field for tenant + Returns: + str -- field name + """ + return get_nested(self.config, ["field_map", "created_date"]) + + @cached_property + def manager_only_next_step_nudge(self): + return self.config.get("manager_only_next_step_nudge", False) + + @cached_property + def manager_only_commit_no_dlf(self): + return self.config.get("manager_only_commit_no_dlf", False) + + @cached_property + def persona_schemas(self): + return self.config.get("persona_schemas") + + @cached_property + def role_schemas(self): + return self.config.get("role_schemas", {}) + + @cached_property + def crr_role_schemas(self): + return self.config.get("crr_role_schemas", {}) + + # + # Complex Fields aka the please dont configure this fields + # + @cached_property + def period_aware_fields(self): + """ + fields that are period aware + + Returns: + list -- [please no] + """ + return get_nested(self.config, ["complex_fields", "period_aware_fields"], []) + + @cached_property + def hier_aware_fields(self): + """ + fields that are hierarchy aware (split fields) + + Returns: + list -- [seriously, dont] + """ + gbm_hier_aware_fields = [ + "forecast" + ] # TODO: @logan what else is always hier aware in gbm + + tenant_hier_aware_fields = get_nested( + self.config, ["complex_fields", "hier_aware_fields"], [] + ) + + dtfo_hier_aware_fields = [] + + if self.amount_field in tenant_hier_aware_fields: + dtfo_hier_aware_fields = [ + "won_amount_diff", + "lost_amount_diff", + "amt", + "stg", + ] + + return ( + get_nested(self.config, ["complex_fields", + "hier_aware_fields"], []) + + gbm_hier_aware_fields + + dtfo_hier_aware_fields + ) + + @cached_property + def revenue_fields(self): + """ + fields that are revenue based + + Returns: + list -- [i beg of you] + """ + return get_nested(self.config, ["complex_fields", "revenue_fields"], []) + + @cached_property + def active_field(self): + return get_nested( + self.config, ["complex_fields", "active_field"], "active_amount" + ) + + @cached_property + def home_page_weekly_metrics_schema(self): + conf = {"columns": ["activity", "Commit Deals", "Best Case Deals", "Most Likely Deals"], + "schema": {"activity": {"type": "string", + "label": "Activity"}, + "Commit Deals": {"type": "cell-block", + "label": "Commit"}, + "Best Case Deals": {"type": "cell-block", + "label": "Best Case"}, + "Most Likely Deals": {"type": "cell-block", + "label": "Most Likely"}}} + return get_nested(self.config, ['new_home_page', 'activity_metrics', 'schema'], conf) + + @cached_property + def deals_to_hide(self): + """ + Deals with values that need to be hidden in UI + :return: + dict -- {'as_of_Stage':'Dummy'} + """ + return self.config.get("deals_to_hide") + + @cached_property + def account_relationships_config(self): + return self.config.get("account_relationships") + + @cached_property + def account_dashboard_config(self): + return self.config.get("account_dashboard") + + @cached_property + def has_wiz_metrics(self): + return self.config.get("has_wiz_metrics") + + @cached_property + def owner_email_field(self): + return self.field_map.get("owner_email", 'OwnerEmail') + + @cached_property + def conditional_writeback(self): + return self.config.get("conditional_writeback") + + @cached_property + def vlookup_writeback_fields(self): + return self.config.get('vlookup_fields', {}) + + @cached_property + def has_wiz_metrics(self): + return self.config.get("has_wiz_metrics") + + # + # Validation + # + def validate(self, config): + """ + validate config + + Arguments: + config {dict} -- config dictionary + + Returns: + tuple - bool(valid config), [error messages] + """ + if not config: + return True, ["no config provided"] + + good_field_map, field_map_message = self._validate_field_map( + config.get("field_map", {}) + ) + good_dlf, dlf_message = self._validate_dlf(config.get("dlf", {})) + good_complex, complex_message = self._validate_complex( + config.get("complex_fields", {}) + ) + good_totals, total_message = self._validate_totals( + config.get("totals", {})) + good_schema, schema_message = self._validate_schema( + config.get("schema", {})) + good_filters, filters_message = self._validate_filters( + config.get("filters", {}) + ) + good_dashboard, dashboard_message = self._validate_dashboard( + config.get("dashboard", {}) + ) + + all_good = all( + [ + good_field_map, + good_dlf, + good_complex, + good_totals, + good_schema, + good_filters, + ] + ) + all_messages = [ + field_map_message, + dlf_message, + complex_message, + total_message, + schema_message, + filters_message, + ] + return all_good, [msg for msg in all_messages if msg] + + def _validate_field_map(self, field_map): + if not field_map: + return False, "no field map provided" + for required_field in [ + "amount", + "stage", + "forecast_category", + "close_date", + "owner_id", + "stage_trans", + ]: + if required_field not in field_map: + return False, "{} field not in map".format(required_field) + + return True, "" + + def _validate_dlf(self, dlf_config): + if not dlf_config: + return True, "" + + for field, field_config in dlf_config.items(): + if "mode" not in field_config: + return False, "no mode provided for {}, config: {}".format( + field, field_config + ) + # TODO: is amount seed field required? + + for dlf_filter in ["locked_filters", "default_filters"]: + for state, filter_ids in field_config.get(dlf_filter, {}).items(): + filt = fetch_filter(filter_ids, self, db=self.db) + if filt is None: + return ( + False, + "{} filter id: {} for {} not in filters collection".format( + dlf_filter, state, filter_ids + ), + ) + + return True, "" + + def _validate_complex(self, complex_config): + return True, "" # TODO: this + + def _validate_totals(self, total_config): + # TODO: this + return True, "" + + def _validate_schema(self, schema_config): + deal_schema = schema_config.get("deal", []) + fields = set() + for field_dtls in deal_schema: + if "field" not in field_dtls: + return False, "no field provided for {}".format(field_dtls) + if "label" not in field_dtls: + return False, "no label provided for {}".format(field_dtls) + if "fmt" not in field_dtls: + return False, "no format provided for {}".format(field_dtls) + if ( + "crm_writable" in field_dtls + and field_dtls["field"] in self.hier_aware_fields + ): + return ( + False, + "no writeback allowed on hierarchy split fields {}".format( + field_dtls + ), + ) + fields.add(field_dtls["field"]) + for field in schema_config.get("deal_fields", {}).keys(): + if "__" not in field and field not in fields: + return ( + False, + "field {} in deal fields not configured in deal schema: {}".format( + field, deal_schema + ), + ) + for optional_schema in ["card_deal_fields", "pull_in_deal_fields"]: + for field in schema_config.get(optional_schema, {}).keys(): + if "__" not in field and field not in fields: + return ( + False, + "field {} in {} not configured in deal schema: {}".format( + field, optional_schema, deal_schema + ), + ) + return True, "" + + def _validate_filters(self, filters): + filter_ids = filters.values() + filter_results = fetch_many_filters( + [[filt_id] for filt_id in filter_ids], self, db=self.db + ) + for filter_id in filter_ids: + name, filt = filter_results[tuple([filter_id])] + if not name: + return False, "filter id: {} not in filters collection".format( + filter_id + ) + return True, "" + + def _validate_dashboard(self, dashboard): + # TODO: me + return True, "" + + # + # Default Configurations + # + @cached_property + def _default_deal_changes(self): + return { + "leaderboards": { + "Amount Changes": { + "categories": ["Increase", "Decrease"], + "arrow": True, + }, + "Biggest Movers": { + "categories": ["Upgraded", "Downgraded"], + "total_name": "Forecast Impact", + "arrow": True, + }, + "Close Date Changes": {"categories": ["Pulled In", "Pushed Out"]}, + "Forecast Category Changes": { + "categories": ["Committed", "Decommitted"] + }, + "Pipeline Changes": {"categories": ["New", "Won", "Lost"]}, + }, + "categories": [ + [ + "Committed", + [{"key": "comm"}, {"key": "actv"}, + {'key': 'pushedout', 'negate': True}], + [self.amount_field, "$sum"], + ], + [ + "Decommitted", + [{"key": "decomm"}, {"key": "actv"}, + {'key': 'pushedout', 'negate': True}], + [self.amount_field, "$sum"], + ], + ["New", [{"key": "new_since_as_of"}, + {'key': 'pushedout', 'negate': True}], + [self.amount_field, "$sum"]], + [ + "Won", + [ + { + "key": ["won_amount_diff", "%(node)s"], + "op": "nested_in_range", + "val": [0, None, True], + }, + {"key": self.stage_trans_field, + "op": "in", "val": ["99"]}, + {'key': 'pushedout', 'negate': True} + ], + ["won_amount_diff", "$sum"], + ] + if self.amount_field in self.hier_aware_fields + else [ + "Won", + [ + { + "key": "won_amount_diff", + "op": "range", + "val": [0, None, True], + }, + {"key": self.stage_trans_field, + "op": "in", "val": ["99"]}, + {'key': 'pushedout', 'negate': True} + ], + ["won_amount_diff", "$sum"], + ], + [ + "Lost", + [ + { + "key": ["lost_amount_diff", "%(node)s"], + "op": "nested_in_range", + "val": [0, None, True], + }, + {"key": self.stage_trans_field, + "op": "in", "val": ["-1"]}, + {'key': 'pushedout', 'negate': True} + ], + ["lost_amount_diff", "$sum"], + ] + if self.amount_field in self.hier_aware_fields + else [ + "Lost", + [ + { + "key": "lost_amount_diff", + "op": "range", + "val": [0, None, True], + }, + {"key": self.stage_trans_field, + "op": "in", "val": ["-1"]}, + {'key': 'pushedout', 'negate': True} + ], + ["lost_amount_diff", "$sum"], + ], + [ + "Increase", + [ + { + "key": ["amt", "%(node)s"], + "op": "nested_in_range", + "val": [0, None, True], + }, + {"key": "actv"}, + {'key': 'pushedout', 'negate': True} + ], + [self.amount_field, "$sum"], + ] + if self.amount_field in self.hier_aware_fields + else [ + "Increase", + [ + {"key": "amt", "op": "range", "val": [0, None, True]}, + {"key": "actv"}, + {'key': 'pushedout', 'negate': True} + ], + [self.amount_field, "$sum"], + ], + [ + "Decrease", + [ + { + "key": ["amt", "%(node)s"], + "op": "nested_in_range", + "val": [None, 0, True, True], + }, + {"key": "actv"}, + {'key': 'pushedout', 'negate': True} + ], + [self.amount_field, "$sum"], + ] + if self.amount_field in self.hier_aware_fields + else [ + "Decrease", + [ + {"key": "amt", "op": "range", + "val": [None, 0, True, True]}, + {"key": "actv"}, + {'key': 'pushedout', 'negate': True} + ], + [self.amount_field, "$sum"], + ], + [ + "Upgraded", + [ + { + "key": ["fcst", "%(node)s"], + "op": "nested_in_range", + "val": [0, None, True, True], + }, + {"key": "actv"}, + {'key': 'pushedout', 'negate': True} + ], + [self.amount_field, "$sum"], + ], + [ + "Downgraded", + [ + { + "key": ["fcst", "%(node)s"], + "op": "nested_in_range", + "val": [None, 0, True, True], + }, + {"key": "actv"}, + {'key': 'pushedout', 'negate': True} + ], + [self.amount_field, "$sum"], + ], + [ + "Pulled In", + [{"key": "pulledin"}, {"key": "actv"}], + [self.amount_field, "$sum"], + ], + [ + "Pushed Out", + [{"key": "pushedout"}, {"key": "actv"}], + [self.amount_field, "$sum"], + ], + ], + "categories_order": [ + ["Previous Pipe", ["Pipe"]], + ["Current Pipe", ["Pipe"]], + ["New", ["New"]], + ["Won", ["Won"]], + ["Lost", ["Lost"]], + ["Date Changes", ["Pulled In", "Pushed Out"]], + ["Amount Changes", ["Increase", "Decrease"]], + ["Commit Changes", ["Committed", "Decommitted"]], + ["Aviso Forecast Changes", ["Upgraded", "Downgraded"]], + ], + "default_key": "Close Date Changes", + "leaderboards_order": [ + { + "i": "cal", + "key": "Close Date Changes", + "label": "Close Date Changes", + }, + {"i": "bars", "key": "Amount Changes", "label": "Amount Changes"}, + { + "i": "therm", + "key": "Forecast Category Changes", + "label": "Forecast Category Changes", + }, + {"i": "flag", "key": "Pipeline Changes", + "label": "Pipeline Changes"}, + {"i": "brain", "key": "Biggest Movers", "label": "Biggest Movers"}, + ], + "categories_amounts": { + "Won": "won_amount_diff", + "Lost": "lost_amount_diff", + }, + } + + @cached_property + def categories_order_labels(self): + return get_nested( + self.config, + ["dashboard", "categories_order_labels"], + self._default_categories_order_labels, + ) + + @cached_property + def _default_categories_order_labels(self): + return { + "Previous Pipe": "Previous Pipe", + "Current Pipe": "Current Pipe", + "New": "New", + "Won": "Won", + "Lost": "Lost", + "Date Changes": "Date Changes", + "Amount Changes": "Amount Changes", + "Commit Changes": "Commit Changes", + "Aviso Forecast Changes": "Aviso Forecast Changes", + } + + @cached_property + def default_milestones(self): + default_milestones = self.config.get('default_milestones', None) + if default_milestones is None: + self.config['default_milestones'] = DEFAULT_MILESTONES + return self.config.get('default_milestones', {}) + + @cached_property + def default_milestones_stages(self): + return self.config.get('default_milestones_stages', {}) + + @cached_property + def default_judg_type(self): + return get_nested(self.config, ["oppmap", "default_judg_type"], "commit") + + @cached_property + def show_deal_type(self): + return get_nested(self.config, ["oppmap", "show_deal_type"], False) + + @cached_property + def default_deal_type(self): + return get_nested(self.config, ["oppmap", "default_deal_type"], "all") + + @cached_property + def oppmap_judg_type_options(self): + return get_nested( + self.config, + ["oppmap", "oppmap_judg_type_options"], + ["commit", "dlf", "most_likely"], + ) + + @cached_property + def oppmap_judg_type_option_labels(self): + return get_nested( + self.config, + ["oppmap", "oppmap_judg_type_option_labels"], + DEFAULT_OPPMAP_JUDGE_TYPE_OPTION_LABELS, + ) + + @cached_property + def alt_amount_val_for_na(self): + return get_nested(self.config, ["amount_field_checks", "alt_amount_val"], 0.0) + + @cached_property + def amount_val_check_enabled(self): + return get_nested( + self.config, ["amount_field_checks", + "amount_val_chk_for_fld"], False + ) + + @cached_property + def versioned_hierarchy(self): + from config import HierConfig + + hier_config = HierConfig() + + return hier_config.versioned_hierarchy + + # Return the dlf_fcst default collection schema along with the additional configured fields(if configured), + # else return None(This will not create the data for the collection) + @cached_property + def dlf_fcst_coll_schema(self): + dlf_fcst_schema = [] + dlf_fcst_schema.extend(DEFAULT_DLF_FCST_COLL_SCHEMA) + dlf_fcst_schema_additional_fields = get_nested( + self.config, ["dlf_fcst_coll_additional_fields"], None + ) + # Add the amount field dynamically + dlf_fcst_schema.append(self.amount_field) + + if dlf_fcst_schema_additional_fields: + dlf_fcst_schema.extend(dlf_fcst_schema_additional_fields) + + return dlf_fcst_schema + + def amount_field_by_pivot(self, node): + if node: + pivot = node.split("#")[0] + if self.pivot_amount_fields is not None: + return self.pivot_amount_fields.get(pivot, self.amount_field) + return self.amount_field + + @cached_property + def custom_fc_ranks_ext(self): + dict_ = self.custom_fc_ranks_default + if "Commit" in dict_: + dict_["commit"] = dict_["Commit"] + if "Most Likely" in dict_: + dict_["most_likely"] = dict_["Most Likely"] + + return dict_ + + @cached_property + def display_insights_card(self): + return get_nested( + self.config, + ["win_score_insights_card", "display_insights_card"], + DEFAULT_DISPLAY_INSIGHTS_CARD, + ) + + @cached_property + def winscore_graph_options(self): + return { + 'change_in_stagetrans': 'Change in StageTrans', + } + + @cached_property + def count_recs_to_display(self): + return get_nested( + self.config, ["win_score_insights_card", + "count_recs_to_display"], 5 + ) + + @cached_property + def oppds_fieldmap(self): + return self.config.get("oppds_fieldmap", {}) + + @cached_property + def demo_report_config_enabled(self): + return self.config.get("demo_report_config_enabled", False) + + @cached_property + def uip_user_fields(self): + return get_nested( + self.config, + ["uip_fields", "account"], + {"fields": ["Email"], "ref_field": "Email"}, + ) + + @cached_property + def uip_account_fields(self): + return get_nested( + self.config, + ["uip_fields", "account"], + { + "fields": ["LeanData__LD_EmailDomains__c"], + "reference": "LeanData__LD_EmailDomains__c", + }, + ) + + @cached_property + def custom_dtfo_fields(self): + return get_nested(self.config, ["dashboard", "custom_dtfo_fields_to_add"], []) + + @cached_property + def filter_totals_config(self): + return self.config.get('filter_totals_config', { + 'enabled': True, + 'daily': True, + 'chipotle': True + }) + + @cached_property + def deal_changes_totals_config(self): + return self.config.get('deal_changes_totals_config', {}) + + @cached_property + def run_insights_batch_size(self): + return self.config.get('run_insights_batch_size', {}) + + @cached_property + def run_gbm_crr_batch_size(self): + return self.config.get('run_gbm_crr_batch_size', {}) + + @cached_property + def future_qtrs_prefetch_count(self): + filter_totals_config = self.filter_totals_config + future_qtrs_prefetch_count = 0 + if filter_totals_config: + future_qtrs_prefetch_count = filter_totals_config.get("future_qtrs_process_count", 0) + return future_qtrs_prefetch_count + + @cached_property + def past_qtrs_prefetch_count(self): + filter_totals_config = self.filter_totals_config + past_qtrs_prefetch_count = 0 + if filter_totals_config: + past_qtrs_prefetch_count = filter_totals_config.get("past_qtrs_process_count", 0) + return past_qtrs_prefetch_count + + @cached_property + def no_update_on_writeback_fields(self): + """ + fields for which we don't need to run filter_totals or snapshot task on update from ui. + + Returns: + list + """ + + return self.config.get('no_update_on_writeback_fields', ["Manager Comments", + "__comments__", + "__comment__", + "comments", + "NextStep", + "NextSteps", + "latest_NextStepsquestions", + "latest_Problems", + "ManagerNotes", + "ProServCommentsNotes", + "latest_CoachingNotes", + "LastStep", + "LegalNotes", + "Manager_Comments", + "Mgr Notes", + "MgrNextSteps", + "TAPNextSteps", + "CustomNextStep", + "Next_Steps", + "Next Step", + "ProServCommentsNotes", + "ForecastNotes", + "ManagerForecastNotes", + "SalesDirectorNotes", + "SlipNotes", + "SEDirectorNotes", + "Notes", + "CoachingNotes", + "CSMNotes", + "latest_CoachingNotes"]) + + @cached_property + def weekly_fm(self): + return self.config.get("weekly_fm", False) + + @cached_property + def yearly_sfdc_view(self): + return self.config.get("yearly_sfdc_view", True) + + @cached_property + def insight_actions_mapping(self): + """ + Serves config to DeepLink API + insight_actions_mapping = {'scenario': {'component': 'filter_id'}, + 'risk': {'opp_map_link': '/*/oppmap/amount/commit/upside/standard/all'} + } + """ + return self.config.get('insight_actions_mapping', {}) + + @cached_property + def enhanced_waterfall_chart(self): + return self.config.get("enhanced_waterfall_chart", False) + + @cached_property + def enabled_file_parsed_load_run(self): + return self.config.get("enabled_file_parsed_load_run", False) diff --git a/config/fm_config.py b/config/fm_config.py index eb7df60..fa790ea 100644 --- a/config/fm_config.py +++ b/config/fm_config.py @@ -1,13 +1,11 @@ import copy import logging -from collections import defaultdict from datetime import timedelta from aviso.settings import sec_context -from config.base_config import BaseConfig -from infra.filters import (FilterError, _valid_filter, fetch_filter, - fetch_many_filters, parse_filters) +from config import BaseConfig, DealConfig +from infra.filters import (FilterError, _valid_filter, fetch_filter, parse_filters) from infra.rules import HIERARCHY_RULES, passes_configured_hierarchy_rules from utils.common import cached_property from utils.date_utils import datetime2epoch, epoch, get_bom, now @@ -17,217 +15,7 @@ logger = logging.getLogger("gnana.%s" % __name__) DEFAULT_ROLE = 'default_role' -DEFAULT_BEST_CASE_VALS = ["Best Case"] -DEFAULT_COMMIT_VALS = [ - "Commit", - "Committed", - "Forecasted", - "commit", - "Forecasted Upside", - "True", - "true", - True, -] - -DEFAULT_PIPELINE_VALS = ["Pipeline", "pipeline"] - -DEFAULT_MOST_LIKELY_VALS = ["Most Likely", "most likely"] - -DEFAULT_RENEWAL_VALS = [ - "Renewal", - "renewal", - "RENEWAL", - "Renewals", - "Recurring", - "Resume", - "subscription renewal", - "support renewal", - "Existing Customer - Maintenance renewal", - "Existing customer - Subscription renewal", - "Existing customer - subscription renewal", - "Maintenance renewal", - "Existing Customer \u2013 maintenance renewal", - "Maintenance Renewal (MR)", - "Existing Business", - "delayed renewal", - "Contract renewal", - "contractual renewal", - "Customer", - "Support Renewal", - "EC renewal", -] - -DEFAULT_DLF_VALS = ["True", "true", True] - -PULL_IN_LIST = [ - "__fav__", - "OpportunityName", - "OpportunityOwner", - "win_prob", - "pullin_prob", - "CloseDate", - "Amount", - "Stage", - "ForecastCategory", - "__id__", - "SFDCObject" -] - -DEFAULT_DLF_FCST_COLL_SCHEMA = [ - "opp_id", - "is_deleted", - "period", - "close_period", - "drilldown_list", - "hierarchy_list", - "dlf.in_fcst", - "update_date", -] - -DEFAULT_DISPLAY_INSIGHTS_CARD = { - "amount": [], - "close_date": ["pushes", "suggested_push"], - "stage": ["sharp_decline", "stage_dur", "stage_age"], - "global": [ - "grouper", - "other_group", - "rare_group", - "score_history", - "close_date_exp", - ], - "score_explanation": [ - "upside_deal", - "deal_amount_reco", - "deal_speed", - "scenario", - "score_history_dip", - "primary_competitor", - "competitor_win_loss", - "recency", - "new_deal", - "custom", - "closing_soon_after_eoq", - "high_leverage_moments", - "risk_insights", - "winscore_projections_upper", - "winscore_projections_lower", - "winscore_insights" - ], - "field_level": [ - "stale_deal", - "cd_change", - "anomaly", - "amount_change", - "no_amount", - "recommit2", - "stale_commit", - "never_pushed", - "engagement_grade", - "yearold", - "closedate", - "past_closedate", - "dlf_bad", - "dlf_good", - - ], - "in_fcst": ["dlf_change"], -} - -DEFAULT_STANDARD_OPPMAP_MAP_TYPES = { - "amount": "Amount", - "count": "Count", - "quartiles": "Quartiles", -} - -DEFAULT_COVID_OPPMAP_MAP_TYPES = {"amount": "Amount", "count": "Count"} - -DEFAULT_OPPMAP_JUDGE_TYPE_OPTION_LABELS = { - "dlf": "DLF", - "commit": "Commit", - "most_likely": DEFAULT_MOST_LIKELY_VALS[0], -} - -OWNER_INSIGHTS = ["owner_insight", "owner_bad", - "owner_amt", "owner_cd", "owner_dct", "owner_cmt"] - -DEFAULT_MILESTONES_NEW = [ - { - "name": "Lead Conversion", - "color": "#f89685", - "items": ['Convert lead to 5% probability'] - }, - { - "name": "Account Engagement", - "color": "#2faadc", - "items": ["Develop customer interest in proceeding with conversations"] - }, - { - "name": "Qualification", - "color": "#107e4e", - "items": ['Completion of Disco call', 'BANT Qualification'] - }, - { - "name": "Identify pain points and Metrics", - "color": "#3d4689", - "items": ["Identification of Pain Points", "Identification of Metrics"] - }, - { - "name": "Identify Champion", - "color": "#46d62e", - "items": ['Champion Identification'] - }, - { - "name": "Identify your stakeholders", - "color": "#ffc22b", - "items": ['Identification of Economic Buyer', 'Identification of Decision Process', - 'Identification of Decision Criteria'] - }, - { - "name": "Approval from Executive board", - "color": "#e1a612", - "items": ['Schedule EB meeting', 'Business reviews'] - }, - { - "name": "Legal Approval", - "color": "#e03e28", - "items": ['Legal Approval', 'Ready for signatures'] - }, - { - "name": "Signatures", - "color": "#6e77c2", - "items": ["Signatures to be done by both the parties"] - }, - { - "name": "Final review", - "color": "#025b8d", - "items": ['Deal desk final review'] - } -] - -DEFAULT_MILESTONES_RENEWAL = [ - { - "name": "Renewal Generated", - "color": "#f89685", - "items": [] - }, - { - "name": "Schedule Meeting with Customer", - "color": "#2faadc", - "items": ["Setup meeting with customer to discuss renewal"] - }, - { - "name": "Verbal Agreement", - "color": "#107e4e", - "items": ['Get verbal agreement to renew'] - }, - { - "name": "Renewal Quote", - "color": "#3d4689", - "items": ["Meeting with EB/ Executive sponsor", "Begin negotiating propoal components", - "Complete the Business Case", - "Get the initial budget approved", "No churn/dollar churn amount agreement"] - } -] + COLLABORATION_RECORDINGS_TAB_DEFAULT_OPTIONS = [{ 'key': 'All', @@ -238,4667 +26,9 @@ 'label': 'My Recordings', 'default': True } - # { - # 'key': 'TEAM', - # 'label': 'My Teams Recording' - # } ] -DEFAULT_MILESTONES = { - 'new': DEFAULT_MILESTONES_NEW, - 'renewal': DEFAULT_MILESTONES_RENEWAL -} - - -def _convert_state(state): - if state == "True": - return True - elif state == "False": - return False - return state - - -def get_node_depth(node): - pass - - -class DealConfig(BaseConfig): - config_name = "deals" - - @cached_property - def rollup_for_writeback(self): - return self.config.get('rollup_for_writeback', False) - - @cached_property - def is_recommended_actions_enabled(self): - return self.config.get('enable_recommended_actions_task', False) - - @cached_property - def dummy_tenant(self): - """ - tenant is using dummy data, is not hooked up to any gbm/etl - """ - return self.config.get("dummy_tenant", False) - - @cached_property - def hide_deal_grid_fields(self): - """ - hide deal grid fields - """ - return self.config.get("hide_deal_grid_fields", []) - - @cached_property - def bootstrapped(self): - """ - tenant has been bootstrapped and has initial data loaded into app - """ - return self.config.get("bootstrapped", True) - - # gateway schema - @cached_property - def gateway_schema(self): - return self.config.get("gateway_schema", {}) - - @cached_property - def is_aviso_tenant(self): - return self.config.get("aviso_tenant", False) - - @cached_property - def forecast_panel(self): - return self.config.get("forecast_panel", {}) - - @cached_property - def ci_top_deals_panel(self): - return self.config.get("ci_top_deals_panel", {}) - - @cached_property - def deal_alert_fields(self): - return self.config.get("deal_alert_fields", []) - - @cached_property - def deal_alert_on(self): - return self.config.get("deal_alert_on", False) - - @cached_property - def traffic_light_criteria(self): - return self.config.get("traffic_light_criteria", False) - - @cached_property - def disable_deal_details_tab(self): - return self.config.get('disable_deal_details_tab', False) - - @cached_property - def disable_deal_history_tab(self): - return self.config.get('disable_deal_history_tab', False) - - @cached_property - def deal_details_config(self): - """ - Fetch the tenant specific navigation config - """ - default_config = { - 'details': 'Details', - 'history': 'History', - 'relationships': 'Relationships', - 'interactions': 'Interactions', - 'deal_room': 'Deal Room', - 'to_do': 'TO DO\'s' - } - return self.config.get('deal_details_config', default_config) - - @cached_property - def make_field_history_public(self): - return self.config.get('make_field_history_public', []) - - @cached_property - def not_deals_tenant(self): - return self.config.get('not_deals_tenant', {}) - - @cached_property - def enable_email_tracking_nudge(self): - return self.config.get('enable_email_tracking_nudge', False) - - @cached_property - def special_pivot_month_closeout_day(self): - return self.config.get("special_pivot_month_closeout_day", 3) - - @cached_property - def crm_url(self): - return self.config.get('crm_url') - - @cached_property - def special_pivot_filters(self): - from infra.filters import fetch_all_filters - allowed_filters = {} - filters = fetch_all_filters(self, grid_only=True, is_pivot_special=True) - special_pivot_filters = self.config.get("special_pivot", {}).get("filters", []) - for filt_id, filt in filters.items(): - if filt_id in special_pivot_filters: - allowed_filters[filt_id] = filt - return allowed_filters - - @cached_property - def raw_crr_schema(self): - if self.persona_schemas: - # Getting the schema based on persona. If the user belongs to multiple personas, then merge the schemas - personas = sec_context.get_current_user_personas() - if not personas: - return self.config.get("CRR_schema") - if len(personas) == 1: - return self.persona_schemas.get(personas[0], self.config.get("CRR_schema")) - final_schema = {} - for persona in personas: - schema = self.persona_schemas.get( - persona, self.config.get("CRR_schema")) - for k, v in schema.items(): - if k not in final_schema: - final_schema[k] = v - else: - if type(v) == dict: - final_schema[k].update(v) - elif type(v) == list: - final_schema[k].extend(v) - else: - logger.error( - "This type of type is not supported in schemas of personas yet. please check" - ) - raise Exception( - "This type of type is not supported in schemas of personas yet. please check" - ) - - return final_schema - user_role = sec_context.get_current_user_role() - if user_role and self.crr_role_schemas.get(user_role, None): - return self.crr_role_schemas.get(user_role, {}) - - return self.config.get("CRR_schema") - - @cached_property - def get_persona_schemas(self): - if self.persona_schemas: - personas = sec_context.get_current_user_personas() - if not personas: - return self.config.get("schema") - if len(personas) == 1: - return self.persona_schemas.get(personas[0], self.config.get("schema")) - final_schema = {} - personas = list(set(personas)) - for persona in personas: - schema = self.persona_schemas.get( - persona, self.config.get("schema")) - for k, v in schema.items(): - if k not in final_schema: - final_schema[k] = v - else: - if type(v) == dict: - final_schema[k].update(v) - elif type(v) == list: - final_schema[k].extend(v) - else: - logger.error( - "This type of type is not supported in schemas of personas yet. please check" - ) - raise Exception( - "This type of type is not supported in schemas of personas yet. please check" - ) - - return final_schema - - @cached_property - def get_user_role_schemas(self): - user_role = sec_context.get_current_user_role() - if user_role and self.role_schemas.get(user_role): - return self.role_schemas.get(user_role, {}) - - @cached_property - def raw_schema(self): - person_schema = self.get_persona_schemas - if person_schema: - return person_schema - user_role_schema = self.get_user_role_schemas - if user_role_schema: - return user_role_schema - return self.config.get("schema") - - # - # Field Map - # - @cached_property - def field_map(self): - """ - mapping of standard aviso deal fields to their tenant specific field names - - Returns: - dict -- {'amount': 'as_of_Amount', ...} - """ - return self.config.get("field_map", {}) - - @cached_property - def add_poc_fields_to_indicator_report(self): - """ - mapping of standard aviso deal fields to their tenant specific field names - - Returns: - dict -- {'amount': 'as_of_Amount', ...} - """ - return self.config.get("add_poc_fields_to_indicator_report", False) - - @cached_property - def leading_indicator_ref_stages(self): - """ - mapping of standard aviso deal fields to their tenant specific field names - - Returns: - dict -- {'amount': 'as_of_Amount', ...} - """ - return self.config.get("leading_indicator_ref_stages", ["Validate", "Stakeholder Alignment"]) - - @cached_property - def leading_indicator_poc_timestamp_fields(self): - """ - mapping of standard aviso deal fields to their tenant specific field names - - Returns: - dict -- {'amount': 'as_of_Amount', ...} - """ - return self.config.get("leading_indicator_poc_timestamp_fields", ["POVStartDate", "POVEndDate"]) - - @cached_property - def leading_indicator_bva_fields(self): - """ - mapping of standard aviso deal fields to their tenant specific field names - - Returns: - dict -- {'amount': 'as_of_Amount', ...} - """ - return self.config.get("leading_indicator_bva_fields", - ["BVA_Presented_to_Customer", "BVA_Presented_to_Customer_transition_timestamp"]) - - @cached_property - def stage_transition_timestamp(self): - """ - mapping of standard aviso deal fields to their tenant specific field names - - Returns: - dict -- {'amount': 'as_of_Amount', ...} - """ - return self.config.get("stage_transition_timestamp", "Stage_transition_timestamp") - - @cached_property - # - # write back fields - # - def writeback_fields(self): - crm_writable_fields = [] - for i in self.config["schema"]["deal"]: - if "crm_writable" in i.keys(): - crm_writable_fields.append(self.config["schema"]["deal_fields"][i['field']]) - return crm_writable_fields - - # - # Deal Alert Extended - # - @cached_property - def deal_alerts_fields_extended(self): - """ - Deal Alert Config for extended fields. - """ - return self.config.get("deal_alerts_fields_extended", None) - - @cached_property - def custom_manager_fc_ranks(self): - return self.config.get("custom_manager_fc_ranks", {}) - - @cached_property - def custom_gvp_fc_ranks(self): - return self.config.get("custom_gvp_fc_ranks", {}) - - @cached_property - def sankey_for_lacework(self): - """ - Added sankey config for lacework to handle CS-8700 - This config will be used for only for lacework to load sankey even when deal results fails. - """ - return self.config.get("sankey_for_lacework", False) - - @cached_property - def high_leverage_moments(self): - """ - High leverage moments config is defined to populate high leverage deals insights. - high_leverage_moments = { - 'forecast_category_order': ['Pipeline', 'Upside', 'Commit', 'Closed'], - 'stage_order': ['1-Validate', '2-Qualify', '3-Compete', '4-Negotiate', '5-Selected', '6-End user PO Issued', '8-Closed Won'], - 'days': [7,14,21,28], - 'hlm_threshold': 0.3 - } - """ - return self.config.get("high_leverage_moments", {}) - - @cached_property - def report_custom_fields_dict(self): - """ - report_custom_fields for a tenant - - Returns: - dict -- field name - """ - return self.config.get("report_custom_fields", {}) - - @cached_property - def crr_amount_field(self): - return get_nested(self.config, ["field_map", "crr_amount"]) or 'forecast' - - @cached_property - def amount_field(self): - """ - name of amount field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "amount"]) - - @cached_property - def news_deal_limit(self): - """ - Fetches the news deal limit from the configuration. - - Returns: - int | None -- The deal limit if configured, else None. - """ - return get_nested(self.config, ["dashboard", "news", "deal_limit"]) - - @cached_property - def accountid_field(self): - """ - name of account id field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "accountid"]) - - @cached_property - def meddicscore_field(self): - """ - name of meddicscore field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "meddicscore"]) - - @cached_property - def amount_prev_wk_field(self): - """ - name of amount field for tenant - - Returns: - str -- field name - """ - amount_field = get_nested(self.config, ["field_map", "amount"]) - amount_prev_wk_field = get_nested(self.config, ["field_map", "amount_prev_wk"]) - if not amount_prev_wk_field and amount_field: - amount_prev_wk_field = amount_field + '_prev_wk' - return amount_prev_wk_field - - @cached_property - def crr_amount_prev_wk_field(self): - """ - name of amount field for tenant - - Returns: - str -- field name - """ - amount_field = get_nested(self.config, ["field_map", "crr_amount"]) - amount_prev_wk_field = get_nested(self.config, ["field_map", "crr_amount_prev_wk"]) - if not amount_prev_wk_field and amount_field: - amount_prev_wk_field = amount_field + '_prev_wk' - return amount_prev_wk_field - - @cached_property - def pivot_amount_fields(self): - """ - map of amount field for tenant based on pivot - - Returns: - dict -- pivot:amount_field_name - """ - return self.config.get("pivot_amount_fields", None) - - @cached_property - def close_date_field(self): - """ - name of close date field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "close_date"]) - - @cached_property - def monthly_close_period_enrichment(self): - return self.config.get("monthly_close_period_enrichment", False) - - @cached_property - def export_close_date_field(self): - """ - name of close date field for export (for jfrog) - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "export_close_date"]) - - @cached_property - def original_close_date_field(self): - """ - name of the close date field which is not modified by rev scheduling code, etc - - Returns: - str -- field name - """ - return self.config.get("original_close_date_field", get_nested(self.config, ["field_map", "close_date"])) - - @cached_property - def crr_groupby_field(self): - """ - name of group by field for special pivot - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "CRR_BAND_DESCR"]) - - @cached_property - def crr_ceo_fields(self): - return get_nested(self.config, ['crr_ceo_fields', []]) - - @cached_property - def stage_field(self): - """ - name of stage field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "stage"]) - - @cached_property - def type_field(self): - """ - name of stage field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "type"]) - - @cached_property - def stage_trans_field(self): - """ - name of stage field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "stage_trans"]) - - @cached_property - def get_additional_ai_forecast_diff_fields(self): - """ - return SFDCObject Config Key so as to get the CRM resource name of the deal. - - Returns: - str -- SDFCObject Config key - - This config also used in csv export(CS-19586) - """ - return self.config.get('additional_ai_forecast_diff_fields', []) - - @cached_property - def forecast_category_field(self): - """ - name of forecast category field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "forecast_category"]) - - @cached_property - def manager_forecast_category_field(self): - """ - name of manager forecast category field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "manager_forecast_category"]) - - @cached_property - def gvp_forecast_category_field(self): - """ - name of gvp forecast category field for tenant - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "gvp_forecast_category"]) - - @cached_property - def extend_deal_change_for(self): - return self.config.get("extend_deal_change_for", []) - - @cached_property - def oppmap_forecast_category_field(self): - """ - name of oppmap forecast category field for tenant - - Returns: - str -- field name - """ - return get_nested( - self.config, - ["oppmap", "forecast_category"], - get_nested(self.config, ["field_map", "forecast_category"]), - ) - - def oppmap_deal_type_grouping(self, type): - """ - New/Renewal type values for oppmap - - Returns: - str -- type value Ex: new - """ - return get_nested( - self.config, - ["oppmap", "deal_type_grouping"], - { - "new": ["New"], - "renewal": ["Renewal"], - "cross_Sell/upsell/extensions": [ - "Add-On", - "Add-On Business", - "Amendment", - "Existing Business", - "Upgrade", - "Upgrade or downgrade", - ], - }, - )[type] - - @cached_property - def oppmap_deal_type_options(self): - """ - deal type options for oppmap - - Returns: - str -- type value Ex: new - """ - return get_nested( - self.config, - ["oppmap", "oppmap_deal_type_options"], - [ - ("all", "ALL"), - ("new", "New"), - ("renewal", "Renewal"), - ("cross_Sell/upsell/extensions", "Cross Sell/Upsell/Extensions"), - ], - ) - - """ - The score cutoff where a deal is considered risky""" - - @cached_property - def opp_map_score_cutoff(self): - return get_nested(self.config, ["opp_map", "score_cutoff"]) - - @cached_property - def owner_field(self): - """ - Owner Id of the deal - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "owner_id"]) - - @cached_property - def owner_name_field(self): - """ - Owner Name of Deal - - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "opp_owner"]) - - @cached_property - def owner_id_fields(self): - """ - mapping of owner id fields to drilldown if drilldowns, else None - - Returns: - list -- list of tuples of (owner id field, drilldown) - """ - default_owner_id_fields = [(self.owner_field, None)] - owner_id_fields = get_nested( - self.config, ["owner_id_fields"], default_owner_id_fields - ) - if isinstance(owner_id_fields, dict): - logger.warning( - "using old config format for owner_id_fields please switch") - owner_id_fields = [(k, v) for k, v in owner_id_fields.items()] - return owner_id_fields - - @cached_property - def user_data_name(self): - return self.config.get("UserData", "User") - - @cached_property - def user_email_fld(self): - return self.config.get("User_email_fld", "Email") - - @cached_property - def user_level_persona_fields(self): - if self.persona_schemas: - # When persona schemas are enabled, we show user_level fields based on the fields of that particular persona. - # So including the user_level_fields config in the schema itself - personas = sec_context.get_current_user_personas() - ret_val = {} - for persona in personas: - for k, v in ( - self.persona_schemas.get(persona, {}) - .get("user_level_fields", {}) - .items() - ): - if k not in ret_val: - ret_val[k] = v - else: - if type(v) == list: - ret_val[k].extend(v) - elif type(v) == dict: - ret_val[k].update(v) - else: - ret_val[k] = v - return ret_val if ret_val else {} - - @cached_property - def user_level_role_fields(self): - user_role = sec_context.get_current_user_role() - if user_role and self.role_schemas.get(user_role): - return self.role_schemas.get(user_role).get("user_level_fields", {}) - - @cached_property - def user_level_fields(self): - user_level_persona_fields = self.user_level_persona_fields - user_level_role_fields = self.user_level_role_fields - user_level_fields = copy.deepcopy(self.config.get("user_level_fields", {})) - if user_level_persona_fields: - for fld in user_level_persona_fields['fields']: - if fld not in user_level_fields['fields']: - user_level_fields['fields'].append(fld) - for fld in user_level_persona_fields['dlf_fields']: - if fld not in user_level_fields['dlf_fields']: - user_level_fields['dlf_fields'].append(fld) - - if user_level_role_fields: - for fld in user_level_role_fields['fields']: - if fld not in user_level_fields['fields']: - user_level_fields['fields'].append(fld) - for fld in user_level_role_fields['dlf_fields']: - if fld not in user_level_fields['dlf_fields']: - user_level_fields['dlf_fields'].append(fld) - logger.info("final user level fields {}".format(user_level_fields)) - - return user_level_fields - - @cached_property - def pivot_special_user_level_fields(self): - if self.persona_schemas: - # When persona schemas are enabled, we show user_level fields based on the fields of that particular persona. - # So including the user_level_fields config in the schema itself - personas = sec_context.get_current_user_personas() - ret_val = {} - for persona in personas: - for k, v in ( - self.persona_schemas.get(persona, {}) - .get("pivot_special_user_level_fields", {}) - .items() - ): - if k not in ret_val: - ret_val[k] = v - else: - if type(v) == list: - ret_val[k].extend(v) - elif type(v) == dict: - ret_val[k].update(v) - else: - ret_val[k] = v - return ret_val if ret_val else self.config.get("pivot_special_user_level_fields", {}) - - user_role = sec_context.get_current_user_role() - if user_role and self.role_schemas.get(user_role): - return self.role_schemas.get(user_role).get("pivot_special_user_level_fields", - self.config.get("pivot_special_user_level_fields", {})) - - return self.config.get("pivot_special_user_level_fields", {}) - - @cached_property - def user_name_fld(self): - return self.config.get("User_name_fld", "Name") - - @cached_property - def restrict_lead_deals(self): - """ - restrict_lead_deals - True if lead deals are supposed to be excluded from the reports_db - lead deal identification - starts with 00Q - """ - return self.config.get('restrict_lead_deals', False) - - # - # Field Values - # - - @cached_property - def best_case_values(self): - """ - values of forecast category field that make a deal be considered in best case - optional: falls back to DEFAULT_BEST_CASE_VALS - Returns: - list -- [best case values] - """ - return get_nested( - self.config, ["field_values", "best_case"], DEFAULT_BEST_CASE_VALS - ) - - @cached_property - def commit_values(self): - """ - values of forecast category field that make a deal be considered in commit - optional: falls back to DEFAULT_COMMIT_VALS - - Returns: - list -- [commit values] - """ - return get_nested(self.config, ["field_values", "commit"], DEFAULT_COMMIT_VALS) - - @cached_property - def pipeline_values(self): - """ - values of forecast category field that make a deal be considered in pipeline - optional: falls back to DEFAULT_PIPELINE_VALS - - Returns: - list -- [pipeline values] - """ - return get_nested( - self.config, ["field_values", "pipeline"], DEFAULT_PIPELINE_VALS - ) - - @cached_property - def renewal_values(self): - """ - Values of renewal type deal. - optional: falls back to DEFAULT_RENEWAL_VALS - - Returns: - list -- [renewal values] - """ - return get_nested( - self.config, ["field_values", "renewal"], DEFAULT_RENEWAL_VALS - ) - - @cached_property - def most_likely_values(self): - """ - values of forecast category field that make a deal be considered most likely - optional: falls back to DEFAULT_MOST_LIKELY_VALS - - Returns: - list -- [most likely values] - """ - return get_nested( - self.config, ["field_values", - "most_likely"], DEFAULT_MOST_LIKELY_VALS - ) - - @cached_property - def dlf_values(self): - """ - values of forecast category field that make a deal be considered in dlf - optional: falls back to DEFAULT_DLF_VALS - - Returns: - list -- [dlf values] - """ - return get_nested(self.config, ["field_values", "dlf"], DEFAULT_DLF_VALS) - - # - # Total Fields - # - @cached_property - def special_pivot_total_fields(self): - """ - deal amount fields to compute totals for in deals grid - totals are unfiltered - - Returns: - list -- [(label, deal amount field, mongo operation)] - """ - tot_fields = [] - defualt_totals = [('forecast', "$sum"), ("crr_in_fcst", "$sum")] - label_map = { - v: k for k, v in self.raw_crr_schema.get("deal_fields", {}).items() - } - for field_dtls in get_nested( - self.config, ["totals", "crr_total_fields"], defualt_totals - ): - try: - field, op = field_dtls - except ValueError: - (field,) = field_dtls - op = "$sum" - label = label_map.get(field, field) - if 'ACT_CRR_value' in field: - label = "ACT_CRR" - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - tot_fields.append((label, field, op)) - return tot_fields - - @cached_property - def special_pivot_subtotal_fields(self): - """ - deal amount fields to compute subtotals for in deals grid - subtotals are filtered - - Returns: - list -- [(label, deal amount field, mongo operation)] - """ - tot_fields = [] - default_subtotals = [('forecast', "$sum"), ("crr_in_fcst", "$sum")] - label_map = { - v: k for k, v in self.raw_crr_schema.get("deal_fields", {}).items() - } - for field_dtls in get_nested( - self.config, ["totals", "crr_subtotal_fields"], default_subtotals - ): - try: - field, op = field_dtls - except ValueError: - (field,) = field_dtls - op = "$sum" - label = label_map.get(field, field) - if 'ACT_CRR_value' in field: - label = "ACT_CRR" - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - tot_fields.append((label, field, op)) - return tot_fields - - @cached_property - def same_total_subtotals(self): - """ - True if tenant requires totals and subtotals to be equal - CS-13349 - AppAnnie - """ - return self.config.get('same_total_subtotals', False) - - @cached_property - def total_fields(self): - """ - deal amount fields to compute totals for in deals grid - totals are unfiltered - - Returns: - list -- [(label, deal amount field, mongo operation)] - """ - tot_fields = [] - defualt_totals = [(self.amount_field, "$sum"), ("in_fcst", "$sum")] - label_map = { - v: k for k, v in self.raw_schema.get("deal_fields", {}).items() - } - for field_dtls in get_nested( - self.config, ["totals", "total_fields"], defualt_totals - ): - try: - field, op = field_dtls - except ValueError: - (field,) = field_dtls - op = "$sum" - label = label_map.get(field, field) - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - tot_fields.append((label, field, op)) - return tot_fields - - @cached_property - def subtotal_fields(self): - """ - deal amount fields to compute subtotals for in deals grid - subtotals are filtered - - Returns: - list -- [(label, deal amount field, mongo operation)] - """ - tot_fields = [] - default_subtotals = [(self.amount_field, "$sum"), ("in_fcst", "$sum")] - label_map = { - v: k for k, v in self.raw_schema.get("deal_fields", {}).items() - } - for field_dtls in get_nested( - self.config, ["totals", "subtotal_fields"], default_subtotals - ): - try: - field, op = field_dtls - except ValueError: - (field,) = field_dtls - op = "$sum" - label = label_map.get(field, field) - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - tot_fields.append((label, field, op)) - return tot_fields - - # - # Display Schema - # - def get_field_format(self, field, pivot=None): - try: - if pivot in self.config.get('special_pivot', []): - return next( - x["fmt"] for x in self.account_fields_config if x["field"] == field - ) - else: - return next( - x["fmt"] for x in self.deal_fields_config if x["field"] == field - ) - except StopIteration: - return None - - def get_field_label(self, field, pivot=None): - try: - if pivot in self.config.get('special_pivot', []): - return next( - x["label"] for x in self.account_fields_config if x["field"] == field - ) - else: - return next( - x["label"] for x in self.deal_fields_config if x["field"] == field - ) - except StopIteration: - return field - - @cached_property - def account_fields_config(self): - """ - all accounts fields available to UI, with description of how to format + label them - - Returns: - list -- [{'field': field, 'fmt': fmt, 'label': label}] - """ - return self.raw_crr_schema.get("deal", []) - - @cached_property - def deal_fields_config(self): - """ - all deal fields available to UI, with description of how to format + label them - - Returns: - list -- [{'field': field, 'fmt': fmt, 'label': label}] - """ - return self.raw_schema.get("deal", []) - - @cached_property - def trimmed_deal_fields_config(self): - fields = {} - for field_config in self.deal_fields_config: - fields.update({field_config.get('field'): field_config}) - return fields.values() - - @cached_property - def special_amounts(self): - """ - Special Amount fields which will be useful to add special amount fields and - - Returns: - list -- {'amount_field': "amount_label"} - """ - return self.raw_schema.get("special_amounts", {}) - - @cached_property - def secondary_deal_fields(self): - """ - secondary deal fields that dont appear in deals grid - - Returns: - set -- {fields} - """ - return { - field_dtls["field"] - for field_dtls in self.deal_fields_config - if field_dtls.get("secondary") - } - - def deal_ue_fields(self, pivot_schema="schema"): - """ - user editable deal fields - - Returns: - set -- {fields} - """ - return [ - field_dtls["field"] - for field_dtls in get_nested(self.config, [pivot_schema, "deal"], []) - if field_dtls.get("user_edit") - ] - - def pivot_secondary_deal_fields(self, pivot_schema="schema"): - """ - pivot secondary deal fields that dont appear in deals grid - - Returns: - set -- {fields} - """ - return { - field_dtls["field"] - for field_dtls in get_nested(self.config, [pivot_schema, "deal"], []) - if field_dtls.get("secondary") - } - - @cached_property - def primary_account_fields(self): - """ - primary account fields that appears in accounts grid - - Returns: - set -- {fields} - """ - return { - field_dtls["field"] - for field_dtls in self.account_fields_config - if field_dtls.get("primary") - } - - @cached_property - def primary_deal_fields(self): - """ - primary deal fields that appears in deals grid - - Returns: - set -- {fields} - """ - return { - field_dtls["field"] - for field_dtls in self.deal_fields_config - if field_dtls.get("primary") - } - - @cached_property - def gateway_deal_fields(self): - """ - secondary deal fields that dont appear in deals grid - - Returns: - set -- {fields} - """ - return { - field_dtls["field"] - for field_dtls in self.deal_fields_config - if field_dtls.get("gateway") - } - - @cached_property - def special_gateway_deal_fields(self): - """ - secondary deal fields that dont appear in deals grid - - Returns: - set -- {fields} - """ - return get_nested( - self.config, ["gateway_schema", "special_gateway_deal_fields"], {} - ) - - @cached_property - def gateway_dlf_expanded(self): - """ - boolean to activate expanded dlf fields with node level information - - Returns: - boolean -- - """ - return get_nested( - self.config, ["gateway_schema", "gateway_dlf_expanded"], False - ) - - @cached_property - def gateway_dlf_expanded_field(self): - """ - expanded dlf field with node level information in {label: field} format - - Returns: - dict -- dlf fields with node level information - default - {'dlfs': 'dlfs'} - """ - return get_nested( - self.config, ["gateway_schema", "gateway_dlf_expanded_field"], {'dlfs': 'dlfs'} - ) - - @cached_property - def special_deal_fields(self): - """ - deal fields that get called out in deal card - - Returns: - list -- [(deal field, fe key, label, format)] - """ - return self.raw_schema.get("special_deal_fields", []) - - def pivot_special_deal_fields(self, schema): - """ - deal fields that get called out in deal card - - Returns: - list -- [(deal field, fe key, label, format)] - """ - return get_nested(self.config, [schema, "special_deal_fields"], []) - - @cached_property - def default_hidden_fields(self): - """ - deal fields that get called out in deal card - - Returns: - list -- [(deal field, fe key, label, format)] - """ - return self.raw_schema.get("default_hidden_fields", []) - - @cached_property - def custom_layout_fields(self): - """ - deal fields that get called out in deal card - - Returns: - list -- [(deal field, fe key, label, format)] - """ - return self.raw_schema.get("deal") - - @cached_property - def deal_fields(self): - """ - deal fields to display in deals grid mapped to their tenant specific field names - - Returns: - dict -- {'OpportunityName': 'opp_name'} - """ - - return self._deal_fields() - - @cached_property - def formula_driven_fields(self): - """ - fields that are formula driven and formula - - Returns: - dictionary where key is field_name and value have formula and source - for e.g.: - {"ACVProb": {"formula": "a * b" - "source": {"a": "Amount", - "b": "Probability" }}} - """ - return self.config.get("formula_driven_fields", {}) - - def _deal_fields(self, gateway_call=False, pivot_schema=None, formula_driven_fields=[], segment=None): - d_fields = { - "alert": "alert", - "dealalert": "dealalert", - } # HACK: get alert into deal ... - - secondary_deal_fields = self.secondary_deal_fields - if pivot_schema: - schema = self.config.get(pivot_schema) - secondary_deal_fields = self.pivot_secondary_deal_fields( - pivot_schema=pivot_schema - ) - else: - schema = self.raw_schema - - for label, field in schema.get("deal_fields", {}).items(): - if label in secondary_deal_fields: - if (gateway_call and label in self.gateway_deal_fields) or label in formula_driven_fields: - pass - else: - continue - if field in self.dlf_fields: - field = ".".join(["dlf", field]) - d_fields[label] = field - if gateway_call: - d_fields.update(self.special_gateway_deal_fields) - # logger.info("deal fields %s" % d_fields) - if "segment_schema" in self.config: - segment_schema = self.config.get('segment_schema', {}) - if segment in segment_schema: - deal_fields = segment_schema[segment]["deal_fields"] - for key, value in deal_fields.items(): - d_fields[key] = value - - return d_fields - - @cached_property - def all_account_fields(self): - return self.raw_crr_schema.get("deal_fields", {}) - - def gateway_fields_from_schema(self, schema='schema'): - schema = self.config.get(schema) - deal_fields_config = schema.get("deal", []) - - gateway_deal_fields = [] - for field_dtls in deal_fields_config: - if field_dtls.get('gateway'): - field = field_dtls['field'] - if field in self.dlf_fields: - field = ".".join(["dlf", field]) - gateway_deal_fields.append([field_dtls['label'], field, field_dtls['fmt']]) - return gateway_deal_fields - - @cached_property - def all_deal_fields(self): - d_fields = { - "alert": "alert", - "dealalert": "dealalert", - } # HACK: get alert into deal ... - for label, field in self.raw_schema.get("deal_fields", {}).items(): - if field in self.dlf_fields: - field = ".".join(["dlf", field]) - d_fields[label] = field - return d_fields - - @cached_property - def filter_priority(self): - return self.config.get("filter_priority", []) - - @cached_property - def card_deal_fields(self): - """ - deal fields to display in deal card mapped to their tenant specific field names - optional: falls back to standard deal_fields - - Returns: - dict -- {'OpportunityName': 'opp_name'} - """ - d_fields = { - "alert": "alert", - "dealalert": "dealalert", - } # HACK: get alert into deal ... - for label, field in self.raw_schema.get( - "card_deal_fields", self.raw_schema.get("deal_fields", {}) - ).items(): - if field in self.dlf_fields: - field = ".".join(["dlf", field]) - if field not in self.raw_schema.get( - "excluded_from_deal_card", ["__comment__"] - ): - d_fields[label] = field - return d_fields - - def card_deal_fields_config(self): - """ - all deal card fields available to UI be it combined/irrespective od deal grid, with description of how to format + label them - - Returns: - list -- [{'field': field, 'fmt': fmt, 'label': label}] - """ - return self.raw_schema.get('deal_card', []) if self.raw_schema.get('deal_card', []) else self.deal_fields_config - - def pivot_deal_fields_config(self, pivot_schema="schema"): - """ - all pivot deal fields available to UI, with description of how to format + label them - - Returns: - list -- [{'field': field, 'fmt': fmt, 'label': label}] - """ - return get_nested(self.config, [pivot_schema, "deal"], []) - - def pivot_card_deal_fields(self, pivot_schema="schema"): - """ - deal fields to display in deal card mapped to their tenant specific field names - optional: falls back to standard deal_fields - - Returns: - dict -- {'OpportunityName': 'opp_name'} - """ - d_fields = { - "alert": "alert", - "dealalert": "dealalert", - } # HACK: get alert into deal ... - for label, field in get_nested( - self.config, - [pivot_schema, "card_deal_fields"], - get_nested(self.config, [pivot_schema, "deal_fields"], {}), - ).items(): - if field in self.dlf_fields: - field = ".".join(["dlf", field]) - if field not in get_nested( - self.config, [pivot_schema, "excluded_from_deal_card"], [ - "__comment__"] - ): - d_fields[label] = field - return d_fields - - def crr_card_fields(self): - return self.config.get('crr_card_fields', {}) - - def crr_card_special_fields(self): - return self.config.get('crr_card_special_fields', {}) - - def crr_card_graph_fields(self): - return self.config.get('crr_card_graph_fields', {}) - - @cached_property - def export_hierarchy_fields(self): - return self.config.get('export_hierarchy_fields', True) - - def export_deal_fields(self, schema="schema", special_pivot=False): - """ - deal fields to display in deals export to their tenant specific field names - optional: falls back to standard deal_fields - - Returns: - list -- [(label, key, fmt) ... ] - """ - d_fields = [] - pivot = schema.split('_')[0] - id = '__id__' if pivot in self.config.get('not_deals_tenant', {}).get('special_pivot', []) else 'opp_id' - if schema != 'schema' and schema in self.config: - pivot_schema = self.config.get(schema) - export_fields = pivot_schema.get("export_deal_fields", []) - if export_fields: - fields_order = [x[0] for x in export_fields] - export_fields = {label: field for label, field in export_fields} - else: - fields_order = [x["field"] for x in pivot_schema.get("deal", [])] - export_fields = pivot_schema.get("deal_fields", {}) - else: - export_fields = self.raw_schema.get("export_deal_fields", []) - if export_fields: - fields_order = [x[0] for x in export_fields] - export_fields = {label: field for label, field in export_fields} - else: - fields_order = [x["field"] for x in self.deal_fields_config] - export_fields = self.raw_schema.get("deal_fields", {}) - - for standard_field, db_field in export_fields.items(): - if standard_field[:2] == "__" and standard_field != '__comment__': - continue - label = self.get_field_label(standard_field, pivot=pivot) - fmt = self.get_field_format(standard_field, pivot=pivot) - if db_field in self.dlf_fields: - db_field = ".".join(["dlf", db_field]) - d_fields.extend( - [ - (label + " Status", standard_field, db_field, "dlf"), - ] - ) - if self.dlf_mode.get(db_field.split(".")[-1], None) != "N": - d_fields.append( - (label, standard_field, db_field, "dlf_amount")) - else: - d_fields.append((label, standard_field, db_field, fmt)) - - field_indices = {x: i for (i, x) in enumerate(fields_order)} - # Sort based on standard field name. - ordered_fields = sorted( - [fld for fld in d_fields if fld[1] in fields_order], - key=lambda x: field_indices[x[1]], - ) - all_fields = ordered_fields + [ - fld for fld in d_fields if fld[1] not in fields_order - ] - if special_pivot: - hierarchy_field_config = ('Hierarchy', '__segs', "list") - return [("Id", id, "str"), hierarchy_field_config] + [(label, db_field, fmt) for - (label, standard_field, db_field, fmt) in all_fields] - if self.export_hierarchy_fields: - hierarchy_field_config = ('Hierarchy', 'drilldown_list', "list") - return [("Id", id, "str"), hierarchy_field_config] + [(label, db_field, fmt) for - (label, standard_field, db_field, fmt) in all_fields] - return [("Id", id, "str")] + [(label, db_field, fmt) for - (label, standard_field, db_field, fmt) in all_fields] - - def export_pdf_deal_fields(self): - export_pdf_fields = self.raw_schema.get("export_pdf_deal_fields", []) - if not export_pdf_fields: - return [] - export_fields = [label for label, _, _ in self.export_deal_fields()] - return [label for label in export_pdf_fields if label in export_fields] - - @cached_property - def reload_post_writeback(self): - if 'reload_post_writeback' not in self.config.get('schema', {}): - return None - return self.config.get('schema').get('reload_post_writeback') - - @cached_property - def export_deal_fields_format(self): - if 'export_deal_fields_format' not in self.config.get('schema', {}): - return None - return self.config.get('schema').get('export_deal_fields_format') - - @cached_property - def pull_in_deal_fields(self): - """ - deal fields to display in pull in deals grid to their tenant specific field names - optional: falls back to standard deal_fields - - Returns: - dict -- {'OpportunityName': 'opp_name'} - """ - d_fields = {"__fav__": "__fav__"} - deal_fields_map = self.raw_schema.get("deal_fields", {}) - for label in PULL_IN_LIST: - d_fields[label] = deal_fields_map.get(label, label) - return d_fields - - @cached_property - def pull_in_fields_order(self): - """ - Force pull in deals columns to have an order - - Returns: - dict -- {'label': labels_of_order, - 'fields': fields_used_in_deals} - """ - return { - "label": PULL_IN_LIST, - "fields": [self.pull_in_deal_fields[l] for l in PULL_IN_LIST], - } - - @cached_property - def opp_template(self): - """ - template to make link to source crm system opportunity - - Returns: - str -- url stub - """ - try: - return sec_context.details.get_config("forecast", "tenant", {}).get( - "opportunity_link", "https://salesforce.com/{oppid}" - ) - except AttributeError: - return "https://salesforce.com/{oppid}" - - # - # Filters - # - @cached_property - def open_filter_criteria(self): - """ - mongo db filter criteria for open deals - - Returns: - dict -- {mongo db criteria} - """ - return fetch_filter([self._open_filter_id], self, db=self.db) - - @cached_property - def open_filter_raw(self): - """ - aviso filter syntax criteria for open deal - - Returns: - list -- [{'op': 'has', 'key': 'amt'}] - """ - return fetch_filter([self._open_filter_id], self, filter_type="raw", db=self.db) - - def open_filter(self, deal): - """ - check if a deal is open or not - - Arguments: - deal {dict} -- deal record - - Returns: - bool -- True if open, False if closed - """ - return self._py_open_func(deal, None, None) - - def won_filter(self, deal): - """ - check if a deal is won or not - - Arguments: - deal {dict} -- deal record - - Returns: - bool -- True if open, False if closed - """ - return self._py_won_func(deal, None, None) - - def lost_filter(self, deal): - """ - check if a deal is open or not - - Arguments: - deal {dict} -- deal record - - Returns: - bool -- True if open, False if closed - """ - return self._py_lost_func(deal, None, None) - - @cached_property - def opp_name_field(self): - """ - name of opportunity name field for tenant - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "opp_name"]) - - @cached_property - def _open_filter_id(self): - return get_nested(self.config, ["filters", "open_filter"]) - - @cached_property - def _favourites_filter_id(self): - return get_nested(self.config, ["filters", "favourites_filter"], 'favourites') - - @cached_property - def _py_open_func(self): - return fetch_filter( - [self._open_filter_id], self, filter_type="python", db=self.db - ) - - @cached_property - def _py_won_func(self): - return fetch_filter( - [self._won_filter_id], self, filter_type="python", db=self.db - ) - - @cached_property - def _py_lost_func(self): - return fetch_filter( - [self._lost_filter_id], self, filter_type="python", db=self.db - ) - - @cached_property - def _all_filter_criteria(self): - """ - mongo db filter criteria for won deals - - Returns: - dict -- {mongo db criteria} - """ - return fetch_filter([self._alldeals_filter_id], self, db=self.db) - - @cached_property - def _alldeals_filter_id(self): - return get_nested(self.config, ["filters", "all_filter"], 'All') - - @cached_property - def multiple_filter_apply_or(self): - return self.config.get("multiple_filter_apply_or", False) - - @cached_property - def ai_driven_deals_buckets(self): - default_expressions = {"Pullins": {"color": '#2bccff'}, - "Aviso AI- Predicted Wins": {"expr": "win_prob_threshold < win_prob < 1.0", - "color": '#800080'}} - return self.config.get("ai_driven_deals_buckets", default_expressions) - - @cached_property - def default_currency(self): - """ - Returns default currency set for the tenant to serve in notifications. - """ - td = sec_context.details - return td.get_config('forecast', 'tenant', {}).get('notifications_default_currency') or '$' - - # event-based nudge configs - @cached_property - def event_based_nudge_config(self): - """ - ... Set nudge test_mode=True/False for testing purpose - ... Set nudge enable=True/False to enable/disable event subscription - """ - config_params = { - "debug": False, - "eb_score_hist_dip_nudge": {'test_mode': True, 'test_email': 'amit.khachane@aviso.com', - 'enable': False}, - "eb_scenario_nudge": {'test_mode': True, 'test_email': 'amit.khachane@aviso.com', 'enable': False}, - "eb_dlf_nudge": {'test_mode': True, 'test_email': 'amit.khachane@aviso.com', 'enable': False}, - "eb_pulledin_nudge": {'test_mode': True, 'test_email': 'amit.khachane@aviso.com', 'enable': False, - "criteria": {'pulledin': True, 'terminal_fate': 'N/A'}}, - } - return get_nested( - self.config, ['nudge_config', 'event_based_nudge_config'], - config_params - ) - - # Ringcentral Nudges Filters - def nudge_filter_criteria(self, filter_id='Open Deals', root_node=None): - """ - mongo db filter criteria for nudge deals - Returns: - dict -- {mongo db criteria} - """ - # replicable_key = config.config.get('schema', {}).get('deal_fields', {}).get('Amount') - config = DealConfig() - if root_node is None: - logger.exception("Root node not found, passing empty criteria") - return {} - criteria = fetch_filter([filter_id], config, root_node=root_node) - return criteria - - # --x--Ringcentral Nudges Filters--x-- - - # - # DLF Config - # - @cached_property - def primary_dlf_field(self): - """ - main dlf field to use for other features like opp map and deal changes - - Returns: - str -- dlf field name - """ - try: - return next( - k for k, v in self.config.get("dlf", {}).items() if "primary" in v - ) - except StopIteration: - return None - - @cached_property - def analytics_dlf_field(self): - """ - dlf field to use in analytics field - - Returns: - str -- dlf field name - """ - try: - return next( - k - for k, v in self.config.get("dlf", {}).items() - if "use_in_pipeline_analytics" in v - ) - except StopIteration: - return None - - @cached_property - def dlf_mode(self): - """ - mapping of dlf field to dlf mode - 'N': no amount - 'O': optional amount - - Returns: - dict -- {'in_fcst': 'N'} - """ - return { - field: field_config["mode"] - for field, field_config in self.config.get("dlf", {}).items() - } - - @cached_property - def dlf(self): - """ - return all dlf - """ - return self.config.get("dlf", {}) - - @cached_property - def dlf_reports(self): - return self.config.get("dlf_reports", False) - - @cached_property - def top_deals_count(self): - """ - return how many deals should be shown in top_deals section in dashboard - """ - return self.config.get("top_deals_count", 30) - - @cached_property - def dlf_fields(self): - """ - all dlf fields - - Returns: - list -- [dlf field] - """ - return self.config.get("dlf", {}).keys() - - @cached_property - def oppmap_dlf_field(self): - return get_nested( - self.config, - ["oppmap", "dlf_field"], - 'in_fcst' - ) - - @cached_property - def dlf_mismatch_default(self): - """ - dlf_mismatch_default: True if we want to show mismatch wrt defaults - """ - return self.config.get("dlf_mismatch_default", False) - - @cached_property - def dlf_amount_field(self): - """ - mapping of dlf field to deal amount field used to back dlf - optional: falls back to tenants amount field - - Returns: - dict -- {'in_fcst': 'amount'} - """ - return { - field: field_config.get("amount_field", self.amount_field) - for field, field_config in self.config.get("dlf", {}).items() - } - - @cached_property - def change_dlf_with_writeback(self): - """ - mapping of field name with it's values. During writeback if there is change in field name - and value matches. DLF is toggled true - - Returns: - dict -- {'ManagerForecastCategory':['Commit','Best Case','Closed Won']} - """ - return self.config.get('change_dlf_with_writeback', []) - - @cached_property - def dlf_secondary_amount_field(self): - """ - mapping of dlf field to deal secondary amount field used to back dlf - - Returns: - dict -- {'in_fcst': 'amount'} - """ - return { - field: field_config.get("secondary_amount_field", "") - for field, field_config in self.config.get("dlf", {}).items() - } - - @cached_property - def multi_dlf(self): - return get_nested(self.config, ["multi_dlf"], False) - - @cached_property - def dlf_adorn_fields(self): - """ - mapping of dlf field to extra deal fields to adorn each on each dlf to see state at time forecast was made - - Returns: - dict -- {'in_fcst': {'win_prob': 'win_prob',}} - """ - return { - field: field_config.get( - "adorn_field", self._default_dlf_adorn_fields) - for field, field_config in self.config.get("dlf", {}).items() - } - - @cached_property - def dlf_crr_adorn_fields(self): - """ - mapping of dlf field to extra deal fields to adorn each on each dlf to see state at time forecast was made - - Returns: - dict -- {'in_fcst': {'win_prob': 'win_prob',}} - """ - return { - field: field_config.get( - "adorn_field", self._default_crr_dlf_adorn_fields) - for field, field_config in self.config.get("dlf", {}).items() - } - - @cached_property - def _default_dlf_adorn_fields(self): - return { - "score": "win_prob", - "stage": self.stage_field, - "forecastcategory": self.forecast_category_field, - "raw_amt": self.amount_field, - "closedate": self.close_date_field, - } - - @cached_property - def _default_crr_dlf_adorn_fields(self): - return { - "score": "win_prob", - "stage": self.stage_field, - "forecastcategory": self.forecast_category_field, - "raw_amt": self.crr_amount_field, - "closedate": self.close_date_field, - } - - def deals_dlf_rendered_config(self, node): - """ - dlf config rendered for consumption by front end - - Returns: - dict -- {dlf config} - """ - dlf_config = {} - for field, dtls in self.config.get("dlf", {}).items(): - mode = dtls["mode"] - dlf_config[field] = { - "has_amt": mode != "N", - "amt_editable": mode == "O", - "option_editable": True, - } - try: - if get_node_depth(node) >= dtls["hide_at_depth"]: - dlf_config[field]["hide"] = True - except: - pass - if "options" in dtls: - dlf_config[field]["options"] = dtls["options"] - # TODO: hide at depth grossness - return dlf_config - - @cached_property - def dlf_rendered_config(self): - """ - dlf config rendered for consumption by front end - - Returns: - dict -- {dlf config} - """ - dlf_config = {} - for field, dtls in self.config.get("dlf", {}).items(): - mode = dtls["mode"] - dlf_config[field] = { - "has_amt": mode != "N", - "amt_editable": mode == "O", - "option_editable": True, - } - if "options" in dtls: - dlf_config[field]["options"] = dtls["options"] - # TODO: hide at depth grossness - return dlf_config - - def dlf_locked_filter(self, deal, field): - """ - check if a deal is locked in or out of forecast - - Arguments: - deal {dict} -- deal record - field {str} -- dlf field name - - Returns: - bool -- True if locked in, False if locked out, None if not locked - """ - for state, filter_func in self._dlf_py_locked_func.get(field, {}).items(): - if filter_func(deal, None, None): - return state - - def ue_locked_filter(self, deal, field, pivot_schema='schema'): - - for state, filter_func in self._user_edit_dlf_py_locked_func(pivot_schema=pivot_schema).get(field, - {}).items(): - if filter_func(deal, None, None): - return state - return False - - def dlf_default_filter(self, deal, field): - """ - check if a deal if default to in our out of forecas - - Arguments: - deal {dict} -- deal record - field {str} -- dlf field name - - Returns: - bool -- True if default in, False if default out - """ - for state, filter_func in self._dlf_py_default_func.get(field, {}).items(): - if filter_func(deal, None, None): - return state - return self._dlf_default_values[field] - - @cached_property - def favourites_filter_criteria(self): - """ - mongo db filter criteria for favourite deals - - Returns: - dict -- {mongo db criteria} - """ - return fetch_filter([self._favourites_filter_id], self, db=self.db) - - @cached_property - def _dlf_default_values(self): - default_values = {} - for field, field_config in self.config.get("dlf").items(): - try: - default_values[field] = field_config["options"][0]["val"] - except (KeyError, IndexError): - default_values[field] = False - return default_values - - @cached_property - def _dlf_py_locked_func(self): - return { - field: { - _convert_state(state): fetch_filter( - filter_ids, self, filter_type="python", db=self.db - ) - for state, filter_ids in field_config.get( - "locked_filters", {} - ).items() - } - for field, field_config in self.config.get("dlf").items() - } - - def _user_edit_dlf_py_locked_func(self, pivot_schema=None): - deal_fields = self.pivot_deal_fields_config(pivot_schema=pivot_schema) - locked_filter_py = {} - for deal_field in deal_fields: - is_ue_field = deal_field.get('user_edit', False) - if is_ue_field: - locked_filters = deal_field.get('locked_filters', {}) - if locked_filters: - locked_filter_py.update({ - deal_field['field']: { - _convert_state(state): fetch_filter( - filter_ids, self, filter_type="python", db=self.db - ) - for state, filter_ids in locked_filters.items() - } - }) - - return locked_filter_py - - @cached_property - def _dlf_py_default_func(self): - return { - field: { - _convert_state(state): fetch_filter( - filter_ids, self, filter_type="python", db=self.db - ) - for state, filter_ids in field_config.get( - "default_filters", {} - ).items() - } - for field, field_config in self.config.get("dlf").items() - } - - # - # Dashboard Config - # Adaptive metrics config - @cached_property - def adaptive_metrics_categories(self): - cats = {} - am_categories = ( - self.config["dashboard"] - .get("adaptive_metrics_categories", {}) - .get("categories", {}) - ) - - for category in am_categories: - cats[category] = [] - - for (field_name, field_filter, field_tot_fields) in am_categories[category]: - - if "handler" in field_filter: - cats[category].append( - (field_name, field_filter, field_tot_fields)) - elif "get_ratio" in field_filter: - cats[category].append( - (field_name, field_filter, field_tot_fields)) - else: - try: - field = self.amount_field - label, op = field_tot_fields - if ( - "amount_fields" - in self.config["dashboard"]["adaptive_metrics_categories"] - ): - if ( - category - in self.config["dashboard"][ - "adaptive_metrics_categories" - ]["amount_fields"] - ): - if ( - field_name - in self.config["dashboard"][ - "adaptive_metrics_categories" - ]["amount_fields"][category] - ): - field = self.config["dashboard"][ - "adaptive_metrics_categories" - ]["amount_fields"][category][field_name] - - except ValueError: - (field,) = field_tot_fields - op = "$sum" - - cats[category].append( - ( - field_name, - parse_filters(field_filter, self), - [[label, field, op]], - ) - ) - - return cats - - @cached_property - def adaptive_metrics_additional_handler_filters(self): - am_handlers = ( - self.config["dashboard"] - .get("adaptive_metrics_categories", {}) - .get("additional_handler_filters", {}) - ) - for handler in am_handlers: - am_handlers[handler] = parse_filters(am_handlers[handler], self) - return am_handlers - - @cached_property - def adaptive_metrics_additinal_info(self): - cats = {} - am_additinal_info = ( - self.config["dashboard"] - .get("adaptive_metrics_categories", {}) - .get("additinal_info_categories", {}) - ) - - for category in am_additinal_info: - cats[category] = defaultdict(dict) - for (field_name, additinal_info_filter) in am_additinal_info[category]: - cats[category][field_name] = defaultdict(dict) - if "h" in additinal_info_filter: - cats[category][field_name]["past_count"] = additinal_info_filter[ - "h" - ] - if "f" in additinal_info_filter: - cats[category][field_name]["future_count"] = additinal_info_filter[ - "f" - ] - if "d" in additinal_info_filter: - cats[category][field_name][ - "difference_with" - ] = additinal_info_filter["d"] - if "tt" in additinal_info_filter: - cats[category][field_name]["tooltip"] = { - "type": "text", - "text": additinal_info_filter["tt"], - } - if "sl" in additinal_info_filter: - cats[category][field_name]["sublabel"] = additinal_info_filter["sl"] - - return cats - - @cached_property - def adaptive_metrics_additional_filters(self): - cats = {} - am_additinal_info = ( - self.config["dashboard"] - .get("adaptive_metrics_categories", {}) - .get("additional_filters", {}) - ) - - for category in am_additinal_info: - cats[category] = defaultdict(dict) - for (field_name, additinal_info_filter) in am_additinal_info[category]: - cats[category][field_name] = defaultdict(dict) - if "close_date" in additinal_info_filter: - cats[category][field_name]["close_date_in"] = additinal_info_filter[ - "close_date" - ] - if "created_date" in additinal_info_filter: - cats[category][field_name]["created_date"] = additinal_info_filter[ - "created_date" - ] - - return cats - - @cached_property - def adaptive_metrics_views_order(self): - order_info = ( - self.config["dashboard"] - .get("adaptive_metrics_categories", {}) - .get("views_order", []) - ) - return order_info - - @cached_property - def adaptive_metrics_cache_level(self): - cache_level = ( - self.config["dashboard"] - .get("adaptive_metrics_cache_level", 2) - ) - return cache_level - - @cached_property - def leaderboard_cache_level(self): - cache_level = ( - self.config["dashboard"] - .get("leaderboard_cache_level", 2) - ) - return cache_level - - @cached_property - def adaptive_metrics_views_format(self): - cats = {} - format_info = ( - self.config["dashboard"] - .get("adaptive_metrics_categories", {}) - .get("views_format", {}) - ) - for category in format_info: - cats[category] = defaultdict(dict) - if "fmt" in format_info[category]: - cats[category]["format"] = format_info[category]["fmt"] - return cats - - @cached_property - def adaptive_metrics_close_date_aware(self): - cats = {} - close_date_aware_info = ( - self.config["dashboard"] - .get("adaptive_metrics_categories", {}) - .get("close_date_aware", {}) - ) - for category in close_date_aware_info: - cats[category] = [] - for field_name in close_date_aware_info[category]: - cats[category].append(field_name) - return cats - - # Coaching Leaderboard config - @cached_property - def coaching_leaderboard_categories(self): - cats = {} - cl_categories = ( - self.config["dashboard"] - .get("coaching_leaderboard_categories", {}) - .get("categories", []) - ) - - # for category in cl_categories: - # cats[category] = [] - # - # for badge_name in cl_categories[category]: - # cats[category].append(badge_name) - - return cl_categories - - @cached_property - def tenant_diff_node(self): - cl_categories = ['lume.com', 'lumenbackup.com', 'netapp_pm.com', ] - - return cl_categories - - @cached_property - def tenant_diff_owner(self): - cl_categories = ['netapp.com'] - - return cl_categories - - @cached_property - def tenant_node_rename(self): - cl_categories = ['cisco.com'] - - return cl_categories - - @cached_property - def pqr_data(self): - """ - value for number of quarters to be considered for calculating time_threshold in deal_velocity badge in coaching - leaderboard. - """ - return self.config.get("dashboard", {}).get("pqr_data", {}) - - @cached_property - def deal_velocity_past_n_qtrs(self): - """ - value for number of quarters to be considered for calculating time_threshold in deal_velocity badge in coaching - leaderboard. - """ - return self.config.get("dashboard", {}).get("deal_velocity_past_n_qtrs", 4) - - # Deals Config - @cached_property - def deal_categories(self): - cats = [] - for cat_label, cat_filter, cat_tot_fields in get_nested( - self.config, ["dashboard", "deal_categories", "categories"], [] - ): - try: - label, op = cat_tot_fields - if self.config["dashboard"]["deal_categories"].get("amount_fields"): - field = self.config["dashboard"]["deal_categories"][ - "amount_fields" - ][cat_label] - else: - field = self.amount_field - except ValueError: - (field,) = cat_tot_fields - op = "$sum" - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - cats.append( - (cat_label, parse_filters( - cat_filter, self), [[label, field, op]]) - ) - return cats - - @cached_property - def totals_segmented_view(self): - """ - if this is set to true for segmented tenant, - subtotals and total will have same value on selection - of particular segment deals - """ - return self.config.get("totals_segmented_view", False) - - # - # Dashboard Category sort fields - # - @cached_property - def category_amount_fields(self): - cat_amt_fields = {} - if self.config["dashboard"]["deal_categories"].get("amount_fields"): - for cat_label, cat_filter, cat_tot_fields in get_nested( - self.config, ["dashboard", "deal_categories", "categories"], [] - ): - cat_amt_fields[cat_label] = self.config["dashboard"]["deal_categories"][ - "amount_fields" - ][cat_label] - return cat_amt_fields - else: - return None - - # - # covid Dashboard Config - # - - @cached_property - def covid_deal_categories(self): - cats = [] - for cat_label, cat_filter, cat_tot_fields in get_nested( - self.config, ["dashboard", - "covid_deal_categories", "categories"], [] - ): - try: - label, op = cat_tot_fields - field = self.amount_field - except ValueError: - (field,) = cat_tot_fields - op = "$sum" - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - cats.append( - (cat_label, parse_filters( - cat_filter, self), [[label, field, op]]) - ) - return cats - - @cached_property - def how_it_changed(self): - return self.config.get('new_home_page', {}).get('how_it_changed', []) - - @cached_property - def pipeline_quality_categories(self): - return self.config.get('new_home_page', {}).get('pipeline_quality_categories', []) - - @cached_property - def deals_stage_map_cached(self): - return self.config.get('deals_stage_map_cached', False) - - @cached_property - def pipe_dev_gbm_fields(self): - default_pipe_dev_gbm_fields = ['node', 'period', '__segs', 'forecast'] - return self.config.get('pipe_dev_gbm_fields', default_pipe_dev_gbm_fields) - - @cached_property - def deal_changes_categories(self): - cats = [] - for cat_label, cat_filter, cat_tot_fields in get_nested( - self.config, - ["dashboard", "deal_changes", "categories"], - self._default_deal_changes["categories"], - ): - try: - field, op = cat_tot_fields - except ValueError: - (field,) = cat_tot_fields - op = "$sum" - label = field - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - cats.append( - ( - cat_label, - parse_filters(cat_filter, self, hier_aware=False), - [[label, field, op]], - ) - ) - return cats - - @cached_property - def deal_changes_categories_won(self): - cats = [] - for cat_label, cat_filter, cat_tot_fields in get_nested( - self.config, - ["dashboard", "deal_changes", "categories"], - self._default_deal_changes["categories"], - ): - if cat_label == "Won": - try: - field, op = cat_tot_fields - except ValueError: - (field,) = cat_tot_fields - op = "$sum" - label = field - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - cats.append( - ( - cat_label, - parse_filters(cat_filter, self, hier_aware=False), - [[label, field, op]], - ) - ) - return cats - - @cached_property - def deal_changes_leaderboards(self): - lbs = get_nested( - self.config, - ["dashboard", "deal_changes", "leaderboards"], - self._default_deal_changes["leaderboards"], - ) - for lb, lb_dtls in lbs.items(): - if "schema" not in lb_dtls: - lb_dtls["schema"] = self.dashboard_deal_format - if "total_name" not in lb_dtls: - lb_dtls["total_name"] = "Amount" - if "arrow" not in lb_dtls: - lb_dtls["arrow"] = False - return lbs - - @cached_property - def deal_changes_leaderboards_label_performer_desc(self): - return get_nested( - self.config, ["dashboard", "deal_changes", "leaderboards", "desc"], "amount" - ) - - @cached_property - def deal_changes_pipe_field(self): - return get_nested( - self.config, ["dashboard", "deal_changes", - "pipe_field"], "tot_won_and_fcst" - ) - - @cached_property - def deal_changes_categories_order(self): - return get_nested( - self.config, - ["dashboard", "deal_changes", "categories_order"], - self._default_deal_changes["categories_order"], - ) - - @cached_property - def deal_changes_leaderboards_order(self): - return get_nested( - self.config, - ["dashboard", "deal_changes", "leaderboards_order"], - self._default_deal_changes["leaderboards_order"], - ) - - @cached_property - def deal_changes_default_key(self): - return get_nested( - self.config, - ["dashboard", "deal_changes", "default_key"], - self._default_deal_changes["default_key"], - ) - - @cached_property - def account_categories(self): - """ - for top account dashboard feature - the filters + labels to split dealts out by - - Returns: - list -- [(cat label, {cat filter}, [cat sum fields]) for each category] - """ - cats = [] - for cat_label, cat_filter, cat_tot_fields in get_nested( - self.config, ["dashboard", "accounts", "categories"] - ): - try: - field, op = cat_tot_fields - except ValueError: - (field,) = cat_tot_fields - op = "$sum" - label = field - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - cats.append( - ( - cat_label, - parse_filters(cat_filter, self, hier_aware=False), - [[label, field, op]], - ) - ) - return cats - - @cached_property - def account_fields_and_ops(self): - """ - mapping from account category label to deal fields and operations to perform on them for top accounts - - Returns: - dict -- {cat label: [(field label, db field name, db operation)]} - """ - cat_field_map = defaultdict(list) - for category, fields_dtls in get_nested( - self.config, ["dashboard", "accounts", "fields"], {} - ).items(): - for field_dtls in fields_dtls: - field, op, _, label, _ = field_dtls - if field in self.dlf_fields: - field = ".".join(["dlf", field, "%(node)s", "dlf_amt"]) - cat_field_map[category].append((field, field, op)) - return cat_field_map - - @cached_property - def account_schema(self): - """ - mapping from account category label to deal schema for category - - Returns: - dict -- {cat label: {deal schema}} - """ - cat_schemas = {} - for category, fields_dtls in get_nested( - self.config, ["dashboard", "accounts", "fields"], {} - ).items(): - cat_schemas[category] = [ - {"fmt": fmt, "label": label, "field": field, - "is_opp_name": is_opp_name} - for field, _, fmt, label, is_opp_name in fields_dtls - ] - return cat_schemas - - @cached_property - def deals_schema(self): - """ - mapping from deal category label to deal schema for category - - Returns: - dict -- {cat label: {deal schema}} - """ - cat_schemas = {} - for category, fields_dtls in get_nested( - self.config, ["dashboard", "deal_categories", "fields"], {} - ).items(): - cat_schemas[category] = [ - {"fmt": fmt, "label": label, "field": field} - for field, _, fmt, label in fields_dtls - ] - return cat_schemas - - @cached_property - def account_group_fields(self): - """ - deal fields to group by for top accounts feature - - Returns: - list -- [db deal fields] - """ - return get_nested(self.config, ["dashboard", "accounts", "group_fields"], []) - - @cached_property - def leaderboard_previous_values(self): - """ - previous values config for all leaderboards - - Returns: - dict -- {cat label: {prev config schema}} - """ - return get_nested(self.config, ["dashboard", "leaderboard_previous_values"], {}) - - @cached_property - def account_sort_fields(self): - """ - deal fields to sort by for top accounts feature - - Returns: - dict -- {cat label: [deal fields]} - """ - sorts = get_nested( - self.config, ["dashboard", "accounts", "sort_fields"], {}) - return { - cat: [ - (field, 1) if isinstance(field, str) else field - for field in fields - ] - for cat, fields in sorts.items() - } - - @cached_property - def deal_changes_categories_amounts(self): - return get_nested( - self.config, - ["dashboard", "deal_changes", "categories_amounts"], - self._default_deal_changes["categories_amounts"], - ) - - @cached_property - def show_winscore_in_dashboard_top_deals(self): - return get_nested( - self.config, - ["dashboard", 'show_winscore_in_dashboard_top_deals'], {}) - - @cached_property - def dashboard_deal_format(self): - # Need a way to make it easier to the correct subset of fields - forecast_cat_label = None - forecast_cat_field = None - for label, deal_field in self.raw_schema.get("deal_fields", {}).items(): - if deal_field == self.forecast_category_field: - forecast_cat_field = label - for x in self.deal_fields_config: - if x["field"] == label: - forecast_cat_label = x["label"] - break - break - return [ - {"fmt": "str", "label": "Opportunity Name", "field": "OpportunityName"}, - {"fmt": "str", "label": "Owner", "field": "OpportunityOwner"}, - {"fmt": "amount", "label": "Amount", "field": "Amount"}, - {"fmt": "excelDate", "label": "Close Date", "field": "CloseDate"}, - {"fmt": "str", "label": forecast_cat_label, "field": forecast_cat_field}, - {"field": "win_prob", "fmt": "prob", "label": "Aviso Score"}, - ] - - # TODO: BS How to change this to use covid fields - @cached_property - def covid_dashboard_deal_format(self): - # Need a way to make it easier to the correct subset of fields - forecast_cat_label = None - forecast_cat_field = None - for label, deal_field in self.raw_schema.get("deal_fields", {}).items(): - if deal_field == self.oppmap_forecast_category_field: - forecast_cat_field = label - for x in self.deal_fields_config: - if x["field"] == label: - forecast_cat_label = x["label"] - break - break - return [ - {"fmt": "str", "label": "Opportunity Name", "field": "OpportunityName"}, - { - "field": "__covid__", - "fmt": "", - "label": self.covid_labellings.get("aviso_column", "Covid"), - }, - {"fmt": "str", "label": "Owner", "field": "OpportunityOwner"}, - {"fmt": "str", "label": forecast_cat_label, "field": forecast_cat_field}, - {"fmt": "excelDate", "label": "Close Date", "field": "CloseDate"}, - {"fmt": "amount", "label": "Amount", "field": "Amount"}, - ] - - @cached_property - def covid_labellings(self): - return get_nested(self.config, ["dashboard", "covid_labellings"], {}) - - @cached_property - def activity_metrics_top_20_deals_filter(self): - # deals filter to find top 20 deals for activity metrics graph in dashboard - return get_nested(self.config, ['dashboard', 'activity_metrics', 'top_20_deals_filter'], {}) - - @cached_property - def week_on_week_filters(self): - default_filters = {'commit': {'ManagerForecastCategory': {'$in': ['Commit']}}, - 'open_pipeline': {'as_of_StageTrans': {'$nin': ['1', '99']}}} - return get_nested(self.config, ['week_on_week_filters'], default_filters) - - @cached_property - def segment_amount(self): - return get_nested(self.config, ["dashboard", "segment_amount"], {}) - - @cached_property - def activity_metrics_deal_filters(self): - default_filter = [[u'Commit Deals', - [{u'key': self.forecast_category_field, u'op': u'in', u'val': [u'Commit']}], - [u'amount', u'$sum']], - [u'Most Likely Deals', - [{u'key': self.forecast_category_field, u'op': u'in', u'val': [u'Most Likely']}], - [u'amount', u'$sum']], - [u'Best Case Deals', - [{u'key': self.forecast_category_field, u'op': u'in', u'val': [u'Best Case']}], - [u'amount', u'$sum']]] - cats = [] - for cat_label, cat_filter, cat_tot_fields in get_nested(self.config, ['new_home_page', - 'activity_metrics', - 'deal_filters'], default_filter): - try: - label, op = cat_tot_fields - field = self.amount_field - except ValueError: - field, = cat_tot_fields - op = '$sum' - if field in self.dlf_fields: - field = '.'.join(['dlf', field, '%(node)s', 'dlf_amt']) - cats.append((cat_label, parse_filters(cat_filter, self, hier_aware=False), [[label, field, op]])) - return cats - - @cached_property - def engagement_grade(self): - return get_nested(self.config, ['field_map', 'engagement_grade'], 'engagement_grade') - - @cached_property - def oppmap_labellings(self): - return get_nested(self.config, ["dashboard", "oppmap_labellings"], {}) - - @cached_property - def standard_oppmap_map_types(self): - oppmap_types = get_nested( - self.config, - ["oppmap", "standard_oppmap_map_types"], - DEFAULT_STANDARD_OPPMAP_MAP_TYPES, - ) - oppmap_type_tuples = [] - for map_type in oppmap_types: - oppmap_type_tuples.append((map_type, oppmap_types[map_type])) - return oppmap_type_tuples - - @cached_property - def covid_oppmap_map_types(self): - oppmap_types = get_nested( - self.config, - ["oppmap", "covid_oppmap_map_types"], - DEFAULT_COVID_OPPMAP_MAP_TYPES, - ) - oppmap_type_tuples = [] - for map_type in oppmap_types: - oppmap_type_tuples.append((map_type, oppmap_types[map_type])) - return oppmap_type_tuples - - @cached_property - def segment_field(self): - return self.config.get("segment_field", None) - - # TODO: make it fuller if possible - - @cached_property - def nudge_insight_facts(self): - """ - config notebook - https://jupyter.aviso.com/user/amitk/notebooks/amitk/Ticket%20Specific/AV-11394.ipynb - """ - return get_nested( - self.config, ['insight_config', 'nudge_insight_facts'], {}) - - @cached_property - def insight_config(self): - return self.config.get("insight_config", {}) - - @cached_property - def insight_task_config(self): - return self.config.get('insight_task_config', {}) - - @cached_property - def custom_stage_ranks(self): - return self.config.get("custom_stage_ranks", {}) - - @cached_property - def custom_fc_ranks(self): - return self.config.get("custom_fc_ranks", {}) - - @cached_property - def custom_fc_ranks_default(self): - dict_ = {"pipeline": 1, "upside": 2, "most likely": 3, "commit": 4} - return self.config.get("custom_fc_ranks", dict_) - - @cached_property - def close_date_pushes_flds(self): - CLOSEDATE_FIELD_MAP = { - "total_pushes": "close_date_total_pushes", - "months_pushed": "close_date_months_pushed", - } - return self.config.get("close_date_pushes_flds", CLOSEDATE_FIELD_MAP) - - @cached_property - def use_grouper_flag(self): - return self.config.get("use_grouper_flag", False) - - @cached_property - def weekly_report_dimensions(self): - return self.config.get("weekly_report_dimensions", []) - - @cached_property - def dimensions(self): - return self.config.get("dimensions", []) - - @cached_property - def fm_config(self): - return self.config.get("fm_config", {}) - - @cached_property - def custom_fc_ranks(self): - return self.config.get("custom_fc_ranks", {}) - - @cached_property - def bookingstimeline(self): - return self.config.get("bookingstimeline", False) - - @cached_property - def anomaly_config(self): - return self.config.get("anomaly_config", {}) - - @cached_property - def stale_nudge_enable(self): - return self.config.get("stale_nudge_enable", False) - - @cached_property - def crm_hygiene_fld(self): - return self.config.get("crm_hygiene_fld", []) - - @cached_property - def past_closedate_enabled(self): - return self.config.get("past_closedate_enabled", False) - - @cached_property - def close_date_thresh(self): - return self.config.get("close_date_thresh", 15) - - @cached_property - def frequent_fld_nudge(self): - return self.config.get("frequent_fld_nudge", False) - - @cached_property - def frequent_fld(self): - return self.config.get("frequent_fld", "NextStep") - - @cached_property - def frequency_eoq_time(self): - return self.config.get("frequency_eoq_time", 14) - - @cached_property - def pipeline_nudge(self): - return self.config.get("pipeline_nudge", False) - - @cached_property - def pipeline_fields(self): - fields = self.config.get("pipeline_fields", {}) - if fields: - plan_field = fields.get("plan_field") - booked_field = fields.get("booked_field") - top_field = fields.get("top_field") - rollup_field = fields.get("rollup_field") - - return { - "plan_field": plan_field, - "booked_field": booked_field, - "top_field": top_field, - "rollup_field": rollup_field, - } - else: - return {} - - @cached_property - def pipeline_ratio(self): - return self.config.get("pipeline_ratio", 3) - - @cached_property - def changed_deals_limit(self): - return self.config.get("changed_deals_limit", 200) - - @cached_property - def exclude_competitors(self): - return self.config.get("exclude_competitors", ["no competition"]) - - @cached_property - def late_stg_thresh(self): - return self.config.get("late_stg_thresh", 40.0) - - @cached_property - def close_date_update(self): - return self.config.get("close_date_update", False) - - @cached_property - def update_thresh(self): - return self.config.get("update_thresh", 30) - - @cached_property - def close_date_thresh(self): - return self.config.get("close_date_thresh", 30) - - @cached_property - def manager_only_cd_no_update(self): - return self.config.get("manager_only_cd_no_update", False) - - # @cached_property - # def at_risk_deals_enabled(self): - # return self.config.get("at_risk_deals_enabled", False) - - # @cached_property - # def low_pipeline_threshold(self): - # return self.config.get('low_pipeline_threshold', 15.0) - - @cached_property - def low_netxq_pipeline_nudge(self): - return self.config.get("low_netxq_pipeline_nudge", False) - - @cached_property - def commit_not_in_dlf(self): - return self.config.get("commit_not_in_dlf_nudge", False) - - @cached_property - def competitor_nudge_enabled(self): - return self.config.get("competitor_nudge_enabled", False) - - # AV-996 - @cached_property - def stale_nudge_thresh(self): - return self.config.get("stale_nudge_thresh", 40) - - @cached_property - def stale_nudge_config(self): - config_params = { - 'deal_filter_id': 'Filter_Closedate_Stage', - 'deal_cnt': None, - 'close_within_days': 10, - "send_to_mgr": True, - "send_to_rep": True, - 'filter_hierarchy': False, - 'nudge_heading': '', - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'stale_nudge_config'], - config_params - ) - - @cached_property - def at_risk_nudge_config(self): - config_params = { - 'deal_cnt': None, - 'send_to_rep': True, - 'senf_to_mgr': True, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - 'send_only_to': [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'at_risk_nudge_config'], - config_params - ) - - # @cached_property - # def score_history_dip_nudge(self): - # return self.config.get("score_history_dip_nudge", False) - - # @cached_property - # def manager_only_score_drop(self): - # return self.config.get("manager_only_score_drop", False) - - @cached_property - def score_hist_dip_nudge_config(self): - config_params = { - 'threshold': 10.0, - 'deal_filter_id': '', - 'deal_cnt': None, - "send_to_mgr": True, - "send_to_rep": True, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'score_hist_dip_nudge_config'], - config_params - ) - - # @cached_property - # def competitor_nudge_config(self): - # return self.config.get("competitor_nudge_config", {}) - @cached_property - def competitor_nudge_config(self): - config_params = { - 'competitor_field': 'Competitor', - 'group_by_field': 'Type', - 'deal_filter_id': 'Competitor Deals', - 'deal_cnt': None, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - 'send_only_to': [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'competitor_nudge_config'], - config_params - ) - - @cached_property - def past_closedate_nudge_config(self): - config_params = { - 'deal_filter_id': 'Filter_Closedate', - 'extra_param': '', - 'full_year_deals_view_cta': True, - 'deal_cnt': None, - "send_to_mgr": True, - "send_to_rep": True, - 'filter_hierarchy': False, - "nudge_heading": "", - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'past_closedate_nudge_config'], - config_params - ) - - @cached_property - def highamount_change_nudge_config(self): - config_params = { - 'deal_filter_id': '', - 'deal_cnt': None, - "send_to_mgr": True, - "send_to_rep": True, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'highamount_change_nudge_config'], - config_params - ) - - @cached_property - def tenant_agg_metrics_nudge_config(self): - config_params = { - 'deal_cnt': None, - 'deal_filter_id': 'Open Deals', - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'tenant_agg_metrics_nudge_config'], - config_params - ) - - @cached_property - def scenario_nudge_config(self): - config_params = { - 'threshold': 20, - 'list_of_stages': [], - 'projection_days': 7, - 'deal_filter_id': 'scenario_deals', - 'deal_cnt': None, - "send_to_mgr": True, - "send_to_rep": True, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'scenario_nudge_config'], - config_params - ) - - @cached_property - def upside_deal_stats_nudge_config(self): - config_params = { - 'industry_fld': 'Industry', - 'filter_types': ['Renewal', 'Renewals'], - 'threshold': 60, - 'win_prob_treshold': 40, - # additional - 'deal_filter_id': 'oppmap/amount/commit/upside', - 'deal_cnt': None, - "send_to_mgr": True, - "send_to_rep": True, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'upside_deal_stats_nudge_config'], - config_params - ) - - @cached_property - def pullin_deals_nudge_config(self): - config_params = { - 'deal_filter_id': '/pull-ins', - 'deal_cnt': None, - "send_to_mgr": True, - "send_to_rep": True, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'pullin_deals_nudge_config'], - config_params - ) - - @cached_property - def rep_closing_metrics_config(self): - config_params = { - 'threshold': 30.0, # Percentage threshold - 'deal_filter_id': 'Open Deals', - 'deal_cnt': None, - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'rep_closing_metrics_config'], - config_params - ) - - @cached_property - def cd_no_update_config(self): - config_params = { - 'recommended_stage': '', # default stage - 'deal_filter_id': 'cd_no_update', - 'deal_cnt': None, - 'send_to_managers_only': False, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - 'send_only_to': [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'cd_no_update_config'], - config_params - ) - - @cached_property - def non_commit_config(self): - config_params = { - 'notif_gap': 7, - 'deal_filter_id': 'non_commit_fast', - 'deal_cnt': None, - 'send_to_mgr': True, - 'send_to_rep': True, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'non_commit_config'], - config_params - ) - - @cached_property - def make_or_break_nudge_config(self): - config_params = { - 'deal_filter_id': 'Open Deals', - 'deal_cnt': None, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'make_or_break_nudge_config'], - config_params - ) - - @cached_property - def drop_in_engagement_grade_nudge_config(self): - config_params = { - 'deal_cnt': None, - 'deal_filter_id': '', - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'drop_in_engagement_grade_nudge_config'], - config_params - ) - - @cached_property - def anomaly_nudge_config(self): - config_params = { - 'deal_filter_id': '', - 'deal_cnt': None, - 'send_to_mgr': True, - 'send_to_rep': True, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - 'send_only_to': [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'anomaly_nudge_config'], - config_params - ) - - @cached_property - def outquater_pipline_nudge_config(self): - config_params = { - 'deal_filter_id': 'Open Deals', - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'outquater_pipline_nudge_config'], - config_params - ) - - @cached_property - def pipline_nudge_config(self): - config_params = { - 'deal_filter_id': 'Open Deals', - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'pipline_nudge_config'], - config_params - ) - - @cached_property - def low_pipeline_nudge_config(self): - config_params = { - 'nextq_coverage_ratio': 4, - 'deal_filter_id': 'nextq_open_deals', - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'low_pipeline_nudge_config'], - config_params - ) - - @cached_property - def late_stage_conversion_ratio_nudge_config(self): - config_params = { - 'upside_deal_filter': '/oppmap/amount/commit/upside', - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'late_stage_conversion_ratio_nudge_config'], - config_params - ) - - @cached_property - def upsell_recommendation_nudge_config(self): - config_params = { - 'filter_specific_nodes': False, - 'deal_filter_id': 'Open Deals', - 'allowed_specific_nodes': {}, - 'prohibited_roles': [], - 'prohibited_emails': [], - 'etl_line_item_name': 'OpportunityLineItem', - 'product_field': 'ProductName' - } - return get_nested( - self.config, ['nudge_config', 'upsell_recommendation_nudge_config'], - config_params - ) - - @cached_property - def past_deals_based_alert_nudge_config(self): - config_params = { - 'filter_specific_nodes': False, - 'deal_filter_id': 'Open Deals', - 'allowed_specific_nodes': {}, - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'past_deals_based_alert_nudge_config'], - config_params - ) - - # @cached_property - # def booking_accuracy_manager_only(self): - # return self.config.get("booking_accuracy_manager_only", False) - @cached_property - def deal_amount_recommendation_nudge_config(self): - config_params = { - 'deal_filter_id': '', - 'deal_cnt': None, - 'send_to_managers_only': False, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'deal_amount_recommendation_nudge_config'], - config_params - ) - - @cached_property - def booking_accuracy_nudge_config(self): - config_params = { - 'deal_filter_id': '', - 'deal_cnt': None, - 'gap': 7, - "send_to_mgr": True, - "send_to_rep": False, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'booking_accuracy_nudge_config'], - config_params - ) - - @cached_property - def discount_nudge_config(self): - config_params = { - 'deal_filter_id': '', - 'deal_cnt': None, - "send_to_mgr": True, - "send_to_rep": True, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'discount_nudge_config'], - config_params - ) - - @cached_property - def market_basket_nudge_config(self): - config_params = { - 'confidence': 50.0, - 'deal_filter_id': '', - 'deal_cnt': None, - 'send_to_managers': False, - 'filter_hierarchy': False, - 'allowed_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'market_basket_nudge_config'], - config_params - ) - - # forecast_dip_nudge - @cached_property - def forecast_dip_enabled(self): - return self.config.get("forecast_dip_enabled", False) - - @cached_property - def forecast_dip_thresh(self): - return self.config.get("forecast_dip_thresh", 10) - - @cached_property - def forecast_dip_nudge_config(self): - config_params = { - 'deal_filter_id': '', - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - 'notif_gap': 7, - "internal_heading": "", - 'send_only_to': [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'forecast_dip_nudge_config'], - config_params - ) - - @cached_property - def conversion_rate_nudge(self): - config_params = { - 'upside_deal_filter': '/oppmap/amount/commit/upside', - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - 'send_only_to': [], - 'prohibited_roles': [], - 'prohibited_emails': [], - 'commit_fld': 'commit' - } - return get_nested( - self.config, ['nudge_config', 'conversion_rate_nudge'], - config_params - ) - - @cached_property - def pace_value_dip_enabled(self): - return self.config.get("pace_value_dip_enabled", False) - - # @cached_property - # def pace_value_dip_thresh(self): - # return self.config.get("pace_value_dip_thresh", -5) - @cached_property - def pace_value_dip_nudge_config(self): - config_params = { - 'notif_gap': 7, - 'pace_value_dip_thresh': -5, - 'no_reps': None, - 'deal_filter_id': '', - 'filter_specific_nodes': False, - 'allowed_specific_nodes': {}, - "send_only_to": [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'pace_value_dip_nudge_config'], - config_params - ) - - @cached_property - def dlf_news_nudge_config(self): - config_params = { - 'up_thresh': .20, - 'down_thresh': .15, - 'no_reps': None, - 'deal_filter_id': '', - 'filter_hierarchy': False, - 'allowed_nodes': {}, - 'send_only_to': [], - 'prohibited_roles': [], - 'prohibited_emails': [] - } - return get_nested( - self.config, ['nudge_config', 'dlf_news_nudge_config'], - config_params - ) - - # --AV-996-- - - # Favorite Deals Nudge - @cached_property - def favorite_deals_nudge_config(self): - config_params = { - "since": 'yest', # bow, bom - "allowed_users": ["amit.khachane@aviso.com"], - "attributes": '', - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, ["nudge_config", - "favorite_deals_nudge_config"], config_params - ) - - @cached_property - def deal_delta_alert_config(self): - """Configurations for Deal Delta Alert Nudge (launchdarkly daily digest)""" - config_params = { - "since": 7, # bow, bom - "allowed_users": ["amit.khachane@aviso.com"], - "attributes": '', - "filter_id": 'Favorites', - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, ["nudge_config", "favorite_deals_nudge_config"], config_params - ) - - @cached_property - def meeting_nextsteps_nudge_config(self): - """Configurations for Pre-Meeting Nudge""" - config_params = { - "allowed_users": [], - "prohibited_roles": [], - "prohibited_emails": [], - "send_only_to": [], - "debug": True, - "save_notifications": True, - } - return get_nested( - self.config, ["nudge_config", "meeting_nextsteps_nudge_config"], config_params) - - @cached_property - def unclassified_meetings_nudge_config(self): - """Configurations for Unclassified Meetings Nudge""" - config_params = { - "targeted_roles": [], - "prohibited_usersids": [], - "send_only_to_usersids": [], - - } - return get_nested( - self.config, ["nudge_config", "unclassified_meetings_nudge_config"], config_params) - - # Jfrog Nudge - @cached_property - def pushed_out_deals_nudge_config(self): - config_params = { - "threshold": 1, - "deal_cnt": None, - "send_to_mgr": True, - "send_to_rep": True, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, ["nudge_config", - "pushed_out_deals_nudge"], config_params - ) - - @cached_property - def dashboard_updates_nudge_config(self): - """Configurations for Dashboard Updates Nudge""" - config_params = { - "targeted_roles": [], - "prohibited_usersids": [], - "send_only_to_usersids": [], - } - return get_nested( - self.config, ["nudge_config", "dashboard_updates_nudge_config"], config_params) - - # RingCentral Nudges - @cached_property - def best_case_nudge_config(self): - config_params = { - "threshold": 50, - "deal_cnt": None, - "send_to_mgr": True, - "send_to_rep": True, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, ["nudge_config", - "best_case_nudge_config"], config_params - ) - - @cached_property - def commit_nudge_config(self): - config_params = { - "threshold": 55, - "deal_cnt": None, - "send_to_mgr": True, - "send_to_rep": True, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, ["nudge_config", "commit_nudge_config"], config_params - ) - - @cached_property - def yearold_nudge_config(self): - config_params = { - "early_stages_thresh": [], - "send_to_mgr": True, - "send_to_rep": True, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - 'title_stages': "", - } - return get_nested( - self.config, ["nudge_config", - "yearold_nudge_config"], config_params - ) - - @cached_property - def yearold_rep_nudge_config(self): - config_params = { - "early_stages_thresh": [], - "deal_cnt": None, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - "title_stages": "", - } - return get_nested( - self.config, ["nudge_config", - "yearold_rep_nudge_config"], config_params - ) - - @cached_property - def stagnant_manager_nudge_config(self): - config_params = { - "threshold": 15, - "send_to_mgr": True, - "send_to_rep": True, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, - ["nudge_config", "stagnant_manager_nudge_config"], - config_params, - ) - - @cached_property - def stagnant_rep_nudge_config(self): - config_params = { - "threshold": 15, - "deal_cnt": None, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, ["nudge_config", - "stagnant_rep_nudge_config"], config_params - ) - - @cached_property - def highvalue_manager_nudge_config(self): - config_params = { - "stage_threshold": 50, - "threshold": 100000, - "send_to_mgr": True, - "send_to_rep": True, - "filter_hierarchy": False, - "filter_id": "highvalue_deals_100k", - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, - ["nudge_config", "highvalue_manager_nudge_config"], - config_params, - ) - - @cached_property - def highvalue_rep_nudge_config(self): - config_params = { - "stage_threshold": 50, - "threshold": 100000, - "deal_cnt": None, - "filter_hierarchy": False, - "filter_id": "highvalue_deals_100k", - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, ["nudge_config", - "highvalue_rep_nudge_config"], config_params - ) - - @cached_property - def closedate_manager_nudge_config(self): - config_params = { - "threshold": 15, - "send_to_mgr": True, - "send_to_rep": True, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, - ["nudge_config", "closedate_manager_nudge_config"], - config_params, - ) - - @cached_property - def closedate_rep_nudge_config(self): - config_params = { - "threshold": 15, - "deal_cnt": None, - "filter_hierarchy": False, - "nudge_heading": "", - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, ["nudge_config", - "closedate_rep_nudge_config"], config_params - ) - - @cached_property - def mismatch_manager_nudge_config(self): - config_params = { - "commit_stage_thresh": 55, - "bestcase_stage_thresh": 50, - "send_to_mgr": True, - "send_to_rep": True, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, - ["nudge_config", "mismatch_manager_nudge_config"], - config_params, - ) - - @cached_property - def mismatch_rep_nudge_config(self): - config_params = { - "commit_stage_thresh": 55, - "bestcase_stage_thresh": 50, - "deal_cnt": None, - "filter_hierarchy": False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, - ["nudge_config", "mismatch_rep_nudge_config"], - config_params, - ) - - @cached_property - def potential_manager_nudge_config(self): - config_params = { - 'arr_threshold': 150000, - 'tcv_threshold': 800000, - "send_to_mgr": True, - "send_to_rep": True, - 'filter_hierarchy': False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, - ['nudge_config', 'potential_manager_nudge_config'], - config_params, - ) - - @cached_property - def potential_rep_nudge_config(self): - config_params = { - 'arr_threshold': 150000, - 'tcv_threshold': 800000, - 'deal_cnt': None, - 'filter_hierarchy': False, - "allowed_nodes": {}, - "send_only_to": [], - "prohibited_roles": [], - "prohibited_emails": [], - } - return get_nested( - self.config, - ['nudge_config', 'potential_rep_nudge_config'], - config_params, - ) - - # --x--RingCentral Nudges--x-- - - @cached_property - def competitor_win_loss_config(self): - return self.config.get("competitor_win_loss_config", {}) - - @cached_property - def scenario_nudge_enabled(self): - return self.config.get("scenario_nudge_enabled", False) - - # @cached_property - # def manager_only_high_amount_change(self): - # return self.config.get("manager_only_high_amount_change", False) - - @cached_property - def high_risk_thresh(self): - return self.config.get("high_risk_thresh", 20) - - @cached_property - def cutoff_change(self): - return self.config.get("high_risk_amount_change_cutoff", 10.0) - - @cached_property - def high_risk_deals_enabled(self): - return self.config.get("high_risk_deals_enabled", False) - - @cached_property - def booking_accuracy_enable(self): - return self.config.get("booking_accuracy_enable", False) - - @cached_property - def tenant_aggregate_metrics(self): - return self.config.get("tenant_aggregate_metrics", False) - - @cached_property - def non_commit_enabled(self): - return self.config.get("non_commit_enabled", False) - - @cached_property - def non_commit_manager_only(self): - return self.config.get("non_commit_manager_only", False) - - @cached_property - def non_commit_dlf_enabled(self): - return self.config.get("non_commit_dlf_enabled", False) - - # groupby_fields_map for mobile api - @cached_property - def groupby_fields_map(self): - return self.config.get("groupby_fields_map", {}) - - # ADDITION OF CONFIGURATIONS TO RESTRICT EMAIL SENDING TO MANAGERS ONLY - @cached_property - def manager_only_past_closedate(self): - return self.config.get("manager_only_past_closedate", False) - - @cached_property - def discount_nudge_enabled(self): - return self.config.get("discount_nudge_enabled", False) - - @cached_property - def nudge_config(self): - return self.config.get("nudge_config", {}) - - @cached_property - def crm_hygiene_fld(self): - return self.config.get("crm_hygiene_fld", []) - - @cached_property - def won_filter_criteria(self): - """ - mongo db filter criteria for won deals - - Returns: - dict -- {mongo db criteria} - """ - return fetch_filter([self._won_filter_id], self, db=self.db) - - @cached_property - def _won_filter_id(self): - return get_nested(self.config, ["filters", "won_filter"]) - - def lost_filter_criteria(self): - - return fetch_filter([self._lost_filter_id], self, db=self.db) - - @cached_property - def _lost_filter_id(self): - return get_nested(self.config, ["filters", "lost_filter"]) - - def alldeal_filter_criteria(self): - - return fetch_filter([self._alldeal_filter_id], self, db=self.db) - - @cached_property - def _alldeal_filter_id(self): - return get_nested(self.config, ["filters", "all_filter"]) - - def pushout_deals_filter_criteria(self): - - return fetch_filter([self._pushout_deals_filter_id], self, db=self.db) - - @cached_property - def _pushout_deals_filter_id(self): - return get_nested(self.config, ["filters", "pushout_deals"]) - - def commit_filter_criteria(self): - - return fetch_filter([self._commit_filter_id], self, db=self.db) - - @cached_property - def _commit_filter_id(self): - return get_nested(self.config, ["filters", "Commit"], 'Commit') - - @cached_property - def created_date_field(self): - """ - name of created date field for tenant - Returns: - str -- field name - """ - return get_nested(self.config, ["field_map", "created_date"]) - - @cached_property - def manager_only_next_step_nudge(self): - return self.config.get("manager_only_next_step_nudge", False) - - @cached_property - def manager_only_commit_no_dlf(self): - return self.config.get("manager_only_commit_no_dlf", False) - - @cached_property - def persona_schemas(self): - return self.config.get("persona_schemas") - - @cached_property - def role_schemas(self): - return self.config.get("role_schemas", {}) - - @cached_property - def crr_role_schemas(self): - return self.config.get("crr_role_schemas", {}) - - # - # Complex Fields aka the please dont configure this fields - # - @cached_property - def period_aware_fields(self): - """ - fields that are period aware - - Returns: - list -- [please no] - """ - return get_nested(self.config, ["complex_fields", "period_aware_fields"], []) - - @cached_property - def hier_aware_fields(self): - """ - fields that are hierarchy aware (split fields) - - Returns: - list -- [seriously, dont] - """ - gbm_hier_aware_fields = [ - "forecast" - ] # TODO: @logan what else is always hier aware in gbm - - tenant_hier_aware_fields = get_nested( - self.config, ["complex_fields", "hier_aware_fields"], [] - ) - - dtfo_hier_aware_fields = [] - - if self.amount_field in tenant_hier_aware_fields: - dtfo_hier_aware_fields = [ - "won_amount_diff", - "lost_amount_diff", - "amt", - "stg", - ] - - return ( - get_nested(self.config, ["complex_fields", - "hier_aware_fields"], []) - + gbm_hier_aware_fields - + dtfo_hier_aware_fields - ) - - @cached_property - def revenue_fields(self): - """ - fields that are revenue based - - Returns: - list -- [i beg of you] - """ - return get_nested(self.config, ["complex_fields", "revenue_fields"], []) - - @cached_property - def active_field(self): - return get_nested( - self.config, ["complex_fields", "active_field"], "active_amount" - ) - - @cached_property - def home_page_weekly_metrics_schema(self): - conf = {"columns": ["activity", "Commit Deals", "Best Case Deals", "Most Likely Deals"], - "schema": {"activity": {"type": "string", - "label": "Activity"}, - "Commit Deals": {"type": "cell-block", - "label": "Commit"}, - "Best Case Deals": {"type": "cell-block", - "label": "Best Case"}, - "Most Likely Deals": {"type": "cell-block", - "label": "Most Likely"}}} - return get_nested(self.config, ['new_home_page', 'activity_metrics', 'schema'], conf) - - @cached_property - def deals_to_hide(self): - """ - Deals with values that need to be hidden in UI - :return: - dict -- {'as_of_Stage':'Dummy'} - """ - return self.config.get("deals_to_hide") - - @cached_property - def account_relationships_config(self): - return self.config.get("account_relationships") - - @cached_property - def account_dashboard_config(self): - return self.config.get("account_dashboard") - - @cached_property - def has_wiz_metrics(self): - return self.config.get("has_wiz_metrics") - - @cached_property - def owner_email_field(self): - return self.field_map.get("owner_email", 'OwnerEmail') - - @cached_property - def conditional_writeback(self): - return self.config.get("conditional_writeback") - - @cached_property - def vlookup_writeback_fields(self): - return self.config.get('vlookup_fields', {}) - - @cached_property - def has_wiz_metrics(self): - return self.config.get("has_wiz_metrics") - - # - # Validation - # - def validate(self, config): - """ - validate config - - Arguments: - config {dict} -- config dictionary - - Returns: - tuple - bool(valid config), [error messages] - """ - if not config: - return True, ["no config provided"] - - good_field_map, field_map_message = self._validate_field_map( - config.get("field_map", {}) - ) - good_dlf, dlf_message = self._validate_dlf(config.get("dlf", {})) - good_complex, complex_message = self._validate_complex( - config.get("complex_fields", {}) - ) - good_totals, total_message = self._validate_totals( - config.get("totals", {})) - good_schema, schema_message = self._validate_schema( - config.get("schema", {})) - good_filters, filters_message = self._validate_filters( - config.get("filters", {}) - ) - good_dashboard, dashboard_message = self._validate_dashboard( - config.get("dashboard", {}) - ) - - all_good = all( - [ - good_field_map, - good_dlf, - good_complex, - good_totals, - good_schema, - good_filters, - ] - ) - all_messages = [ - field_map_message, - dlf_message, - complex_message, - total_message, - schema_message, - filters_message, - ] - return all_good, [msg for msg in all_messages if msg] - - def _validate_field_map(self, field_map): - if not field_map: - return False, "no field map provided" - for required_field in [ - "amount", - "stage", - "forecast_category", - "close_date", - "owner_id", - "stage_trans", - ]: - if required_field not in field_map: - return False, "{} field not in map".format(required_field) - - return True, "" - - def _validate_dlf(self, dlf_config): - if not dlf_config: - return True, "" - - for field, field_config in dlf_config.items(): - if "mode" not in field_config: - return False, "no mode provided for {}, config: {}".format( - field, field_config - ) - # TODO: is amount seed field required? - - for dlf_filter in ["locked_filters", "default_filters"]: - for state, filter_ids in field_config.get(dlf_filter, {}).items(): - filt = fetch_filter(filter_ids, self, db=self.db) - if filt is None: - return ( - False, - "{} filter id: {} for {} not in filters collection".format( - dlf_filter, state, filter_ids - ), - ) - - return True, "" - - def _validate_complex(self, complex_config): - return True, "" # TODO: this - - def _validate_totals(self, total_config): - # TODO: this - return True, "" - - def _validate_schema(self, schema_config): - deal_schema = schema_config.get("deal", []) - fields = set() - for field_dtls in deal_schema: - if "field" not in field_dtls: - return False, "no field provided for {}".format(field_dtls) - if "label" not in field_dtls: - return False, "no label provided for {}".format(field_dtls) - if "fmt" not in field_dtls: - return False, "no format provided for {}".format(field_dtls) - if ( - "crm_writable" in field_dtls - and field_dtls["field"] in self.hier_aware_fields - ): - return ( - False, - "no writeback allowed on hierarchy split fields {}".format( - field_dtls - ), - ) - fields.add(field_dtls["field"]) - for field in schema_config.get("deal_fields", {}).keys(): - if "__" not in field and field not in fields: - return ( - False, - "field {} in deal fields not configured in deal schema: {}".format( - field, deal_schema - ), - ) - for optional_schema in ["card_deal_fields", "pull_in_deal_fields"]: - for field in schema_config.get(optional_schema, {}).keys(): - if "__" not in field and field not in fields: - return ( - False, - "field {} in {} not configured in deal schema: {}".format( - field, optional_schema, deal_schema - ), - ) - return True, "" - - def _validate_filters(self, filters): - filter_ids = filters.values() - filter_results = fetch_many_filters( - [[filt_id] for filt_id in filter_ids], self, db=self.db - ) - for filter_id in filter_ids: - name, filt = filter_results[tuple([filter_id])] - if not name: - return False, "filter id: {} not in filters collection".format( - filter_id - ) - return True, "" - - def _validate_dashboard(self, dashboard): - # TODO: me - return True, "" - - # - # Default Configurations - # - @cached_property - def _default_deal_changes(self): - return { - "leaderboards": { - "Amount Changes": { - "categories": ["Increase", "Decrease"], - "arrow": True, - }, - "Biggest Movers": { - "categories": ["Upgraded", "Downgraded"], - "total_name": "Forecast Impact", - "arrow": True, - }, - "Close Date Changes": {"categories": ["Pulled In", "Pushed Out"]}, - "Forecast Category Changes": { - "categories": ["Committed", "Decommitted"] - }, - "Pipeline Changes": {"categories": ["New", "Won", "Lost"]}, - }, - "categories": [ - [ - "Committed", - [{"key": "comm"}, {"key": "actv"}, - {'key': 'pushedout', 'negate': True}], - [self.amount_field, "$sum"], - ], - [ - "Decommitted", - [{"key": "decomm"}, {"key": "actv"}, - {'key': 'pushedout', 'negate': True}], - [self.amount_field, "$sum"], - ], - ["New", [{"key": "new_since_as_of"}, - {'key': 'pushedout', 'negate': True}], - [self.amount_field, "$sum"]], - [ - "Won", - [ - { - "key": ["won_amount_diff", "%(node)s"], - "op": "nested_in_range", - "val": [0, None, True], - }, - {"key": self.stage_trans_field, - "op": "in", "val": ["99"]}, - {'key': 'pushedout', 'negate': True} - ], - ["won_amount_diff", "$sum"], - ] - if self.amount_field in self.hier_aware_fields - else [ - "Won", - [ - { - "key": "won_amount_diff", - "op": "range", - "val": [0, None, True], - }, - {"key": self.stage_trans_field, - "op": "in", "val": ["99"]}, - {'key': 'pushedout', 'negate': True} - ], - ["won_amount_diff", "$sum"], - ], - [ - "Lost", - [ - { - "key": ["lost_amount_diff", "%(node)s"], - "op": "nested_in_range", - "val": [0, None, True], - }, - {"key": self.stage_trans_field, - "op": "in", "val": ["-1"]}, - {'key': 'pushedout', 'negate': True} - ], - ["lost_amount_diff", "$sum"], - ] - if self.amount_field in self.hier_aware_fields - else [ - "Lost", - [ - { - "key": "lost_amount_diff", - "op": "range", - "val": [0, None, True], - }, - {"key": self.stage_trans_field, - "op": "in", "val": ["-1"]}, - {'key': 'pushedout', 'negate': True} - ], - ["lost_amount_diff", "$sum"], - ], - [ - "Increase", - [ - { - "key": ["amt", "%(node)s"], - "op": "nested_in_range", - "val": [0, None, True], - }, - {"key": "actv"}, - {'key': 'pushedout', 'negate': True} - ], - [self.amount_field, "$sum"], - ] - if self.amount_field in self.hier_aware_fields - else [ - "Increase", - [ - {"key": "amt", "op": "range", "val": [0, None, True]}, - {"key": "actv"}, - {'key': 'pushedout', 'negate': True} - ], - [self.amount_field, "$sum"], - ], - [ - "Decrease", - [ - { - "key": ["amt", "%(node)s"], - "op": "nested_in_range", - "val": [None, 0, True, True], - }, - {"key": "actv"}, - {'key': 'pushedout', 'negate': True} - ], - [self.amount_field, "$sum"], - ] - if self.amount_field in self.hier_aware_fields - else [ - "Decrease", - [ - {"key": "amt", "op": "range", - "val": [None, 0, True, True]}, - {"key": "actv"}, - {'key': 'pushedout', 'negate': True} - ], - [self.amount_field, "$sum"], - ], - [ - "Upgraded", - [ - { - "key": ["fcst", "%(node)s"], - "op": "nested_in_range", - "val": [0, None, True, True], - }, - {"key": "actv"}, - {'key': 'pushedout', 'negate': True} - ], - [self.amount_field, "$sum"], - ], - [ - "Downgraded", - [ - { - "key": ["fcst", "%(node)s"], - "op": "nested_in_range", - "val": [None, 0, True, True], - }, - {"key": "actv"}, - {'key': 'pushedout', 'negate': True} - ], - [self.amount_field, "$sum"], - ], - [ - "Pulled In", - [{"key": "pulledin"}, {"key": "actv"}], - [self.amount_field, "$sum"], - ], - [ - "Pushed Out", - [{"key": "pushedout"}, {"key": "actv"}], - [self.amount_field, "$sum"], - ], - ], - "categories_order": [ - ["Previous Pipe", ["Pipe"]], - ["Current Pipe", ["Pipe"]], - ["New", ["New"]], - ["Won", ["Won"]], - ["Lost", ["Lost"]], - ["Date Changes", ["Pulled In", "Pushed Out"]], - ["Amount Changes", ["Increase", "Decrease"]], - ["Commit Changes", ["Committed", "Decommitted"]], - ["Aviso Forecast Changes", ["Upgraded", "Downgraded"]], - ], - "default_key": "Close Date Changes", - "leaderboards_order": [ - { - "i": "cal", - "key": "Close Date Changes", - "label": "Close Date Changes", - }, - {"i": "bars", "key": "Amount Changes", "label": "Amount Changes"}, - { - "i": "therm", - "key": "Forecast Category Changes", - "label": "Forecast Category Changes", - }, - {"i": "flag", "key": "Pipeline Changes", - "label": "Pipeline Changes"}, - {"i": "brain", "key": "Biggest Movers", "label": "Biggest Movers"}, - ], - "categories_amounts": { - "Won": "won_amount_diff", - "Lost": "lost_amount_diff", - }, - } - - @cached_property - def categories_order_labels(self): - return get_nested( - self.config, - ["dashboard", "categories_order_labels"], - self._default_categories_order_labels, - ) - - @cached_property - def _default_categories_order_labels(self): - return { - "Previous Pipe": "Previous Pipe", - "Current Pipe": "Current Pipe", - "New": "New", - "Won": "Won", - "Lost": "Lost", - "Date Changes": "Date Changes", - "Amount Changes": "Amount Changes", - "Commit Changes": "Commit Changes", - "Aviso Forecast Changes": "Aviso Forecast Changes", - } - - @cached_property - def default_milestones(self): - default_milestones = self.config.get('default_milestones', None) - if default_milestones is None: - self.config['default_milestones'] = DEFAULT_MILESTONES - return self.config.get('default_milestones', {}) - - @cached_property - def default_milestones_stages(self): - return self.config.get('default_milestones_stages', {}) - - @cached_property - def default_judg_type(self): - return get_nested(self.config, ["oppmap", "default_judg_type"], "commit") - - @cached_property - def show_deal_type(self): - return get_nested(self.config, ["oppmap", "show_deal_type"], False) - - @cached_property - def default_deal_type(self): - return get_nested(self.config, ["oppmap", "default_deal_type"], "all") - - @cached_property - def oppmap_judg_type_options(self): - return get_nested( - self.config, - ["oppmap", "oppmap_judg_type_options"], - ["commit", "dlf", "most_likely"], - ) - - @cached_property - def oppmap_judg_type_option_labels(self): - return get_nested( - self.config, - ["oppmap", "oppmap_judg_type_option_labels"], - DEFAULT_OPPMAP_JUDGE_TYPE_OPTION_LABELS, - ) - - @cached_property - def alt_amount_val_for_na(self): - return get_nested(self.config, ["amount_field_checks", "alt_amount_val"], 0.0) - - @cached_property - def amount_val_check_enabled(self): - return get_nested( - self.config, ["amount_field_checks", - "amount_val_chk_for_fld"], False - ) - - @cached_property - def versioned_hierarchy(self): - from config.hier_config import HierConfig - - hier_config = HierConfig() - - return hier_config.versioned_hierarchy - - # Return the dlf_fcst default collection schema along with the additional configured fields(if configured), - # else return None(This will not create the data for the collection) - @cached_property - def dlf_fcst_coll_schema(self): - dlf_fcst_schema = [] - dlf_fcst_schema.extend(DEFAULT_DLF_FCST_COLL_SCHEMA) - dlf_fcst_schema_additional_fields = get_nested( - self.config, ["dlf_fcst_coll_additional_fields"], None - ) - # Add the amount field dynamically - dlf_fcst_schema.append(self.amount_field) - - if dlf_fcst_schema_additional_fields: - dlf_fcst_schema.extend(dlf_fcst_schema_additional_fields) - - return dlf_fcst_schema - - def amount_field_by_pivot(self, node): - if node: - pivot = node.split("#")[0] - if self.pivot_amount_fields is not None: - return self.pivot_amount_fields.get(pivot, self.amount_field) - return self.amount_field - - @cached_property - def custom_fc_ranks_ext(self): - dict_ = self.custom_fc_ranks_default - if "Commit" in dict_: - dict_["commit"] = dict_["Commit"] - if "Most Likely" in dict_: - dict_["most_likely"] = dict_["Most Likely"] - - return dict_ - - @cached_property - def display_insights_card(self): - return get_nested( - self.config, - ["win_score_insights_card", "display_insights_card"], - DEFAULT_DISPLAY_INSIGHTS_CARD, - ) - - @cached_property - def winscore_graph_options(self): - return { - 'change_in_stagetrans': 'Change in StageTrans', - } - - @cached_property - def count_recs_to_display(self): - return get_nested( - self.config, ["win_score_insights_card", - "count_recs_to_display"], 5 - ) - - @cached_property - def oppds_fieldmap(self): - return self.config.get("oppds_fieldmap", {}) - - @cached_property - def demo_report_config_enabled(self): - return self.config.get("demo_report_config_enabled", False) - - @cached_property - def uip_user_fields(self): - return get_nested( - self.config, - ["uip_fields", "account"], - {"fields": ["Email"], "ref_field": "Email"}, - ) - - @cached_property - def uip_account_fields(self): - return get_nested( - self.config, - ["uip_fields", "account"], - { - "fields": ["LeanData__LD_EmailDomains__c"], - "reference": "LeanData__LD_EmailDomains__c", - }, - ) - - @cached_property - def custom_dtfo_fields(self): - return get_nested(self.config, ["dashboard", "custom_dtfo_fields_to_add"], []) - - @cached_property - def filter_totals_config(self): - return self.config.get('filter_totals_config', { - 'enabled': True, - 'daily': True, - 'chipotle': True - }) - - @cached_property - def deal_changes_totals_config(self): - return self.config.get('deal_changes_totals_config', {}) - - @cached_property - def run_insights_batch_size(self): - return self.config.get('run_insights_batch_size', {}) - - @cached_property - def run_gbm_crr_batch_size(self): - return self.config.get('run_gbm_crr_batch_size', {}) - - @cached_property - def future_qtrs_prefetch_count(self): - filter_totals_config = self.filter_totals_config - future_qtrs_prefetch_count = 0 - if filter_totals_config: - future_qtrs_prefetch_count = filter_totals_config.get("future_qtrs_process_count", 0) - return future_qtrs_prefetch_count - - @cached_property - def past_qtrs_prefetch_count(self): - filter_totals_config = self.filter_totals_config - past_qtrs_prefetch_count = 0 - if filter_totals_config: - past_qtrs_prefetch_count = filter_totals_config.get("past_qtrs_process_count", 0) - return past_qtrs_prefetch_count - - @cached_property - def no_update_on_writeback_fields(self): - """ - fields for which we don't need to run filter_totals or snapshot task on update from ui. - - Returns: - list - """ - - return self.config.get('no_update_on_writeback_fields', ["Manager Comments", - "__comments__", - "__comment__", - "comments", - "NextStep", - "NextSteps", - "latest_NextStepsquestions", - "latest_Problems", - "ManagerNotes", - "ProServCommentsNotes", - "latest_CoachingNotes", - "LastStep", - "LegalNotes", - "Manager_Comments", - "Mgr Notes", - "MgrNextSteps", - "TAPNextSteps", - "CustomNextStep", - "Next_Steps", - "Next Step", - "ProServCommentsNotes", - "ForecastNotes", - "ManagerForecastNotes", - "SalesDirectorNotes", - "SlipNotes", - "SEDirectorNotes", - "Notes", - "CoachingNotes", - "CSMNotes", - "latest_CoachingNotes"]) - - @cached_property - def weekly_fm(self): - return self.config.get("weekly_fm", False) - - @cached_property - def yearly_sfdc_view(self): - return self.config.get("yearly_sfdc_view", True) - - @cached_property - def insight_actions_mapping(self): - """ - Serves config to DeepLink API - insight_actions_mapping = {'scenario': {'component': 'filter_id'}, - 'risk': {'opp_map_link': '/*/oppmap/amount/commit/upside/standard/all'} - } - """ - return self.config.get('insight_actions_mapping', {}) - - @cached_property - def enhanced_waterfall_chart(self): - return self.config.get("enhanced_waterfall_chart", False) - - @cached_property - def enabled_file_parsed_load_run(self): - return self.config.get("enabled_file_parsed_load_run", False) - class FMConfig(BaseConfig): @@ -4924,12 +54,12 @@ def get_special_pivots(self): @cached_property def periods_config(self): - from config.periods_config import PeriodsConfig + from config import PeriodsConfig return PeriodsConfig(debug=self.debug, db=self.db) @cached_property def hier_config(self): - from config.hier_config import HierConfig + from config import HierConfig return HierConfig(debug=self.debug, db=self.db) @cached_property diff --git a/config/hier_config.py b/config/hier_config.py index 70cab46..98d559d 100644 --- a/config/hier_config.py +++ b/config/hier_config.py @@ -8,7 +8,7 @@ from aviso.settings import sec_context from fake_data.fake_data import COMPANIES -from infra import HIER_COLL +from infra.constants import HIER_COLL from infra.mongo_utils import create_collection_checksum from infra.read import get_period_and_close_periods, get_period_infos, get_as_of_dates, get_period_begin_end, \ fetch_node_to_parent_mapping_and_labels @@ -1193,15 +1193,6 @@ def versioned_hierarchy(self): """ return self.config.get('versioned', False) - @cached_property - def sort_system_nodes(self): - """ - push system nodes to the end of the hierarchy - Returns: - bool - """ - return self.config.get('sort_system_nodes', False) - @cached_property def logger_draw_tree(self): """ @@ -1213,9 +1204,6 @@ def logger_draw_tree(self): """ return self.config.get('logger_draw_tree',True) - # - # Hierarchy Configuration - # @cached_property def hierarchy_builder(self): """ @@ -1295,38 +1283,6 @@ def deal_partial_hier_drilldowns(self): return {drilldown: dd_dtls['num_levels'] for drilldown, dd_dtls in self.config.get('drilldowns', {}).items() if dd_dtls['drilldown_builder'] == 'deal_partial_hier'} - @cached_property - def hier_display_order(self): - """ - display order for hierarchies in ui - - Returns: - list -- [root node ids] - """ - return self.config.get('hierarchy_display_order', []) - - @cached_property - def pivot_and_root_node_mapping(self): - """ - pivot and root node mapping - - Returns: - dict -- {pivot1: root_node_of_pivot1, - pivot2: root_node_of_pivot2} - """ - return self.config.get('pivot_map', {}) - - @cached_property - def hierarchy_editable(self): - if self.hierarchy_builder == 'collab_fcst': - return False - if all(dd_dtls['drilldown_builder'] == 'deal_only' for dd_dtls in self.config.get('drilldowns', {}).values()): - return False - return True - - # - # Validation - # def validate(self, config): dummy_tenant = config.get('dummy_tenant', False) hier_builder = config.get('hierarchy_builder') diff --git a/config/periods_config.py b/config/periods_config.py index 43127d4..d8c90c9 100644 --- a/config/periods_config.py +++ b/config/periods_config.py @@ -58,13 +58,6 @@ def fm_period(self): """ return self.config.get('fm_period', 'Q') - @cached_property - def period_hidden(self): - """ - Fetches the quarter before which previous year and quarter needs to be hidden - """ - return self.config.get('period_hidden', None) - @cached_property def yearly_fm(self): """ diff --git a/deal_service/tasks.py b/deal_service/tasks.py index a5f4c8a..8c44fb9 100644 --- a/deal_service/tasks.py +++ b/deal_service/tasks.py @@ -1,14 +1,10 @@ import logging -from itertools import groupby -from operator import itemgetter - -from config.fm_config import DealConfig -from config.periods_config import PeriodsConfig -from infra.read import (fetch_ancestors, get_available_quarters_and_months, - get_period_boundaries, get_period_boundaries_monthly, - get_period_boundaries_weekly) + +from config import DealConfig +from config import PeriodsConfig +from infra.read import fetch_ancestors, get_available_quarters_and_months from utils.common import cached_property -from utils.misc_utils import get_nested, merge_nested_dicts, try_values +from utils.misc_utils import try_values from ..tasks import BaseTask @@ -52,101 +48,6 @@ def adorn_hierarchy(self, deal, as_of): return list(set(hierarchy_list)), drilldown_list - def adorn_close_period(self, close_date): - """ - get period deal closes in - - Arguments: - close_date {float} -- close date in xldate format - - Returns: - str -- period mnemonic - """ - try: - return next(mnem for (beg, end, mnem) in self.period_boundaries if beg <= close_date <= end) - except: - return 'N/A' - - def adorn_weekly_period(self, close_date): - """ - get period deal closes in - - Arguments: - close_date {float} -- close date in xldate format - - Returns: - str -- period mnemonic - """ - try: - return next(mnem for (beg, end, mnem) in self.period_boundaries_weekly if beg <= close_date <= end) - except: - return 'N/A' - - def adorn_monthly_period(self, close_date): - """ - get period deal closes in - - Arguments: - close_date {float} -- close date in xldate format - - Returns: - str -- period mnemonic - """ - try: - return next(mnem for (beg, end, mnem) in self.period_boundaries_monthly if beg <= close_date <= end) - except: - return 'N/A' - - def fetch_month_enumeration(self, close_date): - monthly_period = self.adorn_monthly_period(close_date) - if 'M' in monthly_period: - num_str = monthly_period.split('M')[1] # Get the part after 'M' - number = int(num_str) - result = number % 3 - if result == 0: - result = 3 - return result - else: - return 'N/A' - - def split_out_deals(self, deal): - """ - resplit deals :( - elasticsearch cant handle split deals - so were gonna blow out all the splits into individual records - truly the dumbest fucking thing ive ever done ... today - - Arguments: - deal {dict} -- deal object - - Returns: - list -- [deal dicts] - """ - if self.config.hier_aware_fields: - deals = [] - for node_group in self.group_nodes(deal, deal.pop('__segs', [])): - deals.append( - merge_nested_dicts({k: v if k not in self.config.hier_aware_fields else v.get(node_group[0]) - for k, v in deal.iteritems()}, - {'__segs': node_group})) - return deals - return [deal] - - def group_nodes(self, deal, nodes): - """ - group nodes with matching split values - - Arguments: - deal {dict} -- deal object - nodes {list} -- list of nodes - - Returns: - list -- list of lists of grouped nodes - """ - nvals = [{'vals': {field: get_nested(deal, [field, node]) for field in self.config.hier_aware_fields}, - 'node': node} for node in nodes] - return [[x['node'] for x in v] for _, v in groupby(sorted(nvals), key=itemgetter('vals'))] - def get_hierarchy_ancestors(self, as_of, owner_id): try: return self._hier_ancestors[as_of].get(owner_id, [owner_id]) @@ -177,15 +78,6 @@ def period_config(self): def quarters_and_months(self): return get_available_quarters_and_months(self.period_config) - def get_quarter_of_month(self, month): - if "Q" in month: - return month - else: - for quarter, months in self.quarters_and_months.iteritems(): - if month in months: - return quarter - return self.period - @cached_property def _hier_ancestors(self): return {} @@ -194,18 +86,6 @@ def _hier_ancestors(self): def _dd_ancestors(self): return {} - @cached_property - def period_boundaries(self): - return get_period_boundaries(self.period) - - @cached_property - def period_boundaries_weekly(self): - return get_period_boundaries_weekly(self.period) - - @cached_property - def period_boundaries_monthly(self): - return get_period_boundaries_monthly(self.period) - @cached_property def config(self): return DealConfig() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..36f0176 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3.8' + +services: + mongo: + image: mongo + container_name: mongo-test + ports: + - "27017:27017" + environment: + # make sure to match this according to your .env file + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: pass123 + volumes: + - mongo_data:/data/db + +volumes: + mongo_data: diff --git a/domainmodel/__init__.py b/domainmodel/__init__.py index 42e7d99..e69de29 100644 --- a/domainmodel/__init__.py +++ b/domainmodel/__init__.py @@ -1,811 +0,0 @@ -import logging -import re - -from bson import BSON -from aviso.settings import gnana_db, gnana_db2, sec_context -from pymongo import ASCENDING - -from utils import GnanaError, crypto_utils, date_utils, diff_rec - -logger = logging.getLogger('gnana.%s' % __name__) - - -class BaseBulkOperations: - - def __init__(self, collection_name): - raise NotImplementedError() - - def find(self, criteria): - raise NotImplementedError() - - def upsert(self): - raise NotImplementedError() - - def replace_one(self, doc): - raise NotImplementedError() - - def update_one(self, doc): - raise NotImplementedError() - - def insert(self, doc): - raise NotImplementedError() - - def execute(self): - raise NotImplementedError() - - -class Model: - """Base class for all domainmodel objects and provides the basic ORM - capabilities and conventions. - - In addition, this class also provides a - few class level methods that can be used to retrieve objects by keys - and names. - - This base class adds the following into the collection: - - _id - Database ID of the object - - _kind - Type of the object stored in the collection. Most of the time, a - collection holds homogeneous kind of objects, however this is not - guaranteed. - - _version - Each object will store a schema version used to store the object. - In the future it is expected that, an upgrade method is called to - progressively upgrade the data in the collection. - - .. NOTE:: As of now we are storing the version, but no upgrade logic - is implemented. - - object - All the attributes of the object are encoded by calling the encode - method and saved with object key - - During encoding each domainobject should ensure there are no periods - in any dictionaries. This is a limitation in document databases. - - Following Class level variables to be defined by each domain object: - - kind - Kind of the object - - tenant_aware - If the tenant aware is set to true, when saving the objects, - collection name is prefixed with the tenant name from security context - object - - version - Current object schema version - - index_list - A dictionary of indexes to be ensured when saving. When specifying - the index, additional options can be provided as a subdictionary. - - Examples:: - - # Just an index - 'probeid': {} - - # Unique index on object.username field in DB - 'username': {'unique': True} - - # Creating index in descending order, default value is ASCENDING - "created_time": {'direction': DESCENDING} - - # Index to expire documents after certain age. In this example - # document expires after 900 sec + value in object.timestamp - # field - 'timestamp': { - 'expireAfterSeconds': 900 - } - - query_list - A dictionary in the same format as indexes, that must be searchable - but not actually created in the DB as indexes. - - collection_name - Name of the collection in which to store the values. - - .. WARNING: Never use ModelClass.collection_name directly, instead - use ModelClass.getCollectionName() method. Using the method ensures - that tenant prefixing is done properly. - """ - kind = None - tenant_aware = True - collection_name = None - index_list = {} - version = None - index_checked = {} - encrypted = True - query_list = {} - check_sum = None # Used in UIP prepare to compare stage_uip & main_uip records - typed_fields = {} - compressed = False - read_from_primary = False - - def __init__(self, attrs): - if attrs is not None and (u'_id' in attrs or '_id' in attrs): - self.id = attrs[u'_id'] - if u'_kind' not in attrs: - raise ModelError("Kind of object is not defined by subclass") - if attrs[u'_kind'] != self.kind: - raise ModelError( - "Loading incorrect type of object, DB: %s, Required: %s" % - (attrs[u'_kind'], self.kind)) - # Ideally this should not be here. - if self.encrypted and self.tenant_aware and u'_encdata' in attrs and attrs.get('_encdata'): - decdata = BSON( - sec_context.decrypt(attrs[u'_encdata'])).decode() - if decdata: - attrs[u'object'] = decdata - if attrs['_version'] != self.version: - self.upgrade_attrs(attrs) - self.last_modified_time = attrs.get('last_modified_time', None) - self.check_sum = attrs.get('check_sum', None) - self.decode(attrs[u'object']) - else: - self.id = None - self.last_modified_time = None - self.check_sum = None - - def save(self, is_partial=False, bulk_list=None, id_field=None, field_list=None, update_fields=[], conditional=False): - """Save the object into the database. - If the object is initialized with attributes and an _id is found, - during the object creation, it is overwritten. If no _id is present - it is created. - id_field: Is used to update records based on the given field - conditional: If true ensures no updates have been made to this record between the read and the write. - returns the if of the record if the conditional write was successful, otherwise false - """ - if getattr(self, 'read_only', None): - raise GnanaError("Read only object cannot be saved!") - - collection_name = self.getCollectionName() - - # currently database views are used by tenant model - # To save data to tenant directly removing _view from the collection name - if '_view' in collection_name: - collection_name = collection_name.replace('_view', '') - - postgres = getattr(self, "postgres", None) - - self.create_index() - - localattrs = {} - prev_last_modified_time = self.last_modified_time - try: - self.last_modified_time = date_utils.epoch().as_epoch() - except: - self.last_modified_time = None - self.encode(localattrs) - localattrs.pop('last_modified_time', None) - check_sum_value = localattrs.pop('check_sum', None) - object_doc = {} - # field_list need to be pass in case of is_partial=True for task and result_cache save. - # for e.g field_list = ['object.extid', 'object.type'] etc. - if is_partial and field_list: - for i in field_list: - object_doc[i] = localattrs[i.split('.')[1]] - else: - object_doc = {'object': localattrs} - object_doc.update({'_kind': self.kind, - '_version': self.version, - 'last_modified_time': self.last_modified_time}) - if check_sum_value: - object_doc['check_sum'] = check_sum_value - if not field_list: - object_doc = self.update_object_doc(object_doc) - if bulk_list is not None: - if conditional: - raise Exception("Conditionals not supported by bulk saving") - - if not self.id and not id_field: - bulk_list.insert(object_doc) - else: - if self.id: - object_doc['_id'] = self.id - id_field = '_id' if not id_field else id_field - if isinstance(id_field, str): - id_field_list = [id_field] - else: - id_field_list = id_field - criteria = self.bulk_criteria(id_field_list, object_doc) - if is_partial: - bulk_list.find(criteria).upsert().update_one(object_doc) - else: - bulk_list.find(criteria).upsert().replace_one(object_doc) - return None - else: - if getattr(self, "postgres", None): - if conditional: - raise Exception("Conditionals not implemented for posgres") - self.id = gnana_db2.saveDocument( - collection_name, object_doc, self.id, is_partial=is_partial, update_fields=update_fields) - #logger.info('document updated in postgres with id %s', self.id) - else: - self.id = gnana_db.saveDocument( - collection_name, object_doc, self.id, is_partial=is_partial, update_fields=update_fields, - conditional=conditional, prev_last_modified_time=prev_last_modified_time) - #logger.info('document saved with id %s', self.id) - - return self.id - - def bulk_criteria(self, id_field, object_doc): - field_list = [] - for field in id_field: - if 'object.' in field: - f = field[len('object.'):] - field_list.append({field: object_doc['object'][f]}) - else: - field_list.append({field: object_doc[field]}) - criteria = {'$and': [x for x in field_list]} - return criteria - - def return_as_object_doc(self): - localattrs = {} - self.encode(localattrs) - localattrs.pop('last_modified_time', None) - check_sum_value = localattrs.pop('check_sum', None) - object_doc = {'_id': str(self.id), - 'object': localattrs, - '_kind': self.kind, - '_version': self.version, - 'last_modified_time': self.last_modified_time} - if check_sum_value: - object_doc['check_sum'] = check_sum_value - object_doc = self.update_object_doc(object_doc) - return object_doc - - def update_object_doc(self, object_doc): - localattrs = object_doc['object'] - localattrs = self.check_postgres_typed_fields(localattrs) - object_doc['object'] = localattrs - if self.tenant_aware and self.encrypted and sec_context.details.is_encrypted: - encdata = sec_context.encrypt(BSON.encode(localattrs), self) - if encdata: - self.encdata = encdata - object_doc.update({ - "_encdata": encdata, - "object": crypto_utils.extract_index(self.index_list, localattrs) - }) - object_doc["object"].update( - crypto_utils.extract_index(self.query_list, localattrs) - ) - return object_doc - - @classmethod - def bulk_ops(cls): - class PostGress_Bulk_Operations(BaseBulkOperations): - - def __init__(self, collection_name, encrypted, id_field): - self.collection_name = collection_name - self.encrypted = encrypted - self.bulk_list = [] - self.bulk_update = [] - self.find_objs = {} - self.id_field = id_field if id_field else 'extid' - - def execute(self): - # pass - if self.bulk_list: - logger.info("Using postgres bulk insert") - gnana_db2.postgres_bulk_execute( - self.bulk_list, self.collection_name) - if self.bulk_update: - logger.info("Using postgres bulk update") - gnana_db2.postgres_bulk_update(self.bulk_update) - - def insert(self, doc): - self.bulk_list.append(doc) - - def upsert(self): - return self - - def _replace(self, doc): - self.bulk_update.append( - gnana_db2.postgres_bulk_update_string(self.collection_name, doc)) - - def find(self, criteria): - if criteria: - obj = gnana_db2.findDocument(self.collection_name, criteria) - if obj: - self.find_objs[sec_context.encrypt(obj['object'][self.id_field], self)] = obj - return self - - def replace_one(self, doc): - if doc['object'][self.id_field] in self.find_objs: - saved_rec = self.find_objs.pop(doc['object'][self.id_field]) - doc['_id'] = saved_rec['_id'] - self._replace(doc) - else: - self.insert(doc) - - def update_one(self, doc): - saved_rec = self.find_objs.pop(doc['object'][self.id_field]) - saved_rec.update(doc) - self._replace(saved_rec) - - class MongoBulkOperations(BaseBulkOperations): - - def __init__(self, collection_name): - self.bulk_list = gnana_db.db[cls.getCollectionName()].initialize_unordered_bulk_op() - - def execute(self): - try: - self.bulk_list.execute() - except Exception as e: - no_op = re.match('No operations to execute', e.message) - if not no_op: - logger.exception('bulk list failed with error %s', e) - raise e - else: - logger.info('bulk list failed with No operations to execute error') - - def insert(self, doc): - self.bulk_list.insert(doc) - - def upsert(self): - return self.bulk_list.upsert() - - def _replace(self, doc): - raise Exception('replace is not supported for Mongo, this is specific to postgres') - - def find(self, criteria): - return self.bulk_list.find(criteria) - - def replace_one(self, doc): - self.bulk_list.replace_one(doc) - - def update_one(self, doc): - self.bulk_list.update_one(doc) - - class CompressedMongoBulk(BaseBulkOperations): - - def __init__(self, collection_name): - self.collection_name = collection_name - self.bulk_list = list() - self.bulk_update = dict() - - def find(self, criteria): - return self - - def upsert(self): - return self - - def replace_one(self, doc): - self.bulk_update[doc['object']['extid']] = doc - - def update_one(self, doc): - return self.replace_one(doc) - - def insert(self, doc): - self.bulk_list.append(doc) - - def execute(self): - if self.bulk_list: - cls.compressedSave(self.bulk_list) - if self.bulk_update: - cls.UpdateAndSave(self.bulk_update) - del self.bulk_list - del self.bulk_update - self.bulk_list = list() - self.bulk_update = dict() - - if getattr(cls, "postgres", False): - return PostGress_Bulk_Operations(cls.getCollectionName(), cls.encrypted, getattr(cls, 'id_field', None)) - elif cls.compressed: - return CompressedMongoBulk(cls.getCollectionName()) - else: - return MongoBulkOperations(cls.getCollectionName()) - - @classmethod - def bulk_insert(cls, rec_list): - cls.create_index() - tenant_details = sec_context.details - is_tenant_encrypted = tenant_details.is_encrypted - docs_to_insert = [] - for rec in rec_list: - localattrs = {} - rec.last_modified_time = date_utils.epoch().as_epoch() - rec.encode(localattrs) - localattrs.pop('last_modified_time', None) - check_sum_value = localattrs.pop('check_sum', None) - query_fields = [] - postgres = False - if getattr(cls, "postgres", None): - query_fields = cls.all_known_fields - postgres = True - if cls.typed_fields: - localattrs = cls.check_postgres_typed_fields(localattrs) - object_doc = {'object': localattrs, - '_kind': cls.kind, - '_version': cls.version, - 'last_modified_time': rec.last_modified_time} - if check_sum_value: - object_doc['check_sum'] = check_sum_value - if is_tenant_encrypted and cls.encrypted: - docs_to_insert.append(crypto_utils.encrypt_record(cls.index_list, object_doc, query_fields, postgres, - cls=cls)) - else: - docs_to_insert.append(object_doc) - if docs_to_insert: - if getattr(cls, "postgres", None): - gnana_db2.insert(cls.getCollectionName(), docs_to_insert) - else: - gnana_db.insert(cls.getCollectionName(), docs_to_insert) - - @classmethod - def copy_collection(cls, newcls, criteria={}, batch_size=5000): - if getattr(cls, "postgres", None): - cur = gnana_db2.findDocuments( - cls.getCollectionName(), criteria, auto_decrypt=True) - else: - cur = gnana_db.findDocuments( - cls.getCollectionName(), criteria, auto_decrypt=True) - count = cur.count() - start_size = 0 - while start_size < count: - to_process = min(count - start_size, batch_size) - newcls.bulk_insert(cur[start_size:start_size + to_process]) - start_size += to_process - - @classmethod - def truncate_or_drop(cls, criteria=None): - if criteria is None: - if getattr(cls, "postgres", None): - return gnana_db2.dropCollection(cls.getCollectionName()) - else: - return gnana_db.dropCollection(cls.getCollectionName()) - else: - if getattr(cls, "postgres", None): - return gnana_db2.truncateCollection(cls.getCollectionName(), criteria, cls.typed_fields) - else: - return gnana_db.truncateCollection(cls.getCollectionName(), criteria)['n'] - - @classmethod - def renameCollection(cls, new_col_name, overwrite=False): - if getattr(cls, "postgres", None): - gnana_db2.renameCollection( - cls.getCollectionName(), new_col_name, overwrite) - else: - gnana_db.renameCollection( - cls.getCollectionName(), new_col_name, overwrite) - - @classmethod - def create_index(cls): - postgres = getattr(cls, "postgres", None) - collection_name = cls.getCollectionName() - if '_view' in collection_name: - collection_name = collection_name.replace('_view', '') - if not cls.index_checked.get(collection_name, False): - for x in cls.index_list: - index_creation_method = None - if x in cls.typed_fields: - typed_field_type = cls.typed_fields[x]['type'] - is_array = re.search("(ARRA\w+)", typed_field_type) or re.search("[\[]", typed_field_type) - if is_array: - index_creation_method = 'gin' - options = cls.index_list[x] - if 'index_spec' in cls.index_list[x]: - options = options.copy() - index_spec = options.pop('index_spec') - else: - index_field_list = x.split('~') - index_spec = [] - for f in index_field_list: - direction = options.pop('direction', ASCENDING) - index_fld = ("object.%s" % f, direction) - index_spec.append(index_fld) - if postgres: - gnana_db2.ensureIndex(collection_name, index_spec, options, method=index_creation_method) - else: - gnana_db.ensureIndex(collection_name, index_spec, options, method=index_creation_method) - - # Special indexes that need to be added for all models are to be - # defined here - if postgres: - gnana_db2.ensureIndex( - collection_name, "last_modified_time", {}) - else: - gnana_db.ensureIndex(collection_name, "last_modified_time", {}) - cls.index_checked[collection_name] = True - # Updating index list of class with last_modified_time to save it to saved_index_info - cls.index_list.update({"last_modified_time": {}}) - all_query_fields_new = { - 'index_list': loaded_index_list(cls.index_list), - 'query_list': loaded_index_list(cls.query_list) - } - all_query_fields_old = None - try: - all_query_fields_old = gnana_db.findDocument( - sec_context.name + '.saved_index_info', - {'collection': collection_name} - ) - except: - pass - if (all_query_fields_old is None or - diff_rec(all_query_fields_new, all_query_fields_old['index_info'])): - if not all_query_fields_old: - all_query_fields_old = {'collection': collection_name} - all_query_fields_old['index_info'] = all_query_fields_new - try: - if sec_context.name != 'administrative.domain': - gnana_db.saveDocument( - sec_context.name + '.saved_index_info', - all_query_fields_old - ) - except Exception as e: - logger.exception(f"Got Exception while saving index info: {e}") - - @classmethod - def getCollectionName(cls): - """ - Return the collection name to be used for the class. - - .. Warning:: Do not cache this value, as it will change based on - the context. - """ - if cls.tenant_aware: - return sec_context.name + "." + cls.collection_name - else: - return cls.collection_name - - @classmethod - def getBySpecifiedCriteria(cls, criteria, check_unique=False): - """Find an object by given criteria. The caller can specify any - MongoDB-blessed criteria to find a specific document. - - check_unique - When multiple objects match the field value, passing check_unique as - True will explicitly checking nothing else matched. Otherwise it - returns the first matching object - - .. NOTE:: - - Generally you should use the unique_indexes and not depend - on this mechanism. - """ - if getattr(cls, "postgres", None): - attrs = gnana_db2.findDocument(cls.getCollectionName(), - criteria, - check_unique) - else: - attrs = gnana_db.findDocument(cls.getCollectionName(), - criteria, - check_unique, - read_from_primary=cls.read_from_primary, - tenant_aware=cls.tenant_aware) - if attrs: - return cls(attrs) - else: - return None - - @classmethod - def getByFieldValue(cls, field, value, check_unique=False): - """Find an object by given field value. ``object.`` is automatically - prepended to the field name provided. An object of the class on which - this method is called will be created, hence this method will work for - all domainmodel objects. - - forgive - When multiple objects match the field value, passing forgive as - True will return the first matching object. If it is False, an - exception is raised. - - TODO: Refactor this - use getBySpecifiedCriteria once the tests pass - """ - if getattr(cls, "postgres", None): - attrs = gnana_db2.findDocument(cls.getCollectionName(), - {'object.' + field: - sec_context.encrypt( - value) if cls.tenant_aware and cls.encrypted else value}, - check_unique) - else: - attrs = cls.get_db().findDocument(cls.getCollectionName(), - {'object.' + field: - sec_context.encrypt( - value) if cls.tenant_aware and cls.encrypted else value}, - check_unique, read_from_primary=cls.read_from_primary, - tenant_aware=cls.tenant_aware - ) - - return cls(attrs) if attrs else None - - @classmethod - def getAllByFieldValue(cls, field, value): - """Find all objects by given field value. ``object.`` is automatically - prepended to the field name provided. An object of the class on which - this method is called will be created, hence this method will work for - all domainmodel objects. - - - TODO: Refactor this - use getBySpecifiedCriteria once the tests pass - """ - try: - if getattr(cls, "postgres", None): - my_iter = gnana_db2.findAllDocuments(cls.getCollectionName(), - {'object.' + field: sec_context.encrypt(value) - if cls.tenant_aware and cls.encrypted else value}) - else: - my_iter = gnana_db.findAllDocuments(cls.getCollectionName(), - {'object.' + field: sec_context.encrypt(value) - if cls.tenant_aware and cls.encrypted else value}, - read_from_primary=cls.read_from_primary, - tenant_aware=cls.tenant_aware) - for attrs in my_iter: - yield cls(attrs) - except: - logger.exception("Exception raised while finding values %s-%s-%s" % - (cls.getCollectionName(), field, value)) - - @classmethod - def getAll(cls, criteria=None, fieldList=[], return_dict=False): - """Find all objects. An object of the class on which - this method is called will be created, hence this method will work for - all domainmodel objects. - """ - if criteria is None: - criteria = {} - # currently we are supporting the fieldList only for postgres as mongo has encrypted data, there are other steps to be - # done. to support field projection for mongo - try: - if not getattr(cls, 'postgres', None): - fieldList = [] - my_iter = cls.get_db().findAllDocuments( - cls.getCollectionName(), criteria, cls.typed_fields, fieldList, - read_from_primary=cls.read_from_primary, - tenant_aware=cls.tenant_aware) - if return_dict: - for attrs in my_iter: - yield attrs['object'] - else: - for attrs in my_iter: - cls_obj = cls(attrs) - if fieldList: - # Making the class object to be read only if only some fields are requested. - setattr(cls_obj, 'read_only', True) - yield cls_obj - except Exception as e: - logger.exception(e) - - @classmethod - def getByKey(cls, key): - """Find an object by key - """ - try: - if getattr(cls, "postgres", None): - attrs = gnana_db2.retrieve(cls.getCollectionName(), key) - else: - attrs = gnana_db.retrieve(cls.getCollectionName(), key, read_from_primary=cls.read_from_primary, - tenant_aware=cls.tenant_aware) - except StopIteration: - attrs = None - return attrs and cls(attrs) or None - - @classmethod - def getByName(cls, name): - """Find an object by name. This is shorthand to getByFieldValue - """ - return cls.getByFieldValue('name', name) - - # Nothing to encode, id is used automatically - def encode(self, attrs): - """Called before saving to get a dictionary representation of the - object suitable for saving into a document database. Make sure to - call the super class method in the implementations. No need to return - anything, just updating the attrs is enough. - """ - attrs['last_modified_time'] = self.last_modified_time - attrs['check_sum'] = self.check_sum - return attrs - - # Nothing to decode - def decode(self, attrs): - """Called to reconstruct the object from the dictionary. Initialize - any required variables and make sure to call the super class - """ - self.last_modified_time = self.last_modified_time or attrs.get(u'last_modified_time', None) - self.check_sum = self.check_sum or attrs.get(u'check_sum', None) - - @classmethod - def remove(cls, objid): - """Remove the object from the database - """ - if objid: - if getattr(cls, "postgres", None): - gnana_db2.removeDocument(cls.getCollectionName(), objid) - else: - gnana_db.removeDocument(cls.getCollectionName(), objid) - else: - raise ModelError("Can't remove unbound db object") - - def _remove(self): - self.remove(self.id) - - @classmethod - def list_all_collections(cls, prefix): - if getattr(cls, "postgres", None): - return gnana_db2.collection_names(prefix) - else: - return gnana_db.collection_names(prefix) - - @classmethod - def check_postgres_typed_fields(cls, localattrs): - if not cls.typed_fields: - return localattrs - for key in localattrs: - if key in cls.typed_fields.keys(): - typed_field_type = cls.typed_fields[key]['type'] - is_array = re.search("(ARRA\w+)", typed_field_type) or re.search("[\[]", typed_field_type) - if is_array and not isinstance(localattrs[key], list): - if localattrs[key]: - localattrs[key] = [localattrs[key]] - return localattrs - - @classmethod - def get_db(cls): - return gnana_db2 if getattr(cls, "postgres", None) else gnana_db - - @classmethod - def getDistinctValues(cls, key, criteria={}): - encrypted = False - if cls.tenant_aware and cls.encrypted and sec_context.details.is_encrypted: - encrypted = True - if getattr(cls, "postgres", None): - return gnana_db2.getDistinctValues(cls.getCollectionName(), key, criteria=criteria, encrypted=encrypted) - else: - return gnana_db.getDistinctValues(cls.getCollectionName(), key, criteria=criteria, - encrypted=encrypted, - read_from_primary=cls.read_from_primary, - tenant_aware=cls.tenant_aware) - - @classmethod - def queryExecutor(cls, statement, return_dict=False): - if getattr(cls, "postgres", None): - try: - my_iter = gnana_db2.postgres_query_executor(cls.getCollectionName(), statement) - if return_dict: - for attrs in my_iter: - yield attrs['object'] - else: - for attrs in my_iter: - yield cls(attrs) - except Exception as e: - logger.exception(e) - raise e - - @classmethod - def get_count(cls, criteria=None): - if getattr(cls, "postgres", None): - return cls.get_db().find_count(cls.getCollectionName(), criteria) - else: - if criteria is None: - criteria = {} - return cls.get_db().find_count(cls.getCollectionName(), criteria, read_from_primary=cls.read_from_primary, - tenant_aware=cls.tenant_aware) - - -class ModelError(Exception): - - def __init__(self, error): - logger.error(error) - self.error = error - -def loaded_index_list(index_list_class): - index_list_db = {} - for k, v in index_list_class.iteritems(): - name = k.replace('.', '~') - index_list_db[name] = v.copy() - if 'index_spec' in v: - index_list_db[name]['key'] = v.get('index_spec') - else: - index_field_list = k.split('~') - index_spec = [] - for f in index_field_list: - index_fld = ("object.%s" % f, ASCENDING) - index_spec.append(index_fld) - index_list_db[name]['key'] = index_spec - return index_list_db diff --git a/domainmodel/app.py b/domainmodel/app.py index 47f57cc..7b90455 100644 --- a/domainmodel/app.py +++ b/domainmodel/app.py @@ -1,12 +1,7 @@ -import base64 import datetime import json import logging -import re -import time -from datetime import UTC - -import pytz +from datetime import timezone as UTC from aviso.framework import tracer from aviso.settings import (POOL_PREFIX, WORKER_POOL, adhoc_task_validity, archive_analyticengine_validity, gnana_db2, @@ -18,15 +13,9 @@ taskactive_analyticengine_validity, taskactive_validity, archive_task_validity) from celery import current_task -from Crypto.Hash import SHA, SHA512 -from Crypto.PublicKey import RSA -from Crypto.Signature import PKCS1_PSS -from domainmodel import Model +from domainmodel.model import Model from utils import date_utils -from utils.common import ip_match -from utils.string_utils import random_string - logger = logging.getLogger('gnana.%s' % __name__) class AccountEmails(Model): @@ -89,11 +78,6 @@ def create_postgres_table(self): return gnana_db2.postgres_table_creator(schema.format(table_name=table_name)) -class UserError(Exception): - - def __init__(self, error): - self.error = error - class TaskError(Exception): @@ -101,487 +85,6 @@ def __init__(self, error): self.error = error -class User(Model): - """ Stores and retrieves user information in a tenant - specific collection""" - collection_name = 'user' - version = 3 - postgres = True - kind = "domainmodel.app.User" - tenant_aware = True - index_list = { - 'username': {'unique': True} - } - # Prevents this object from being encrypted. - encrypted = False - USER_ROLES = ( - (None, 'Unknown'), - ('ceo', 'CEO'), - ('cfo', 'CFO'), - ('cro', 'CRO'), - ('customer_success', 'Customer Success'), - ('sales_regional_vp', 'Sales Regional VP'), - ('sales_vp_director', 'Sales VP / Director'), - ('sales_manager', 'Sales Manager'), - ('sales_ops_vp_director', 'Sales Ops VP / Director'), - ('sales_ops_manager', 'Sales Ops Manager'), - ('sales_ops', 'Sales Ops'), - ('sales_rep', 'Sales Rep'), - ('finance_vp', 'Finance VP'), - ('finance_manager', 'Finance Manager'), - ('finance_analyst', 'Finance Analyst'), - ('it', 'IT') - ) - - def __init__(self, attrs=None): - - # Original attributes - self.name = None - self.username = None - self.email = None - self.secondary_emails = [] - self.app_modules = [] - self.password_hash = None - self.last_login = None - - self.valid_ips = {} - self.valid_devices = {} - - # the above field would save the device - # user tried to login from the validation code for the - # device and the time at which he tried to login - self.pwd_validation_code = None - # datetime.datetime.now() - self.pwd_validation_expiry = date_utils.now() - self.roles = {} - self.account_locked = False - self.login_attempts = 0 - self.failed_login_time = 0 - - self.edit_dims = None - - # Added to allow users to have long lived sessions - tenant_details = sec_context.details - self.user_timeout = tenant_details.get_config('security', 'session_timeout', 120) - - # Added to distinguish users created for customers - # and for gnana to introspect results - - # True if the user is a customer false if the user is a gnana employee. - self.is_customer = True - self.user_role = None - - # Additional attrbibutes for new functionality - self.is_disabled = False - self.preset_validation_code = None - self.show_experimental = False - self.password_salt = None - - self.ssh_keys = {} - # Added to validate new user through url - self.uuid = None - self.uuid_validation_expiry = None - self.activation_status = None - self.mail_date = None - self.is_second_login = False - # Used only for SSO users - self.linked_accounts = [] - self.linked_to = "" - self.refresh_token = "" - self.user_id = "" - # [{'device_id': '234', 'token': '92w'}] - self.notification_tokens = [] # stores users firebase tokens for sending mobile notifications - self.email_token = "" - super(User, self).__init__(attrs) - - def set_ssh_pub_key(self, key_value): - content = key_value.split() - # Create the key to validate it - RSA.importKey(' '.join(content)) - self.ssh_keys[content[-1]] = key_value - - def get_code(self): - if(self.pwd_validation_code and - self.pwd_validation_expiry > date_utils.now()): - return self.pwd_validation_code - return None - - def set_code(self, code): - self.pwd_validation_code = code - self.pwd_validation_expiry = (date_utils.now() + - datetime.timedelta(days=1)) - - validation_code = property(get_code, set_code) - - def add_devices(self, device): - time_now = time.time() - first_record = { - 'first_used': time_now, - 'user_agent': 'Administrative Addition', - } - - print(f'Current Device is {device}') - - if device not in self.valid_devices: - self.valid_devices[device] = {} - self.valid_devices[device].update(first_record) - - def add_notification_token(self, device_id, token): - device_id = device_id if isinstance(device_id, dict) else {"identifier":device_id} - if not self.notification_tokens: - self.notification_tokens = [] - for idx, device_token in enumerate(self.notification_tokens): - if device_token['device_id'] == device_id: - device_token['token'] = token - self.notification_tokens[idx] = device_token - break - else: - self.notification_tokens.append({'device_id': device_id, - 'token': token}) - - def remove_notification_token(self, device_id): - if not self.notification_tokens: - raise Exception('No devices to delete') - # If device_id is not there raise exception - device_id_str = device_id["identifier"] if isinstance(device_id, dict) else device_id - for idx, device_token in enumerate(self.notification_tokens): - device_token_str = device_token["device_id"]["identifier"] if isinstance(device_token["device_id"], dict) else device_token["device_id"] - if device_id_str == device_token_str: - del self.notification_tokens[idx] - - def reset_all_devices(self): - self.valid_devices = {} - - def remove_device(self, device): - try: - return self.valid_devices.pop(device) - except: - return False - - def add_ip(self, ip): - - # Check the format - expression = re.compile('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(/[\d]+)?$') - if not expression.match(ip): - return False - - first_record = { - 'first_used': time.time(), - 'user_agent': 'Administrative Addition' - } - - if ip not in self.valid_ips: - self.valid_ips[ip] = {} - self.valid_ips[ip].update(first_record) - return True - - def reset_all_ips(self): - self.valid_ips = {} - - def remove_ip(self, ip): - try: - return self.valid_ips.pop(ip) - except: - return False - - def new_validation(self, device, ip, user_agent): - time_now = time.time() - first_record = { - 'first_used': time_now, - 'first_used_from': ip - } - if device not in self.valid_devices: - self.valid_devices[device] = {} - self.valid_devices[device].update(first_record) - - if not 'user_agent' in self.valid_devices[device]: - self.valid_devices[device]['user_agent'] = user_agent - - ip_used = ip_match(ip, self.valid_ips) - - if ip_used is None: - tdetails = sec_context.details - ip_factors = tdetails.get_config('security', 'ip_factors', {}) - strict_ip_check = ip_factors.get('strict_ip_check', True) - if strict_ip_check: - ip_used = ip - else: - ip_segments = ip.split('.') - ip_segments[-1] = '0' - ip_used = '.'.join(ip_segments) + '/24' - - if ip_used not in self.valid_ips: - self.valid_ips[ip_used] = {} - - self.valid_ips[ip_used].update(first_record) - if not 'user_agent' in self.valid_ips[ip_used]: - self.valid_ips[ip_used]['user_agent'] = user_agent - - def set_last_login(self, device, ip, user_agent): - time_now = time.time() - last_record = {'last_used': time_now, 'last_used_from': ip} - if device in self.valid_devices: - self.valid_devices[device].update(last_record) - - ip_used = ip_match(ip, self.valid_ips) - if ip_used in self.valid_ips: - self.valid_ips[ip_used].update(last_record) - - def is_valid_ip(self, ip): - return ip_match(ip, self.valid_ips.keys()) - - def is_valid_device(self, device): - return device in self.valid_devices - - def get_refresh_token(self): - return self.refresh_token - - def encode(self, attrs): - attrs['name'] = self.name - if self.username: - attrs['username'] = self.username.lower() - else: - raise UserError("Missing username") - if self.email: - attrs['email'] = self.email - else: - raise UserError("Missing Email") - - if self.secondary_emails: - attrs['secondary_emails'] = self.secondary_emails - else: - attrs['secondary_emails'] = [] - - if self.app_modules: - attrs['app_modules'] = self.app_modules - else: - attrs['app_modules'] = [] - - if self.password_hash: - attrs['password'] = self.password_hash - else: - # To support postgres update - attrs['password'] = None - if self.last_login: - attrs['last_login'] = self.last_login - else: - attrs['last_login'] = None - - if self.validation_code: - attrs['validation_code'] = self.pwd_validation_code - attrs['validation_expiry'] = self.pwd_validation_expiry - else: - attrs['validation_code'] = None - attrs['validation_expiry'] = None - - # We are flatting the priv and dim assignment to avoid MongoDB - # restrictions - new_roles = {} - for r, role_val in self.roles.items(): - new_roles[r] = [] - for priv_val in role_val.itervalues(): - new_roles[r].extend(priv_val.values()) - attrs['roles'] = new_roles - attrs['user_timeout'] = self.user_timeout - attrs['is_customer'] = self.is_customer - attrs['user_role'] = self.user_role - attrs['edit_dims'] = self.edit_dims - - attrs['is_disabled'] = self.is_disabled - attrs['preset_validation_code'] = self.preset_validation_code - attrs['password_salt'] = self.password_salt - attrs['show_experimental'] = self.show_experimental - attrs['account_locked'] = self.account_locked - attrs['login_attempts'] = self.login_attempts - attrs['failed_login_time'] = self.failed_login_time - attrs['ssh_keys'] = self.ssh_keys.items() if not self.postgres else self.ssh_keys - if self.uuid: - attrs['uuid'] = self.uuid - attrs['uuid_validation_expiry'] = self.uuid_validation_expiry - attrs['activation_status'] = self.activation_status - else: - attrs['uuid'] = None - attrs['uuid_validation_expiry'] = None - attrs['activation_status'] = None - - attrs['valid_devices'] = self.valid_devices - attrs['valid_ips'] = {'items': [(k, v) for k, v in self.valid_ips.items()]} - - if self.mail_date: - attrs['mail_date'] = self.mail_date - else: - attrs['mail_date'] = None - attrs['is_second_login'] = self.is_second_login - attrs["linked_accounts"] = self.linked_accounts - attrs['refresh_token'] = self.refresh_token - attrs['user_id'] = self.user_id - attrs['notification_tokens'] = self.notification_tokens - attrs['email_token'] = self.email_token - if self.linked_to: - attrs["linked_to"] = self.linked_to - return super(User, self).encode(attrs) - - def decode(self, attrs): - self.name = attrs.get('name') - self.user_timeout = attrs.get('user_timeout', 120) - self.username = attrs.get('username') - self.password_hash = attrs.get('password') - self.email = attrs.get('email') - self.secondary_emails = attrs.get('secondary_emails',[]) - self.app_modules = attrs.get('app_modules', []) - self.last_login = attrs.get('last_login') - if self.last_login: - self.last_login = pytz.utc.localize(self.last_login) - self.pwd_validation_code = attrs.get('validation_code') - self.pwd_validation_expiry = attrs.get('validation_expiry') - if self.pwd_validation_expiry: - self.pwd_validation_expiry = pytz.utc.localize(self.pwd_validation_expiry) - self.roles = {} - - # Read the roles back and put them in the right place - new_roles = attrs.get('roles', {}) - for r, flat_list in new_roles.items(): - self.roles[r] = {} - for priv_tuple in flat_list: - priv, dim, write, delegate = priv_tuple - if priv not in self.roles[r]: - self.roles[r][priv] = {} - self.roles[r][priv][dim] = [priv, dim, write, delegate] - - self.edit_dims = attrs.get('edit_dims') - self.is_customer = attrs.get('is_customer') - self.user_role = attrs.get('user_role') - self.is_disabled = attrs.get('is_disabled', False) - self.preset_validation_code = attrs.get('preset_validation_code') - self.password_salt = attrs.get('password_salt', None) - self.show_experimental = attrs.get('show_experimental', False) - self.account_locked = attrs.get('account_locked', False) - self.login_attempts = attrs.get('login_attempts', 0) - self.failed_login_time = attrs.get('failed_login_time', 0) - self.ssh_keys = dict(attrs.get('ssh_keys', [])) - self.uuid = attrs.get('uuid') - self.uuid_validation_expiry = attrs.get('uuid_validation_expiry') - self.activation_status = attrs.get('activation_status') - self.mail_date = attrs.get('mail_date', None) - self.is_second_login = attrs.get('is_second_login', False) - self.linked_accounts = attrs.get("linked_accounts") or [] - self.linked_to = attrs.get("linked_to", "") - self.refresh_token = attrs.get('refresh_token', "") - self.user_id = attrs.get('user_id') - self.notification_tokens = attrs.get('notification_tokens') - self.email_token = attrs.get('email_token') - - tdetails = sec_context.details - now = time.time() - - def load_last_n(name, config_name): - device_factors = tdetails.get_config('security', config_name, {}) - hard_limit = device_factors.get('hard_limit', 60) * 24 * 60 * 60 - soft_limit = device_factors.get('soft_limit', 15) * 24 * 60 * 60 - max_devices = device_factors.get('max_entries', 20) - - valid_device_list = [] - admin_list = [] - attr_value = attrs.get(name, None) - if not attr_value: - return {} - - # Postgres workaround to add items - if 'items' in attr_value: - attr_value = attr_value['items'] - - if isinstance(attr_value, dict): - attr_value = attr_value.items() - - for d, d_info in attr_value: - if d_info.get('user_agent', '') == 'Administrative Addition': - admin_list.append([None, d, d_info]) - elif now < d_info.get('first_used', 0) + hard_limit and now < d_info.get('last_used', 0) + soft_limit: - valid_device_list.append([d_info.get('last_used', 0), d, d_info]) - - # Trim the list to first max_devices - valid_device_list = sorted(valid_device_list, reverse=True) - if len(valid_device_list) > max_devices: - valid_device_list = valid_device_list[0:max_devices] - return dict((x[1], x[2]) for x in admin_list + valid_device_list) - - # Load the valid devices and ips with a limit of last max_devices - self.valid_devices = load_last_n('valid_devices', 'device_factors') - self.valid_ips = load_last_n('valid_ips', 'ip_factors') - - super(User, self).decode(attrs) - - def upgrade_attrs(self, attrs): - if attrs['_version'] < 2: - attrs['object']['roles'] = dict((k, {}) for k in attrs['object']['roles']) - if attrs['_version'] < 3: - new_roles = {} - for r, role_val in attrs['object'].get('roles', {}).items(): - new_roles[r] = [] - for priv_val in role_val.itervalues(): - new_roles[r].append(priv_val) - attrs['object']['roles'] = new_roles - - @classmethod - def getUserByLogin(cls, username): - return cls.getByFieldValue('username', username.lower()) - - @classmethod - def getUserByKey(cls, key): - return cls.getByKey(key) - - @classmethod - def get_by_user_id(cls, user_id): - return cls.getByFieldValue('user_id', user_id) - - def verify_password(self, password): - password = password.strip() - if password.startswith('SIGNATURE::'): - dummy, dummy, t, dummy, signature = password.split(':') - for k in self.ssh_keys.itervalues(): - try: - key = RSA.importKey(k) - h = SHA.new() - h.update(t) - verifier = PKCS1_PSS.new(key) - if verifier.verify(h, base64.b64decode(signature)): - if abs(time.time() - float(t)) < 20: - return True - else: - logger.error('Trying to login using old signature for %s', self.username) - return False - except: - logger.exception('Unknown error') - return False - else: - cipher = SHA512.new(password + self.password_salt if self.password_salt else password) - hashed_pwd = cipher.hexdigest() - return hashed_pwd == self.password_hash - - def set_password(self, password): - password = password.strip() - if password.startswith('SIGNATURE::'): - raise UserError('Password may not start with that prefix') - self.password_salt = random_string() - cipher = SHA512.new(password + self.password_salt) - self.password_hash = cipher.hexdigest() - - def get_password(self): - raise UserError("Password can't be retrieved") - - password = property(get_password, set_password) - - def has_password(self): - return True if self.password_hash else False - - @property - def user_role_label(self): - try: - USER_ROLES = sec_context.details.get_config(category='forecast', - config_name='tenant').get('user_roles', None) or User.USER_ROLES - except: - USER_ROLES = User.USER_ROLES - return dict(USER_ROLES).get(self.user_role) @@ -1459,10 +962,6 @@ def decode(self, attrs): self.time_taken = attrs['time_taken'] self.run_type = attrs['run_type'] - @classmethod - def getByKey(cls, path): - return cls.getByFieldValue('path', path) - @classmethod def addv2statslog(cls, path, mem_used, cpu_perc, time_taken, run_type): stats = cls() diff --git a/domainmodel/csv_data.py b/domainmodel/csv_data.py index bd01e64..d61410d 100644 --- a/domainmodel/csv_data.py +++ b/domainmodel/csv_data.py @@ -1,5 +1,5 @@ import hashlib -from domainmodel import Model +from domainmodel.model import Model from utils import GnanaError from aviso.settings import sec_context, gnana_db2, gnana_db, sec_context @@ -506,62 +506,6 @@ def getByFieldValue(cls, field, value, check_unique=False): raise GnanaError("TooManyMatchesFound") return ret_val - @classmethod - def truncate_or_drop(cls, criteria=None, drop=False): - if drop: - # truncate or drop the collection - ret = super(CSVData, cls).truncate_or_drop(criteria=criteria) - # clear the data belongs to the cleared table in the promotion table - table_name = (CSVPromotionData.getCollectionName()).replace('.', '$') - statement = """delete from "{table}" """.format(table=table_name) - statement += """where "csv_type"='{csv_type}' and "csv_name"='{csv_name}' - """.format(csv_type=cls.csv_type, csv_name=cls.csv_name) - delete = True - if criteria: - delete = False - mnemonic = cls.get_mnemonic_from_criteria(criteria) - if mnemonic: - if isinstance(mnemonic, str): - statement += """ "quarter"='{quarter}' """.format(quarter=mnemonic) - delete = True - if isinstance(mnemonic, list): - statement += """ "quarter" in ({quarters})""".format( - quarters=','.join(q for q in mnemonic)) - delete = True - if delete: - cls.queryExecutor(statement) - return ret - - # call the csv upload process - kwargs = {'csv_name': cls.csv_name, 'csv_type': cls.csv_type, 'dd_type': 'complete', 'static_type': 'complete'} - return cls.truncate(criteria, **kwargs) - - @classmethod - @csv_version_decorator - def truncate(cls, criteria, **kwargs): - if criteria is None: - criteria = {} - # prerequisite - # should be switched to a version and switched version should be unpromoted one - snapshot_info_from_context = sec_context.csv_version_info - if not snapshot_info_from_context.get('snapshot'): - raise GnanaError("Please Switch to an unpromoted version and try again..") - if snapshot_info_from_context.get('promoted'): - raise GnanaError("Please Use a version which is unpromoted") - - # upload all the records into the version with mark as deleted - recs_to_reupload = [] - for record in cls.getAll(criteria): - rec = {} - record.encode(rec) - rec.update(rec.pop('dynamic_fields', None)) - # make record as marked_deleted true - rec.update({'marked_delete': True}) - recs_to_reupload.append(rec) - from tasks import csv_tasks - ret = csv_tasks.csvdatauploadprocess(recs_to_reupload, **kwargs) - return ret['updates'] - @classmethod def renameCollection(cls, new_col_name, overwrite=False): # update the promotion table with the new csv name diff --git a/domainmodel/datameta.py b/domainmodel/datameta.py index d8ff195..b0ab04d 100644 --- a/domainmodel/datameta.py +++ b/domainmodel/datameta.py @@ -12,14 +12,13 @@ local_db, sec_context) from django.http.response import HttpResponseNotFound -from domainmodel import Model, ModelError +from domainmodel.model import Model, ModelError from domainmodel.uip import (InboxEntry, InboxFileEntry, PartitionData, UIPRecord) from utils import GnanaError, forwardmap, update_dict from utils.config_utils import config_pattern_expansion from utils.date_utils import datetime2epoch, get_a_date_time_as_float_some_how from utils.math_utils import excelToFloat -from utils.string_utils import first15 logger = logging.getLogger('gnana.%s' % __name__) @@ -139,14 +138,6 @@ def apply_config(self, module_path, params): class SnapShotFileType(BasicFileType): - def get_inbox_record(self, extid, record, ts): - for field_name in self.id_fields: - try: - record[field_name] = first15(record[field_name]) - except KeyError: - pass - return {'extid': extid, 'when': ts, 'values': record} - def apply_inbox_record(self, uip_record, inbox_record, fld_config={}): for name, value in inbox_record.values.items(): uip_record.add_feature_value( @@ -242,31 +233,8 @@ def apply_inbox_record(self, uip_record, inbox_record, fld_config={}): 'snapshot': SnapShotFileType, 'history': HistoryFileType, 'usage': UsageFileType, - # 'tree': None, } - -# TODO: commenting this out because not used in hierarchy sync task -# --- Model Support --- -# model_types = { -# 'forecast.Forecast': Forecast, -# 'forecast2.Forecast2': Forecast2, -# 'forecast3.Forecast3': Forecast3, -# 'forecast4.Forecast4': Forecast4, -# 'forecast5.Forecast5': Forecast5, -# 'forecast.UnbornForecast': UnbornBaseModel, -# 'forecast3.Unborn3': Unborn3, -# 'forecast2.Forecast2_agg': Forecast2_agg, -# 'forecast2.Forecast2_no_ds': Forecast2_no_ds, -# 'forecast.FwdEPF': ForwardExistingPipeForecast, -# 'forecast.FwdNPF': ForwardNewPipeForecast, -# 'forecast.trajectoryforecast': TrajectoryForecast, -# 'forecast.trajectoryforecast2': TrajectoryForecast2, -# 'datatools.Counter': RecordCounter, -# 'account.Churn': account_churn, -# 'forecast2.EpfFromCsv': EpfFromCsv, -# } - class ModelDescription: """ Value object class to store the model configuration. diff --git a/domainmodel/dataset_maps.py b/domainmodel/dataset_maps.py index f5cabce..bbbc00e 100644 --- a/domainmodel/dataset_maps.py +++ b/domainmodel/dataset_maps.py @@ -6,18 +6,13 @@ from aviso.settings import sec_context -from domainmodel.datameta import UIPIterator from tasks.fields import parse_field from utils import GnanaError logger = logging.getLogger('gnana.%s' % __name__) -''' Possible copy_option values ''' copy_option_values = ['use_last', 'use_first', 'use_NA', 'use_Error'] def build_id_source(key, config, ds): - # TODO - Introduce a new type field to specify if a ID Source is a Prepare or Reduce type. - # if id_fields in the config not contains primary ID of the Source ds. - # then Reduce the uip_obj_list to a single uip_obj base_config = SourceMap(key, config) if base_config.reln_join_field not in base_config.id_fields: return AggregateSourceMap(key, config, ds) @@ -141,48 +136,6 @@ def get_map_details(self, output_fields_only=False): map_details['attributes'] = other_attr return map_details - def save_map_details(self, map_def, map_details, dataset_name, comments=''): - new_config = self.config_input - other_attr = map_details["attributes"] - new_config["all_fields"] = other_attr["all_fields"]['value'] - if 'reduce_maps' in self.config_input.keys(): - if "id_fields" in self.config_input.keys(): - new_config['id_fields'] = other_attr["id_fields"]['value'] - else: - data = map_details["field_info"]["data"] - fld_defs = self.config_input["fields_config"] - new_fld_defs = {} - for d in data: - if d["out_fld"] in fld_defs.keys(): - new_fld_defs[d["out_fld"]] = fld_defs[d["out_fld"]] - else: - new_fld_defs[d["out_fld"]] = {} - - if d["out_fld"] != d["in_fld"][0]: - if d["in_fld"][0] != '' and d["in_fld"][0] != '-': - new_fld_defs[d["out_fld"]]["in_fld"] = d["in_fld"][0] - else: - if "in_fld" in new_fld_defs[d["out_fld"]].keys(): - del new_fld_defs[d["out_fld"]]["in_fld"] - - if d["parser_fallback_lambda"] != '' and d["parser_fallback_lambda"] != '-': - new_fld_defs[d["out_fld"]]["parser_fallback_lambda"] = d["parser_fallback_lambda"] - else: - if "parser_fallback_lambda" in new_fld_defs[d["out_fld"]].keys(): - del new_fld_defs[d["out_fld"]]["parser_fallback_lambda"] - if d["parser"] != '' and d["parser"] != '-': - new_fld_defs[d["out_fld"]]["parser"] = d['parser'] - else: - if "parser" in new_fld_defs[d["out_fld"]].keys(): - del new_fld_defs[d["out_fld"]]["parser"] - - new_config["fields_config"] = new_fld_defs - - from feature import Feature - Feature().commit_dataset_config_changes(dataset_name, 'ConfigUI', - [('set_value', 'maps.' + map_def, new_config)], - comments) - def get_source_ds_fields(self): source_flds = {} req_flds = set() @@ -333,20 +286,6 @@ def process(self, uip_obj): uip_obj.compress(field_list) return uip_obj - def pre_cache(self, cache_options): - self.cache_options = cache_options - - def get_records(self): - record_iterator = UIPIterator(self.cache_options.src_uip_ds, - {'object.extid': {'$in': self.cache_options.source_id_list}}, - None) - - for record in record_iterator: - uip_obj = self.build_uip_obj(record) - features = record.all_features() - self.apply_fields_config(uip_obj, features) - yield uip_obj - class AggregateSourceMap(SourceMap): def __init__(self, fn_def_params, config, ds=None): @@ -356,21 +295,6 @@ def __init__(self, fn_def_params, config, ds=None): # single batch for the reduce to work self.batch_size = None - def pre_cache(self, cache_options): - record_iterator = UIPIterator(cache_options.src_uip_ds, - {'object.extid': {'$in': cache_options.source_id_list}}, - None) - for record in record_iterator: - uip_obj = self.build_uip_obj(record) - features = record.all_features() - self.apply_fields_config(uip_obj, features) - self.uip_reduce_dict['~'.join([str(uip_obj.getLatest(x)) - for x in self.id_fields])].append(uip_obj) - - def get_records(self): - for extid, uip_obj_list in self.uip_reduce_dict.iteritems(): - yield extid, uip_obj_list - def _aggregate(self, extid, uip_obj_list): if self.reln_join_field not in self.id_fields: uip_obj = self.stage_ds.DatasetClass() diff --git a/domainmodel/model.py b/domainmodel/model.py new file mode 100644 index 0000000..42e7d99 --- /dev/null +++ b/domainmodel/model.py @@ -0,0 +1,811 @@ +import logging +import re + +from bson import BSON +from aviso.settings import gnana_db, gnana_db2, sec_context +from pymongo import ASCENDING + +from utils import GnanaError, crypto_utils, date_utils, diff_rec + +logger = logging.getLogger('gnana.%s' % __name__) + + +class BaseBulkOperations: + + def __init__(self, collection_name): + raise NotImplementedError() + + def find(self, criteria): + raise NotImplementedError() + + def upsert(self): + raise NotImplementedError() + + def replace_one(self, doc): + raise NotImplementedError() + + def update_one(self, doc): + raise NotImplementedError() + + def insert(self, doc): + raise NotImplementedError() + + def execute(self): + raise NotImplementedError() + + +class Model: + """Base class for all domainmodel objects and provides the basic ORM + capabilities and conventions. + + In addition, this class also provides a + few class level methods that can be used to retrieve objects by keys + and names. + + This base class adds the following into the collection: + + _id + Database ID of the object + + _kind + Type of the object stored in the collection. Most of the time, a + collection holds homogeneous kind of objects, however this is not + guaranteed. + + _version + Each object will store a schema version used to store the object. + In the future it is expected that, an upgrade method is called to + progressively upgrade the data in the collection. + + .. NOTE:: As of now we are storing the version, but no upgrade logic + is implemented. + + object + All the attributes of the object are encoded by calling the encode + method and saved with object key + + During encoding each domainobject should ensure there are no periods + in any dictionaries. This is a limitation in document databases. + + Following Class level variables to be defined by each domain object: + + kind + Kind of the object + + tenant_aware + If the tenant aware is set to true, when saving the objects, + collection name is prefixed with the tenant name from security context + object + + version + Current object schema version + + index_list + A dictionary of indexes to be ensured when saving. When specifying + the index, additional options can be provided as a subdictionary. + + Examples:: + + # Just an index + 'probeid': {} + + # Unique index on object.username field in DB + 'username': {'unique': True} + + # Creating index in descending order, default value is ASCENDING + "created_time": {'direction': DESCENDING} + + # Index to expire documents after certain age. In this example + # document expires after 900 sec + value in object.timestamp + # field + 'timestamp': { + 'expireAfterSeconds': 900 + } + + query_list + A dictionary in the same format as indexes, that must be searchable + but not actually created in the DB as indexes. + + collection_name + Name of the collection in which to store the values. + + .. WARNING: Never use ModelClass.collection_name directly, instead + use ModelClass.getCollectionName() method. Using the method ensures + that tenant prefixing is done properly. + """ + kind = None + tenant_aware = True + collection_name = None + index_list = {} + version = None + index_checked = {} + encrypted = True + query_list = {} + check_sum = None # Used in UIP prepare to compare stage_uip & main_uip records + typed_fields = {} + compressed = False + read_from_primary = False + + def __init__(self, attrs): + if attrs is not None and (u'_id' in attrs or '_id' in attrs): + self.id = attrs[u'_id'] + if u'_kind' not in attrs: + raise ModelError("Kind of object is not defined by subclass") + if attrs[u'_kind'] != self.kind: + raise ModelError( + "Loading incorrect type of object, DB: %s, Required: %s" % + (attrs[u'_kind'], self.kind)) + # Ideally this should not be here. + if self.encrypted and self.tenant_aware and u'_encdata' in attrs and attrs.get('_encdata'): + decdata = BSON( + sec_context.decrypt(attrs[u'_encdata'])).decode() + if decdata: + attrs[u'object'] = decdata + if attrs['_version'] != self.version: + self.upgrade_attrs(attrs) + self.last_modified_time = attrs.get('last_modified_time', None) + self.check_sum = attrs.get('check_sum', None) + self.decode(attrs[u'object']) + else: + self.id = None + self.last_modified_time = None + self.check_sum = None + + def save(self, is_partial=False, bulk_list=None, id_field=None, field_list=None, update_fields=[], conditional=False): + """Save the object into the database. + If the object is initialized with attributes and an _id is found, + during the object creation, it is overwritten. If no _id is present + it is created. + id_field: Is used to update records based on the given field + conditional: If true ensures no updates have been made to this record between the read and the write. + returns the if of the record if the conditional write was successful, otherwise false + """ + if getattr(self, 'read_only', None): + raise GnanaError("Read only object cannot be saved!") + + collection_name = self.getCollectionName() + + # currently database views are used by tenant model + # To save data to tenant directly removing _view from the collection name + if '_view' in collection_name: + collection_name = collection_name.replace('_view', '') + + postgres = getattr(self, "postgres", None) + + self.create_index() + + localattrs = {} + prev_last_modified_time = self.last_modified_time + try: + self.last_modified_time = date_utils.epoch().as_epoch() + except: + self.last_modified_time = None + self.encode(localattrs) + localattrs.pop('last_modified_time', None) + check_sum_value = localattrs.pop('check_sum', None) + object_doc = {} + # field_list need to be pass in case of is_partial=True for task and result_cache save. + # for e.g field_list = ['object.extid', 'object.type'] etc. + if is_partial and field_list: + for i in field_list: + object_doc[i] = localattrs[i.split('.')[1]] + else: + object_doc = {'object': localattrs} + object_doc.update({'_kind': self.kind, + '_version': self.version, + 'last_modified_time': self.last_modified_time}) + if check_sum_value: + object_doc['check_sum'] = check_sum_value + if not field_list: + object_doc = self.update_object_doc(object_doc) + if bulk_list is not None: + if conditional: + raise Exception("Conditionals not supported by bulk saving") + + if not self.id and not id_field: + bulk_list.insert(object_doc) + else: + if self.id: + object_doc['_id'] = self.id + id_field = '_id' if not id_field else id_field + if isinstance(id_field, str): + id_field_list = [id_field] + else: + id_field_list = id_field + criteria = self.bulk_criteria(id_field_list, object_doc) + if is_partial: + bulk_list.find(criteria).upsert().update_one(object_doc) + else: + bulk_list.find(criteria).upsert().replace_one(object_doc) + return None + else: + if getattr(self, "postgres", None): + if conditional: + raise Exception("Conditionals not implemented for posgres") + self.id = gnana_db2.saveDocument( + collection_name, object_doc, self.id, is_partial=is_partial, update_fields=update_fields) + #logger.info('document updated in postgres with id %s', self.id) + else: + self.id = gnana_db.saveDocument( + collection_name, object_doc, self.id, is_partial=is_partial, update_fields=update_fields, + conditional=conditional, prev_last_modified_time=prev_last_modified_time) + #logger.info('document saved with id %s', self.id) + + return self.id + + def bulk_criteria(self, id_field, object_doc): + field_list = [] + for field in id_field: + if 'object.' in field: + f = field[len('object.'):] + field_list.append({field: object_doc['object'][f]}) + else: + field_list.append({field: object_doc[field]}) + criteria = {'$and': [x for x in field_list]} + return criteria + + def return_as_object_doc(self): + localattrs = {} + self.encode(localattrs) + localattrs.pop('last_modified_time', None) + check_sum_value = localattrs.pop('check_sum', None) + object_doc = {'_id': str(self.id), + 'object': localattrs, + '_kind': self.kind, + '_version': self.version, + 'last_modified_time': self.last_modified_time} + if check_sum_value: + object_doc['check_sum'] = check_sum_value + object_doc = self.update_object_doc(object_doc) + return object_doc + + def update_object_doc(self, object_doc): + localattrs = object_doc['object'] + localattrs = self.check_postgres_typed_fields(localattrs) + object_doc['object'] = localattrs + if self.tenant_aware and self.encrypted and sec_context.details.is_encrypted: + encdata = sec_context.encrypt(BSON.encode(localattrs), self) + if encdata: + self.encdata = encdata + object_doc.update({ + "_encdata": encdata, + "object": crypto_utils.extract_index(self.index_list, localattrs) + }) + object_doc["object"].update( + crypto_utils.extract_index(self.query_list, localattrs) + ) + return object_doc + + @classmethod + def bulk_ops(cls): + class PostGress_Bulk_Operations(BaseBulkOperations): + + def __init__(self, collection_name, encrypted, id_field): + self.collection_name = collection_name + self.encrypted = encrypted + self.bulk_list = [] + self.bulk_update = [] + self.find_objs = {} + self.id_field = id_field if id_field else 'extid' + + def execute(self): + # pass + if self.bulk_list: + logger.info("Using postgres bulk insert") + gnana_db2.postgres_bulk_execute( + self.bulk_list, self.collection_name) + if self.bulk_update: + logger.info("Using postgres bulk update") + gnana_db2.postgres_bulk_update(self.bulk_update) + + def insert(self, doc): + self.bulk_list.append(doc) + + def upsert(self): + return self + + def _replace(self, doc): + self.bulk_update.append( + gnana_db2.postgres_bulk_update_string(self.collection_name, doc)) + + def find(self, criteria): + if criteria: + obj = gnana_db2.findDocument(self.collection_name, criteria) + if obj: + self.find_objs[sec_context.encrypt(obj['object'][self.id_field], self)] = obj + return self + + def replace_one(self, doc): + if doc['object'][self.id_field] in self.find_objs: + saved_rec = self.find_objs.pop(doc['object'][self.id_field]) + doc['_id'] = saved_rec['_id'] + self._replace(doc) + else: + self.insert(doc) + + def update_one(self, doc): + saved_rec = self.find_objs.pop(doc['object'][self.id_field]) + saved_rec.update(doc) + self._replace(saved_rec) + + class MongoBulkOperations(BaseBulkOperations): + + def __init__(self, collection_name): + self.bulk_list = gnana_db.db[cls.getCollectionName()].initialize_unordered_bulk_op() + + def execute(self): + try: + self.bulk_list.execute() + except Exception as e: + no_op = re.match('No operations to execute', e.message) + if not no_op: + logger.exception('bulk list failed with error %s', e) + raise e + else: + logger.info('bulk list failed with No operations to execute error') + + def insert(self, doc): + self.bulk_list.insert(doc) + + def upsert(self): + return self.bulk_list.upsert() + + def _replace(self, doc): + raise Exception('replace is not supported for Mongo, this is specific to postgres') + + def find(self, criteria): + return self.bulk_list.find(criteria) + + def replace_one(self, doc): + self.bulk_list.replace_one(doc) + + def update_one(self, doc): + self.bulk_list.update_one(doc) + + class CompressedMongoBulk(BaseBulkOperations): + + def __init__(self, collection_name): + self.collection_name = collection_name + self.bulk_list = list() + self.bulk_update = dict() + + def find(self, criteria): + return self + + def upsert(self): + return self + + def replace_one(self, doc): + self.bulk_update[doc['object']['extid']] = doc + + def update_one(self, doc): + return self.replace_one(doc) + + def insert(self, doc): + self.bulk_list.append(doc) + + def execute(self): + if self.bulk_list: + cls.compressedSave(self.bulk_list) + if self.bulk_update: + cls.UpdateAndSave(self.bulk_update) + del self.bulk_list + del self.bulk_update + self.bulk_list = list() + self.bulk_update = dict() + + if getattr(cls, "postgres", False): + return PostGress_Bulk_Operations(cls.getCollectionName(), cls.encrypted, getattr(cls, 'id_field', None)) + elif cls.compressed: + return CompressedMongoBulk(cls.getCollectionName()) + else: + return MongoBulkOperations(cls.getCollectionName()) + + @classmethod + def bulk_insert(cls, rec_list): + cls.create_index() + tenant_details = sec_context.details + is_tenant_encrypted = tenant_details.is_encrypted + docs_to_insert = [] + for rec in rec_list: + localattrs = {} + rec.last_modified_time = date_utils.epoch().as_epoch() + rec.encode(localattrs) + localattrs.pop('last_modified_time', None) + check_sum_value = localattrs.pop('check_sum', None) + query_fields = [] + postgres = False + if getattr(cls, "postgres", None): + query_fields = cls.all_known_fields + postgres = True + if cls.typed_fields: + localattrs = cls.check_postgres_typed_fields(localattrs) + object_doc = {'object': localattrs, + '_kind': cls.kind, + '_version': cls.version, + 'last_modified_time': rec.last_modified_time} + if check_sum_value: + object_doc['check_sum'] = check_sum_value + if is_tenant_encrypted and cls.encrypted: + docs_to_insert.append(crypto_utils.encrypt_record(cls.index_list, object_doc, query_fields, postgres, + cls=cls)) + else: + docs_to_insert.append(object_doc) + if docs_to_insert: + if getattr(cls, "postgres", None): + gnana_db2.insert(cls.getCollectionName(), docs_to_insert) + else: + gnana_db.insert(cls.getCollectionName(), docs_to_insert) + + @classmethod + def copy_collection(cls, newcls, criteria={}, batch_size=5000): + if getattr(cls, "postgres", None): + cur = gnana_db2.findDocuments( + cls.getCollectionName(), criteria, auto_decrypt=True) + else: + cur = gnana_db.findDocuments( + cls.getCollectionName(), criteria, auto_decrypt=True) + count = cur.count() + start_size = 0 + while start_size < count: + to_process = min(count - start_size, batch_size) + newcls.bulk_insert(cur[start_size:start_size + to_process]) + start_size += to_process + + @classmethod + def truncate_or_drop(cls, criteria=None): + if criteria is None: + if getattr(cls, "postgres", None): + return gnana_db2.dropCollection(cls.getCollectionName()) + else: + return gnana_db.dropCollection(cls.getCollectionName()) + else: + if getattr(cls, "postgres", None): + return gnana_db2.truncateCollection(cls.getCollectionName(), criteria, cls.typed_fields) + else: + return gnana_db.truncateCollection(cls.getCollectionName(), criteria)['n'] + + @classmethod + def renameCollection(cls, new_col_name, overwrite=False): + if getattr(cls, "postgres", None): + gnana_db2.renameCollection( + cls.getCollectionName(), new_col_name, overwrite) + else: + gnana_db.renameCollection( + cls.getCollectionName(), new_col_name, overwrite) + + @classmethod + def create_index(cls): + postgres = getattr(cls, "postgres", None) + collection_name = cls.getCollectionName() + if '_view' in collection_name: + collection_name = collection_name.replace('_view', '') + if not cls.index_checked.get(collection_name, False): + for x in cls.index_list: + index_creation_method = None + if x in cls.typed_fields: + typed_field_type = cls.typed_fields[x]['type'] + is_array = re.search("(ARRA\w+)", typed_field_type) or re.search("[\[]", typed_field_type) + if is_array: + index_creation_method = 'gin' + options = cls.index_list[x] + if 'index_spec' in cls.index_list[x]: + options = options.copy() + index_spec = options.pop('index_spec') + else: + index_field_list = x.split('~') + index_spec = [] + for f in index_field_list: + direction = options.pop('direction', ASCENDING) + index_fld = ("object.%s" % f, direction) + index_spec.append(index_fld) + if postgres: + gnana_db2.ensureIndex(collection_name, index_spec, options, method=index_creation_method) + else: + gnana_db.ensureIndex(collection_name, index_spec, options, method=index_creation_method) + + # Special indexes that need to be added for all models are to be + # defined here + if postgres: + gnana_db2.ensureIndex( + collection_name, "last_modified_time", {}) + else: + gnana_db.ensureIndex(collection_name, "last_modified_time", {}) + cls.index_checked[collection_name] = True + # Updating index list of class with last_modified_time to save it to saved_index_info + cls.index_list.update({"last_modified_time": {}}) + all_query_fields_new = { + 'index_list': loaded_index_list(cls.index_list), + 'query_list': loaded_index_list(cls.query_list) + } + all_query_fields_old = None + try: + all_query_fields_old = gnana_db.findDocument( + sec_context.name + '.saved_index_info', + {'collection': collection_name} + ) + except: + pass + if (all_query_fields_old is None or + diff_rec(all_query_fields_new, all_query_fields_old['index_info'])): + if not all_query_fields_old: + all_query_fields_old = {'collection': collection_name} + all_query_fields_old['index_info'] = all_query_fields_new + try: + if sec_context.name != 'administrative.domain': + gnana_db.saveDocument( + sec_context.name + '.saved_index_info', + all_query_fields_old + ) + except Exception as e: + logger.exception(f"Got Exception while saving index info: {e}") + + @classmethod + def getCollectionName(cls): + """ + Return the collection name to be used for the class. + + .. Warning:: Do not cache this value, as it will change based on + the context. + """ + if cls.tenant_aware: + return sec_context.name + "." + cls.collection_name + else: + return cls.collection_name + + @classmethod + def getBySpecifiedCriteria(cls, criteria, check_unique=False): + """Find an object by given criteria. The caller can specify any + MongoDB-blessed criteria to find a specific document. + + check_unique + When multiple objects match the field value, passing check_unique as + True will explicitly checking nothing else matched. Otherwise it + returns the first matching object + + .. NOTE:: + + Generally you should use the unique_indexes and not depend + on this mechanism. + """ + if getattr(cls, "postgres", None): + attrs = gnana_db2.findDocument(cls.getCollectionName(), + criteria, + check_unique) + else: + attrs = gnana_db.findDocument(cls.getCollectionName(), + criteria, + check_unique, + read_from_primary=cls.read_from_primary, + tenant_aware=cls.tenant_aware) + if attrs: + return cls(attrs) + else: + return None + + @classmethod + def getByFieldValue(cls, field, value, check_unique=False): + """Find an object by given field value. ``object.`` is automatically + prepended to the field name provided. An object of the class on which + this method is called will be created, hence this method will work for + all domainmodel objects. + + forgive + When multiple objects match the field value, passing forgive as + True will return the first matching object. If it is False, an + exception is raised. + + TODO: Refactor this - use getBySpecifiedCriteria once the tests pass + """ + if getattr(cls, "postgres", None): + attrs = gnana_db2.findDocument(cls.getCollectionName(), + {'object.' + field: + sec_context.encrypt( + value) if cls.tenant_aware and cls.encrypted else value}, + check_unique) + else: + attrs = cls.get_db().findDocument(cls.getCollectionName(), + {'object.' + field: + sec_context.encrypt( + value) if cls.tenant_aware and cls.encrypted else value}, + check_unique, read_from_primary=cls.read_from_primary, + tenant_aware=cls.tenant_aware + ) + + return cls(attrs) if attrs else None + + @classmethod + def getAllByFieldValue(cls, field, value): + """Find all objects by given field value. ``object.`` is automatically + prepended to the field name provided. An object of the class on which + this method is called will be created, hence this method will work for + all domainmodel objects. + + + TODO: Refactor this - use getBySpecifiedCriteria once the tests pass + """ + try: + if getattr(cls, "postgres", None): + my_iter = gnana_db2.findAllDocuments(cls.getCollectionName(), + {'object.' + field: sec_context.encrypt(value) + if cls.tenant_aware and cls.encrypted else value}) + else: + my_iter = gnana_db.findAllDocuments(cls.getCollectionName(), + {'object.' + field: sec_context.encrypt(value) + if cls.tenant_aware and cls.encrypted else value}, + read_from_primary=cls.read_from_primary, + tenant_aware=cls.tenant_aware) + for attrs in my_iter: + yield cls(attrs) + except: + logger.exception("Exception raised while finding values %s-%s-%s" % + (cls.getCollectionName(), field, value)) + + @classmethod + def getAll(cls, criteria=None, fieldList=[], return_dict=False): + """Find all objects. An object of the class on which + this method is called will be created, hence this method will work for + all domainmodel objects. + """ + if criteria is None: + criteria = {} + # currently we are supporting the fieldList only for postgres as mongo has encrypted data, there are other steps to be + # done. to support field projection for mongo + try: + if not getattr(cls, 'postgres', None): + fieldList = [] + my_iter = cls.get_db().findAllDocuments( + cls.getCollectionName(), criteria, cls.typed_fields, fieldList, + read_from_primary=cls.read_from_primary, + tenant_aware=cls.tenant_aware) + if return_dict: + for attrs in my_iter: + yield attrs['object'] + else: + for attrs in my_iter: + cls_obj = cls(attrs) + if fieldList: + # Making the class object to be read only if only some fields are requested. + setattr(cls_obj, 'read_only', True) + yield cls_obj + except Exception as e: + logger.exception(e) + + @classmethod + def getByKey(cls, key): + """Find an object by key + """ + try: + if getattr(cls, "postgres", None): + attrs = gnana_db2.retrieve(cls.getCollectionName(), key) + else: + attrs = gnana_db.retrieve(cls.getCollectionName(), key, read_from_primary=cls.read_from_primary, + tenant_aware=cls.tenant_aware) + except StopIteration: + attrs = None + return attrs and cls(attrs) or None + + @classmethod + def getByName(cls, name): + """Find an object by name. This is shorthand to getByFieldValue + """ + return cls.getByFieldValue('name', name) + + # Nothing to encode, id is used automatically + def encode(self, attrs): + """Called before saving to get a dictionary representation of the + object suitable for saving into a document database. Make sure to + call the super class method in the implementations. No need to return + anything, just updating the attrs is enough. + """ + attrs['last_modified_time'] = self.last_modified_time + attrs['check_sum'] = self.check_sum + return attrs + + # Nothing to decode + def decode(self, attrs): + """Called to reconstruct the object from the dictionary. Initialize + any required variables and make sure to call the super class + """ + self.last_modified_time = self.last_modified_time or attrs.get(u'last_modified_time', None) + self.check_sum = self.check_sum or attrs.get(u'check_sum', None) + + @classmethod + def remove(cls, objid): + """Remove the object from the database + """ + if objid: + if getattr(cls, "postgres", None): + gnana_db2.removeDocument(cls.getCollectionName(), objid) + else: + gnana_db.removeDocument(cls.getCollectionName(), objid) + else: + raise ModelError("Can't remove unbound db object") + + def _remove(self): + self.remove(self.id) + + @classmethod + def list_all_collections(cls, prefix): + if getattr(cls, "postgres", None): + return gnana_db2.collection_names(prefix) + else: + return gnana_db.collection_names(prefix) + + @classmethod + def check_postgres_typed_fields(cls, localattrs): + if not cls.typed_fields: + return localattrs + for key in localattrs: + if key in cls.typed_fields.keys(): + typed_field_type = cls.typed_fields[key]['type'] + is_array = re.search("(ARRA\w+)", typed_field_type) or re.search("[\[]", typed_field_type) + if is_array and not isinstance(localattrs[key], list): + if localattrs[key]: + localattrs[key] = [localattrs[key]] + return localattrs + + @classmethod + def get_db(cls): + return gnana_db2 if getattr(cls, "postgres", None) else gnana_db + + @classmethod + def getDistinctValues(cls, key, criteria={}): + encrypted = False + if cls.tenant_aware and cls.encrypted and sec_context.details.is_encrypted: + encrypted = True + if getattr(cls, "postgres", None): + return gnana_db2.getDistinctValues(cls.getCollectionName(), key, criteria=criteria, encrypted=encrypted) + else: + return gnana_db.getDistinctValues(cls.getCollectionName(), key, criteria=criteria, + encrypted=encrypted, + read_from_primary=cls.read_from_primary, + tenant_aware=cls.tenant_aware) + + @classmethod + def queryExecutor(cls, statement, return_dict=False): + if getattr(cls, "postgres", None): + try: + my_iter = gnana_db2.postgres_query_executor(cls.getCollectionName(), statement) + if return_dict: + for attrs in my_iter: + yield attrs['object'] + else: + for attrs in my_iter: + yield cls(attrs) + except Exception as e: + logger.exception(e) + raise e + + @classmethod + def get_count(cls, criteria=None): + if getattr(cls, "postgres", None): + return cls.get_db().find_count(cls.getCollectionName(), criteria) + else: + if criteria is None: + criteria = {} + return cls.get_db().find_count(cls.getCollectionName(), criteria, read_from_primary=cls.read_from_primary, + tenant_aware=cls.tenant_aware) + + +class ModelError(Exception): + + def __init__(self, error): + logger.error(error) + self.error = error + +def loaded_index_list(index_list_class): + index_list_db = {} + for k, v in index_list_class.iteritems(): + name = k.replace('.', '~') + index_list_db[name] = v.copy() + if 'index_spec' in v: + index_list_db[name]['key'] = v.get('index_spec') + else: + index_field_list = k.split('~') + index_spec = [] + for f in index_field_list: + index_fld = ("object.%s" % f, ASCENDING) + index_spec.append(index_fld) + index_list_db[name]['key'] = index_spec + return index_list_db diff --git a/domainmodel/tenant.py b/domainmodel/tenant.py deleted file mode 100644 index fff0ad1..0000000 --- a/domainmodel/tenant.py +++ /dev/null @@ -1,208 +0,0 @@ -import datetime -import threading -import uuid -from aviso.settings import event_context, sec_context -from pymongo import DESCENDING - -from domainmodel import Model -from utils import date_utils -from utils.date_utils import EpochClass - - -class TenantEvents(Model): - collection_name = "tenant_events" - version = 1.0 - encrypted = False - index_list = { - "created_time": {"direction": DESCENDING}, - "expires": {'expireAfterSeconds': 10}, - "color": {}, - 'username': {} - } - - def __init__(self, attrs=None): - self.username = None - self.message = None - self.created_time = None - self.end_time = None - self.color = None - self.parent_id = None - self.event_id = None - self.exception_value = None - self.params = None - self.expires = date_utils.now() + datetime.timedelta(90) - super(TenantEvents, self).__init__(attrs) - - def encode(self, attrs): - attrs['username'] = self.username - attrs['message'] = self.message - attrs['created_time'] = self.created_time - attrs['color'] = self.color - attrs['params'] = self.params - attrs['end_time'] = self.end_time - attrs['parent_id'] = self.parent_id - attrs['event_id'] = self.event_id - attrs['exception_value'] = self.exception_value - attrs['expires'] = self.expires - return attrs - - def decode(self, attrs): - self.username = attrs['username'] - self.message = attrs['message'] - self.created_time = attrs['created_time'] - self.color = attrs['color'] - self.params = attrs.get('params', None) - self.end_time = attrs.get('end_time', None) - self.parent_id = attrs['parent_id'] - self.event_id = attrs['event_id'] - self.exception_value = attrs.get('exception_value', None) - self.expires = attrs['expires'] - - @classmethod - def create_tenant_log(cls, message, color, params, start_time, end_time, parent_id, event_id): - log_message = cls() - log_message.message = message - log_message.color = color - log_message.params = params - log_message.created_time = start_time - log_message.end_time = end_time - log_message.parent_id = parent_id - log_message.event_id = event_id - log_message.username = sec_context.user_name + "@" + sec_context.login_tenant_name - log_message.save() - return log_message - - @classmethod - def update_tenant_log(cls, event_id, end_time, exception_value): - cursor = cls.getByFieldValue('event_id', event_id) - cursor.end_time = end_time - cursor.exception_value = exception_value - cursor.save(id_field='event_id') - - @classmethod - def get_records(cls, color=None, username=None, str_msg=None, end_time=None, records=100, sub_events=False): - criteria = {} - if color: - criteria.update({'object.color': {'$in': color}}) - if str_msg: - regfind = r'('+str_msg+'){1}' - criteria.update({'object.message': {'$regex': regfind}}) - if username: - criteria.update({'object.username': username}) - if end_time == 'in_process': - criteria.update({'object.end_time': None}) - if type(end_time) is int: - criteria.update({'object.end_time': {'$lt': end_time}}) - if not sub_events: - criteria.update({'object.parent_id': None}) - records = int(records) - db_to_use = cls.get_db() - cursor = db_to_use.findDocuments( - name=cls.getCollectionName(), - criteria=criteria, - sort=[('object.created_time', -1)], - ).limit(records) - for x in cursor: - obj = cls(x) - yield {'username': obj.username, - 'color': obj.color, - 'time': obj.created_time, - 'message': obj.message, - 'end_time': obj.end_time, - 'parent_id': obj.parent_id, - 'event_id': obj.event_id, - 'params': obj.params - } - - @classmethod - def get_hierarchy(cls, event_id=None): - if not event_id: - return - return TenantEvents().get_event_tree(TenantEvents().get_root(event_id)) - - def get_root(self, event_id): - if not self.get_parent(event_id): - return event_id - else: - return self.get_root(self.get_parent(event_id)) - - def get_event_tree(self, current_id): - if not current_id: - return - else: - dic = {} - cursor = self.getAllByFieldValue('event_id', current_id) - for obj in cursor: - dic = {'username': obj.username, - 'color': obj.color, - 'time': obj.created_time, - 'message': obj.message, - 'end_time': obj.end_time, - 'parent_id': obj.parent_id, - 'event_id': obj.event_id, - 'params': obj.params, - 'children': [] - } - child_cursor = self.getAllByFieldValue('parent_id', current_id) - lst = list(child_cursor) - child_id = None - if lst: - for i in lst: - child_id = i.event_id - dic['children'].append(self.get_event_tree(child_id)) - return dic - - def get_parent(self, child_id): - rec = self.getByFieldValue('event_id', child_id) - parent = rec.parent_id - return parent - - -class TenantEventContext: - event_stack = threading.local() - - def __init__(self, message, color, params): - self.message = message - self.color = color - self.params = params - self.start_time = None - self.end_time = None - self.event_id = str(uuid.uuid4()) - self.parent = None - self.model_obj = None - self.exception_value = None - - def __enter__(self): - self.start_time = EpochClass().as_epoch() - try: - self.parent = self.event_stack.stack[-1] # for sub-events just append in the list with their id - self.event_stack.stack.append(self.event_id) - except (IndexError, AttributeError): - parent_eventid = event_context.event_id - if parent_eventid: - self.parent = parent_eventid - else: - self.parent = None - self.event_stack.stack = [self.event_id] # gives AttributeError and starts the list with parent id - event_context.set_event_context(self.event_id) - self.model_obj = TenantEvents.create_tenant_log(self.message, self.color, self.params, self.start_time, - self.end_time, self.parent, self.event_id) - return self - - def __exit__(self, exc_type, exc_value, exc_traceback): - self.event_stack.stack = self.event_stack.stack[0:-1] - if exc_type: - self.exception_value = "Type: " + str(exc_type) + ", Value: " + str(exc_value) - if self.model_obj: - self.model_obj.end_time = EpochClass().as_epoch() - self.model_obj.exception_value = self.exception_value - self.model_obj.save() - -def tenant_events(message, color): - def tenant_event_decorator(fn): - def new_func(*args, **kwargs): - with TenantEventContext(message, color, kwargs): - response = fn(*args, **kwargs) - return response - return new_func - return tenant_event_decorator \ No newline at end of file diff --git a/domainmodel/uip.py b/domainmodel/uip.py index 527b666..38dbd37 100644 --- a/domainmodel/uip.py +++ b/domainmodel/uip.py @@ -10,7 +10,7 @@ from aviso import settings from aviso.settings import sec_context, gnana_db -from domainmodel import Model +from domainmodel.model import Model from tasks.fields import yyyymmdd_to_xl from utils import GnanaError from utils.date_utils import get_a_date_time_as_float_some_how, datetime2xl, epoch, xl2datetime diff --git a/domainmodel/uip_maps.py b/domainmodel/uip_maps.py index d033bad..e756f95 100644 --- a/domainmodel/uip_maps.py +++ b/domainmodel/uip_maps.py @@ -152,26 +152,9 @@ def __init__(self, feature, config, stage_ds=None): self.stage_ds = stage_ds super(DBMap, self).__init__(feature, config, stage_ds) - def get_map_details(self, output_fields_only=False): - return None - def process(self, uip_obj): raise Exception('abstract class') - def get_prepare_criteria(self): - # Populate the prepare criteria - return self.prepare_criteria - - def cache_criteria(self): - # Populate the cache criteria based on the config - return self.cache_criteria - - def pre_cache(self, cache_options=None): - pass - - def build_wrt_db(self): - pass - class OneToManyDBMap(DBMap): @@ -276,53 +259,6 @@ def get_map_details(self, output_fields_only=False): map_details["attributes"] = other_attr return map_details - def save_map_details(self, map_def, map_details, dataset_name, comments=''): - new_config = self.config - other_attr = map_details["attributes"] - if "lookup_fld" in self.config.keys(): - new_config["lookup_fld"] = other_attr["lookup_fld"]['value'] - if "exec_rank" in self.config.keys(): - new_config["exec_rank"] = other_attr["exec_rank"]['value'] - if "log_warnings" in self.config.keys(): - new_config["log_warnings"] = other_attr["log_warnings"]['value'] - - data = map_details["field_info"]["data"] - fld_defs = self.config["fld_defs"] - new_fld_defs = {} - for d in data: - if d["out_fld"] in fld_defs.keys(): - new_fld_defs[d["out_fld"]] = fld_defs[d["out_fld"]] - else: - new_fld_defs[d["out_fld"]] = {} - if d["out_fld"] != d["req_fields"]: - if d['req_fields'] != '' and d['req_fields'] != '-': - new_fld_defs[d["out_fld"]]["req_fields"] = d["req_fields"] - else: - if "req_fields" in new_fld_defs[d["out_fld"]].keys(): - del new_fld_defs[d["out_fld"]]["req_fields"] - if d["fallback_val"] != '' and d["fallback_val"] != '-': - new_fld_defs[d["out_fld"]]["fallback_val"] = d["fallback_val"] - else: - if "fallback_val" in new_fld_defs[d["out_fld"]].keys(): - del new_fld_defs[d["out_fld"]]["fallback_val"] - new_config["fld_defs"] = new_fld_defs - - from feature import Feature - Feature().commit_dataset_config_changes(dataset_name, 'ConfigUI', - [('set_value', 'maps.' + map_def, new_config)], - comments) - - def get_source_ds_fields(self): - source_flds = {} - req_flds = set() - fld_defs = self.config['fld_defs'] - if self.config['ds_name']: - req_flds.add(self.config['lookup_fld']) - for _, defs in fld_defs.items(): - req_flds |= set(defs["req_fields"]) - source_flds[self.config['ds_name']] = req_flds - return source_flds - def process(self, uip_obj): """ Join using the history of by looking it up in . diff --git a/feature/__init__.py b/feature/__init__.py deleted file mode 100644 index 949b1ab..0000000 --- a/feature/__init__.py +++ /dev/null @@ -1,195 +0,0 @@ -import copy -import time - -from utils.mail_utils import backup_and_mail_changes -from aviso.settings import CNAME, CNAME_DISPLAY_NAME, gnana_db2, sec_context - -from domainmodel import datameta -from domainmodel.tenant import tenant_events -from tasks.stage import ds_stage_config -from tasks.targetspecTasks import target_spec_tasks -from utils.mail_utils import send_mail2 - - -class Feature: - ui_name = 'FeatureX' - ui_description = 'This will allow you to do configuration changes.' - category = "Misc" - - """ If you want to order your feature in UI, update it here """ - ui_feature_order = ("Forecast Management", "Data Science", "ETL & Integration", "Reports", "Misc") - - def __init__(self): - self.host = 'https://'+(CNAME_DISPLAY_NAME if CNAME_DISPLAY_NAME is not None else CNAME)+'.aviso.com' - self.all_config = None - self.all_datasets_config = {} - self.all_target_spec = {} - - def getModes(self): - return self.modes.keys() - - def getModeDescription(self, featuremode): - return self.modes[featuremode] - - def getCurrentMode(self): - raise Exception('Must be implemented by subclass') - - def getRequiredInputs(self, featuremode): - raise Exception('Must be implemented by subclass') - - def getCurrentConfig(self, featuremode): - raise Exception('Must be implemented by subclass') - - def getRestrictedModes(self, all_feature_modes): - return {} - - def getRequiredJobDetails(self,featuremode): - return {} - - def configure(self, featuremode, **kwargs): - @tenant_events('Configure '+self.ui_name+' in '+featuremode+ "(" +kwargs['comment']+")", 'blue') - def run_configuration(): - return self.do_configuration(featuremode, kwargs['config'], kwargs['comment']) - return run_configuration() - - def get_nested_config(self, nesting, default=None): - if not self.all_config: - t_details = sec_context.details - self.all_config = copy.deepcopy(t_details.get_all_config()) - - config_dict = copy.deepcopy(self.all_config) - for x in nesting[:-1]: - config_dict = config_dict.get(x, {}) - - return config_dict.get(nesting[-1], default) - - def commit_tenant_config_changes(self, change_list, comments): - # Commit the tenant config changes - # To remove a config_name key append (category, config_name) to change list - # To set config to a config_name key append (category, config_name, config ) to change_list - - if not change_list: - return - t_details = sec_context.details - for each_change in change_list: - each_change_length = len(each_change) - if each_change_length == 2: - if t_details.get_config(each_change[0], each_change[1]): - t_details.remove_config(each_change[0], each_change[1]) - elif each_change_length == 3: - t_details.set_config(each_change[0], each_change[1], each_change[2]) - else: - raise Exception('change_list is not good') - - # Backup and notify - tenant_name = sec_context.name - tenant_name = tenant_name.lower() - current_user_name = sec_context.user_name - current_user_name = current_user_name.lower() - new_tenant_details = t_details.get_all_config() - backup_and_mail_changes(self.all_config, new_tenant_details, tenant_name, current_user_name, comments, - 'Configure_UI') - - def get_dataset(self, ds_type): - known_datasets = [] - known_dataset = gnana_db2.findDocuments(datameta.Dataset.getCollectionName(), {'object.ds_type': ds_type}, {}) - for kd in known_dataset: - known_datasets.append(kd['object']['name']) - return known_datasets - - def get_dataset_nested_config(self, **kwargs): - dataset = kwargs.get('dataset', 'OppDS') - stage = kwargs.get('stage', None) - path = kwargs.get('path', None) - default_ret = kwargs.get('default_return_val', None) - if not self.all_datasets_config.get(dataset, None): - ds = datameta.Dataset - self.all_datasets_config[dataset] = copy.deepcopy(ds.getByNameAndStage(dataset, stage).get_as_map()) - ds_config_dict = copy.deepcopy(self.all_datasets_config[dataset]) - - nesting = path.split('.') - for config_key in nesting[:-1]: - ds_config_dict = ds_config_dict.get(config_key, {}) - - return ds_config_dict.get(nesting[-1], default_ret) - - def commit_dataset_config_changes(self, dataset, stage, change_list, comments): - """ - dataset: dataset name - stage: stage name - change_list: a list of tuples where each tuple contains action, path and new_value - **if action is 'remove_path' then we should pass new_value as None - comments: comments - """ - if not change_list: - return - ds = datameta.Dataset.getByName(dataset) - ds.stages[stage] = {'changes': [], - 'comment': comments} - for each_change in change_list: - ds.apply(stage, each_change[0], each_change[1], each_change[2]) - ds.save() - - # Backup - tenant_name = sec_context.name - tenant_name = tenant_name.lower() - current_user_name = sec_context.user_name - current_user_name = current_user_name.lower() - ds_stage_config([current_user_name, tenant_name], stage, ds, uip_merge=False, dataset=dataset, - stage=stage, approver=current_user_name) - - def get_target_spec_nested_config(self, target_spec, nesting, default=None): - if not self.all_target_spec.get(target_spec, None): - ts = datameta.TargetSpec.getByName(target_spec) - if not ts: - self.all_target_spec[target_spec] = ts - return default - self.all_target_spec[target_spec] = copy.deepcopy(ts.encode({})) - if not nesting: - return copy.deepcopy(self.all_target_spec[target_spec]) - - ts_config_dict = copy.deepcopy(self.all_target_spec[target_spec]) - for x in nesting[:-1]: - ts_config_dict = ts_config_dict.get(x, {}) - - return ts_config_dict.get(nesting[-1], default) - - def commit_target_spec_changes(self, target_name, action, comments, new_vlaue, module_path=None): - current_user_name = sec_context.user_name - target_spec = datameta.TargetSpec.getByFieldValue('name', target_name) - old_target_spec = {} - if target_spec: - old_target_spec = copy.deepcopy(target_spec.__dict__) - target_spec_tasks(current_user_name, target_name, new_vlaue, False, action, old_target_spec, - module_path, comments) - - def get_tenant_endpoint(self, endpoint_name): - tnt_details = sec_context.details - try: - cred = tnt_details.get_credentials(endpoint_name) - return cred - except: - return None - - def set_tenant_endpoint(self, endpoint_name, creds): - tnt_details = sec_context.details - credentials = creds - credentials['updated_time'] = int(time.time()) - credentials['sent_mail'] = False - tnt_details.save_credentials(endpoint_name, credentials) - tnt_details.save() - cname = CNAME_DISPLAY_NAME if CNAME_DISPLAY_NAME else CNAME - send_mail2( - 'endpoint_change.txt', - 'notifications@aviso.com', - tnt_details.get_config('receivers', 'endpoint_changes', ['gnackers@aviso.com']), - reply_to='Data Science Team ', - tenantname=sec_context.name, cName=cname, - endpoint=endpoint_name, - modifier=sec_context.user_name, user_name=sec_context.user_name, - ) - try: - cred = tnt_details.get_credentials(endpoint_name) - return cred - except: - return None diff --git a/fm_service/__init__.py b/fm_service/__init__.py index a2dc8d8..e69de29 100644 --- a/fm_service/__init__.py +++ b/fm_service/__init__.py @@ -1,115 +0,0 @@ -from django.http import HttpResponseBadRequest - -from config.fm_config import DealConfig, FMConfig -from infra.read import get_current_period, node_is_valid -from infra.read import period_is_active as active_period -from infra.read import validate_period -from utils.common import MicroAppView, cached_property - - -class FMView(MicroAppView): - """ - Base class for all fm service views - """ - validators = [] - - @cached_property - def config(self): - return FMConfig(debug=self.debug) - - @cached_property - def deal_config(self): - return DealConfig() - - -# Error Codes -UPLOAD_ERROR_400 = "Tried to upload for a non-user-entered field." -INVALID_PERIOD = 'The provided period is not valid' -INVALID_NODE = 'No such node for the specified quarter' -INACTIVE_PERIOD = 'The provided period is not an active period' -MALFORMED_RECORDS = 'The uploaded records are not in the correct format, fields required: {}' -EXCEL_FORMAT_CHANGE = 'Please ensure the date column headers must be in yyyy-mm-dd format' - - -# API Validators -def field_is_user_entered(request, config): - field = request.GET.get('field') - if field not in config.user_entered_fields and field not in config.forecast_service_editable_fields: - return HttpResponseBadRequest(UPLOAD_ERROR_400) - - -def period_is_active(request, config): - period = request.GET.get('period', get_current_period()) - if not active_period(period, config=None, quarter_editable=config.quarter_editable, - component_periods_editable=config.component_periods_editable, - future_quarter_editable=config.future_qtrs_editable_count, - past_quarter_editable=config.past_qtrs_editable_count): - return HttpResponseBadRequest(INACTIVE_PERIOD) - - -def period_is_valid(request, config): - period = request.GET.get('period', get_current_period()) - if not validate_period(period): - return HttpResponseBadRequest(INVALID_PERIOD) - - -def node_is_editable(request, config): - node = request.GET.get('node') - period = request.GET.get('period', get_current_period()) - if not node_is_valid(node, period): - return HttpResponseBadRequest(INVALID_NODE) - - -def node_can_access_field(request, config): - node = request.GET.get('node') - fields = request.GET.getlist('field') - # TODO: me - - -def records_are_valid(records, config): - required_fields = ['period', 'node', 'field', 'val'] - for record in records: - if any((field not in record for field in required_fields)): - return HttpResponseBadRequest(MALFORMED_RECORDS.format(required_fields)) - if record['field'] not in config.user_entered_fields and record['field'] not in config.forecast_service_editable_fields: - return HttpResponseBadRequest(UPLOAD_ERROR_400) - - -def validating_excel_column_headers(field): - default_fields = {'period', 'node', 'label'} - if field not in default_fields: - if len(field) != 10: - return EXCEL_FORMAT_CHANGE - if (field[4] != '-' or field[7] != '-'): - return EXCEL_FORMAT_CHANGE - -def get_all_waterfall_fields(): - default_fields = ['type'] - for i in range(1, 14): - default_fields.append('week' + str(i)) - - return default_fields - -def get_all_waterfall_track_fields(): - default_fields = ['Node', 'Label'] - for i in range(1, 14): - default_fields.append('week' + str(i)) - - return default_fields - - -def validate_waterfall_headers(fields): - default_fields = get_all_waterfall_fields() - - for field in fields: - if field not in default_fields: - return False - - return True - -def validate_waterfall_track_headers(fields): - default_fields = get_all_waterfall_track_fields() - for field in fields: - if field not in default_fields: - return False - return True diff --git a/fm_service/fetch.py b/fm_service/fetch.py index c08439a..5644f52 100644 --- a/fm_service/fetch.py +++ b/fm_service/fetch.py @@ -1,35 +1,29 @@ -import itertools import logging from aviso.framework.views import GnanaView from aviso.settings import sec_context -from config.fm_config import DealConfig -from fm_service import FMView, node_can_access_field, period_is_valid +from config import DealConfig, FMConfig from infra.read import (fetch_boundry, fetch_eligible_nodes_for_segment, fetch_users_nodes_and_root_nodes, get_current_period, get_period_as_of, get_period_begin_end, get_period_range, get_time_context) +from utils.common import cached_property from utils.date_utils import epoch, get_eod, get_nextq_mnem_safe from utils.misc_utils import index_of, try_int -from utils.mongo_reader import get_snapshot_window_feature_data logger = logging.getLogger('gnana.%s' % __name__) -class FMFetch(FMView): - http_method_names = ['get'] - restrict_to_roles = GnanaView.Role.All_Roles +class FMFetch: - validators = [period_is_valid, node_can_access_field] + @cached_property + def config(self): + return FMConfig() - def get(self, request, *args, **kwargs): - try: - req_param = request.GET - except: - req_param = request - - return self._get_metadata(req_param, *args, **kwargs) + @cached_property + def deal_config(self): + return DealConfig() def _get_metadata(self, request, *args, **kwargs): self.api_request = request.get('api_request', None) @@ -115,94 +109,3 @@ def _get_metadata(self, request, *args, **kwargs): fetch_eligible_nodes_for_segment(self.time_context.now_timestamp, segment, period=self.period, boundary_dates=self.boundary_dates)] self.fm_recs = {} - - - def get_field_value(self, period, node, field, segment, timestamp, key="val",**kwargs): - week_start = kwargs.get("start") - week_end = kwargs.get("end") - from tasks.hierarchy.hierarchy_utils import hier_switch - if self.config.fields[field].get("alt_hier"): - alt_hier_flds = self.config.fields[field]['source'] - hiers = itertools.chain(*[alt_hier['hiers'] for alt_hier in [self.config.alt_hiers]]) - alt_nodes = list({hier_switch(hnode, *hier) for hier in hiers for hnode in [node]}) - source1 = self.fm_recs[(period, node, alt_hier_flds[0], segment, timestamp)][key] - alt_nodes.remove(node) - try: - source2 = self.fm_recs[(period, alt_nodes[0], alt_hier_flds[1], segment, timestamp)][key] - source = [source1, source2] - val = eval(self.config.fields[field]['func']) - except: - val = 0 - else: - if week_start and week_end: - val = self._get_value_for_timestamp_range(period, node, field, segment,(week_start, week_end),key=key) - else: - val = self.fm_recs.get((period, node, field, segment, timestamp), {}).get(key, None) - return val - - def _get_value_for_timestamp_range(self, period, node, field, segment, timestamp_range, key): - """ - Retrieve 'val' from self.fm_recs based on a range of timestamps. - - Args: - period (str): The period (e.g., '2025Q4'). - node (str): The node identifier. - field (str): The field name. - segment (str): The segment name. - timestamp_range (tuple): A tuple specifying the start and end of the timestamp range (start, end). - key (str): The key to retrieve from the dictionary. - - Returns: - The value associated with the key within the timestamp range, or None if not found. - """ - start, end = timestamp_range - for rec_key, rec_value in self.fm_recs.items(): - # Unpack the key tuple - try: - rec_period, rec_node, rec_field, rec_segment, rec_timestamp = rec_key - except: - continue - # Check if the key matches the desired attributes and the timestamp falls within the range - if ( - rec_period == period and - rec_node == node and - rec_field == field and - rec_segment == segment and - start <= rec_timestamp <= end - ): - return rec_value.get(key, None) - return None - - def _get_snapshot_feature_data(self): - return get_snapshot_window_feature_data(self.node, self.config) - - def node_email_mapping(self): - from domainmodel.app import User - - db_to_use = User.get_db() - users = db_to_use.findDocuments(User.getCollectionName(), {}) - user_mapping = {} - for item in users: - if not isinstance(item, dict) or 'object' not in item or 'email' not in item['object']: - continue # Skip invalid user entries - user_list = item['object']['roles']['user'] - email = item['object']['email'] - if '@' not in email: - continue # Skip if email is malformed - username, domain = email.split('@') - - if domain == "aviso.com" or domain == "administrative.domain": - continue - for user in user_list: - if 'results' not in user: - continue - node = user[1] # Ensure this is correct, adjust if needed - if not node: - continue # Skip if node is missing or empty - # Initialize self.user_mapping[node] as a list if not already done - if node not in user_mapping: - user_mapping[node] = [] - - if email not in user_mapping[node]: - user_mapping[node].append(email) - return user_mapping diff --git a/fm_service/forecast_schedule.py b/fm_service/forecast_schedule.py index 4137e02..b78d4b2 100644 --- a/fm_service/forecast_schedule.py +++ b/fm_service/forecast_schedule.py @@ -1,343 +1,18 @@ import copy -import datetime -import json import logging -from collections import OrderedDict -import pytz from aviso.settings import sec_context from fm_service.fetch import FMFetch -from infra.read import (fetch_ancestors, fetch_children, fetch_descendants, - fetch_node_to_parent_mapping_and_labels, - get_period_begin_end) +from infra.read import fetch_ancestors, fetch_descendants from infra.write import upload_forecast_schedule -from utils.date_utils import epoch from utils.mongo_reader import get_forecast_schedule logger = logging.getLogger('gnana.%s' % __name__) timezone_map = {'GMT': 'GMT', 'IST': 'Asia/Kolkata', 'PST': 'US/Pacific'} -class FMSchedule(FMFetch): - """ - fetch fm schedule for a node and its direct children - - Parameters: - node -- hierarchy node id 0050000FLN2C9I2 - - Returns: - dict -- fm schedule - # "Global#Global": { -# "heading": "Global", -# "forecastWindow": "lock", //unlock || lock -# "forecastTimestamp": "1723672225549", -# "recurring": true, // true or false -# "unlockPeriod": "month", //month or quarter -# "unlockFreq": "month", // month, dats or weekdays -# "unlockDay": 1, // Between 1-31 -# "unlocktime": "21:00:00", -# "lockPeriod": "month", //month or quarter -# "lockFreq": "month", // month, dats or weekdays -# "lockDay": 1, // Between 1-31 -# "locktime": "21:00:00", -# "timeZone": "pst" -# } - """ - http_method_names = ['get', 'post'] - - def get(self, request, *args, **kwargs): - super(FMSchedule, self).get(request, *args, **kwargs) - self.admin_node = self.config.admin_node - descendants = [(rec['node'], rec['descendants'][0], rec['descendants'][1]) - for rec in - fetch_descendants(self.as_of, [self.node], levels=2, include_children=True, drilldown=True, - period=self.period, boundary_dates=self.boundary_dates)] - - node_children_check = {node: any(node_children) for node, node_children, _ in descendants} - nodes = node_children_check.keys() - timestamp = None - if self.is_versioned: - timestamp, _ = get_period_begin_end(self.period) - _, labels = fetch_node_to_parent_mapping_and_labels(timestamp, - drilldown=True, - period=self.period, - boundary_dates=self.boundary_dates) - fm_schedule = get_forecast_schedule(nodes) - as_of = epoch().as_epoch() - schedules = {} - if not fm_schedule: - for node in nodes: - heading = labels[node] - schedules[node] = { - "heading": heading, - "forecastWindow": self.config.forecast_window_default, - "forecastTimestamp": as_of, - "recurring": False} - records = [] - timeZone = 'US/Pacific' - for sch in fm_schedule: - heading = labels[sch['node_id']] - node_id = sch['node_id'] - recurring = sch['recurring'] - lockPeriod = sch.get('lockPeriod', 'month') - lockFreq = sch.get('lockFreq', 'month') - unlockfreq = sch.get('unlockFreq', 'month') - unlockDay = int(sch.get('unlockDay', 0)) - unlocktime = sch['unlocktime'].split(":") if 'unlocktime' in sch else ["00", "00"] - lockDay = int(sch['lockDay']) if 'lockDay' in sch else 0 - locktime = sch['locktime'].split(":") if 'locktime' in sch else ["00", "00"] - timeZone_original = sch['timeZone'] if 'timeZone' in sch else "PST" - timeZone = timezone_map[timeZone_original.upper()] - lock_on_saved = sch['lock_on'] if 'lock_on' in sch else None - unlock_on_saved = sch['unlock_on'] if 'unlock_on' in sch else None - - if sch['recurring']: - lock_on = None - unlock_on = None - current_year = datetime.datetime.now().year - current_month = datetime.datetime.now().month - if lockPeriod == 'month' or (lockPeriod == 'quarter' and lock_on is None): - unlock_on = datetime.datetime(current_year, current_month, unlockDay, - int(unlocktime[0]), int(unlocktime[1]), 0) - zone = pytz.timezone(timeZone) - unlock_on = zone.localize(unlock_on) - if lockFreq == 'month': - lock_on = datetime.datetime(current_year, current_month, lockDay, - int(locktime[0]), int(locktime[1]), 0) - zone = pytz.timezone(timeZone) - lock_on = zone.localize(lock_on) - elif lockFreq == 'days' or lockFreq == 'weekdays': - lock_on = unlock_on + datetime.timedelta(lockDay) - forecastWindow = "unlock" - forecastTimestamp = epoch(unlock_on).as_epoch() if unlock_on else None - if as_of >= epoch(lock_on).as_epoch(): - forecastWindow = "lock" - forecastTimestamp = epoch(lock_on).as_epoch() - if as_of >= epoch(unlock_on).as_epoch() and epoch(unlock_on).as_epoch() > epoch(lock_on).as_epoch(): - forecastWindow = "unlock" - forecastTimestamp = epoch(unlock_on).as_epoch() - - schedules[sch['node_id']] = { - "heading": heading, - "forecastWindow": forecastWindow, - "forecastTimestamp": forecastTimestamp, - "recurring": recurring, - "lockPeriod": lockPeriod, - "unlockFreq": unlockfreq, - "unlockDay": unlockDay, - "unlocktime": (":").join(unlocktime) if isinstance(unlocktime, list) else unlocktime, - "lockPeriod": lockPeriod, - "lockFreq": lockFreq, - "lockDay": lockDay, - "locktime": (":").join(locktime) if isinstance(locktime, list) else locktime, - "timeZone": timeZone_original - } - record_to_update = {} - if lock_on is not None and lock_on_saved != epoch(lock_on).as_epoch(): - record_to_update['node_id'] = node_id - record_to_update['lock_on'] = epoch(lock_on).as_epoch() - if unlock_on is not None and unlock_on_saved != epoch(unlock_on).as_epoch(): - record_to_update['node_id'] = node_id - record_to_update['unlock_on'] = epoch(unlock_on).as_epoch() - if record_to_update: - records.append(record_to_update) - else: - schedules[sch['node_id']] = { - "heading": heading, - "forecastWindow": sch['status_non_recurring'], - "forecastTimestamp": sch['non_recurring_timestamp'], - "recurring": recurring, - "lockPeriod": lockPeriod, - "unlockFreq": unlockfreq, - "unlockDay": unlockDay, - "unlocktime": (":").join(unlocktime) if isinstance(unlocktime, list) else unlocktime, - "lockPeriod": lockPeriod, - "lockFreq": lockFreq, - "lockDay": lockDay, - "locktime": (":").join(locktime) if isinstance(locktime, list) else locktime, - "timeZone": timeZone_original - } - child_order = [x['node'] for x in fetch_children(self.as_of, [self.node])] - schedules = self.add_admin_forecastwindow(nodes, schedules, timeZone) - schedules_ordered = OrderedDict() - for ordered_node in [self.node] + child_order: - if ordered_node in schedules: - schedules_ordered[ordered_node] = schedules[ordered_node] - else: - schedules_for_node = copy.deepcopy(schedules[self.node]) - schedules_ordered[ordered_node] = schedules_for_node - schedules_for_node['node_id'] = ordered_node - records.append(schedules_for_node) - if records: - upload_forecast_schedule(nodes, records) - return schedules_ordered - - - def add_admin_forecastwindow(self, nodes, schedules, timeZone): - if self.admin_node in nodes: - admin_node_schedule = schedules.get(self.admin_node, {}) - admin_node_forecastwindow = admin_node_schedule.get('forecastWindow', self.config.forecast_window_default) - for node, details in schedules.items(): - schedules[node]['admin_forecastWindow'] = admin_node_forecastwindow - else: - admin_node_schedule = get_forecast_schedule([self.admin_node]) - forecastWindow = self.config.forecast_window_default - for sch in admin_node_schedule: - as_of = epoch().as_epoch() - if sch.get('recurring'): - lockPeriod = sch.get('lockPeriod', 'month') - lockFreq = sch.get('lockFreq', 'month') - unlockDay = int(sch.get('unlockDay', 0)) - unlocktime = sch['unlocktime'].split(":") if 'unlocktime' in sch else ["00", "00"] - lockDay = int(sch['lockDay']) if 'lockDay' in sch else 0 - locktime = sch['locktime'].split(":") if 'locktime' in sch else ["00", "00"] - timeZone_original = sch['timeZone'] if 'timeZone' in sch else "PST" - timeZone = timezone_map[timeZone_original.upper()] - lock_on = None - unlock_on = None - current_year = datetime.datetime.now().year - current_month = datetime.datetime.now().month - if lockPeriod == 'month' or (lockPeriod == 'quarter' and lock_on is None): - unlock_on = datetime.datetime(current_year, current_month, unlockDay, - int(unlocktime[0]), int(unlocktime[1]), 0) - zone = pytz.timezone(timeZone) - unlock_on = zone.localize(unlock_on) - if lockFreq == 'month': - lock_on = datetime.datetime(current_year, current_month, lockDay, - int(locktime[0]), int(locktime[1]), 0) - zone = pytz.timezone(timeZone) - unlock_on = zone.localize(lock_on) - - elif lockFreq == 'days' or lockFreq == 'weekdays': - lock_on = unlock_on + datetime.timedelta(lockDay) - forecastWindow = "unlock" - if as_of >= epoch(lock_on).as_epoch(): - forecastWindow = "lock" - if as_of >= epoch(unlock_on).as_epoch() and epoch(unlock_on).as_epoch() > epoch(lock_on).as_epoch(): - forecastWindow = "unlock" - else: - forecastWindow = sch.get('status_non_recurring', forecastWindow) - for node, details in schedules.items(): - schedules[node]['admin_forecastWindow'] = forecastWindow - return schedules - - def post(self, request, *args, **kwargs): - super(FMSchedule, self).post(request, *args, **kwargs) - post_data = json.loads(request.read()) - recurring = post_data.get('recurring') - current_username = sec_context.get_effective_user().username - timestamp = None - if recurring: - unlockFreq = post_data.get('unlockFreq') - unlockDay = int(post_data.get('unlockDay')) - unlocktime_original = post_data.get('unlocktime') - unlocktime = post_data.get('unlocktime').split(":") - lockPeriod = post_data.get('lockPeriod') - unlockPeriod = post_data.get('unlockPeriod', lockPeriod) - lockFreq = post_data.get('lockFreq') - lockDay = int(post_data.get('lockDay')) - locktime_original = post_data.get('locktime') - locktime = post_data.get('locktime').split(":") - timeZone_original = 'PST' - timeZone = "US/Pacific" - if 'timeZone' in post_data and post_data['timeZone'] in timezone_map: - timeZone = timezone_map[post_data['timeZone'].upper()] - timeZone_original = post_data['timeZone'] - - else: - status = post_data.get('status', None) - timestamp = post_data.get("forecastTimestamp", None) - nodes = [self.node] - self.node_descendants = {node for level in list(fetch_descendants(self.as_of, - [self.node], - levels=20, - drilldown=True))[0]['descendants'] - for node in level.keys()} - self.node_descendants.add(self.node) - nodes = self.node_descendants - nodes.add('Global#!') - nodes.add('Global#not_in_hier') - records = [] - if recurring: - lock_on = None - unlock_on = None - current_year = datetime.datetime.now().year - current_month = datetime.datetime.now().month - if lockPeriod == 'month' or (lockPeriod == 'quarter' and lock_on is None): - unlock_on = datetime.datetime(current_year, current_month, unlockDay, - int(unlocktime[0]), int(unlocktime[1]), 0) - zone = pytz.timezone(timeZone) - unlock_on = zone.localize(unlock_on) - if lockFreq == 'month': - lock_on = datetime.datetime(current_year, current_month, lockDay, - int(locktime[0]), int(locktime[1]), 0) - zone = pytz.timezone(timeZone) - lock_on = zone.localize(lock_on) - elif lockFreq == 'days' or lockFreq == 'weekdays': - lock_on = unlock_on + datetime.timedelta(lockDay) - for node in nodes: - records.append({"recurring": recurring, - "unlockPeriod": unlockPeriod, - "unlockFreq": unlockFreq, - "unlockDay": unlockDay, - "unlocktime": unlocktime_original, - "lockPeriod": lockPeriod, - "lockFreq": lockFreq, - "lockDay": lockDay, - "locktime": locktime_original, - "timeZone": timeZone_original, - "node_id": node, - "lock_on": epoch(lock_on).as_epoch(), - "unlock_on": epoch(unlock_on).as_epoch() - }) - else: - timestamp = timestamp if timestamp is not None else epoch().as_epoch() - for node in nodes: - records.append({"status_non_recurring": status, - "non_recurring_timestamp": timestamp, - "recurring": False, - "node_id": node}) - user_list = self.get_user_list() - msg = "Forecast window is {}ed by your admin to fill - up the forecast data".format(status) - from infra import send_notifications - notif = [] - user_list_final = {} - for username, u in user_list.items(): - if username == current_username: - continue - user_nodes_ = [] - try: - role = u['roles'].get('user', None) - if role: - for i in role: - if i[0] == 'results': - if i[1] != '!': - user_nodes_.append(i[1]) - except: - pass - user_nodes_in_descendants = False - for user_node in user_nodes_: - if user_node in self.node_descendants: - user_nodes_in_descendants = True - if user_nodes_in_descendants: - user_list_final[username] = u - for username, rec in user_list_final.items(): - notif.append({'recipients': [rec['userId']], - "email": rec['email'], - "tenant_name": sec_context.name, - "nudge_name": "lock_unlock_status", - "lock_unlock_message": msg, - "skip_header_footer": True}) - send_notifications('lock_unlock_status', notif, epoch().as_xldate(), - tonkean=False) - if records: - return upload_forecast_schedule(nodes, records) - - def get_user_list(self): - from utils.collab_connector_utils import get_dict_of_all_fm_users - all_fm_users = get_dict_of_all_fm_users(consider_admin=False) - return all_fm_users +class FMScheduleBase(FMFetch): def update_fm_schedule(self): self.node_descendants = {node for level in list(fetch_descendants(self.as_of, @@ -370,7 +45,7 @@ def update_fm_schedule(self): upload_forecast_schedule(new_nodes, records) -class FMScheduleClass(FMSchedule): +class FMSchedule(FMScheduleBase): def __init__(self, period, nodes_having_changes, *args, **kwargs): self.metrics = logger.new_metrics() self.debug = kwargs.get("debug", False) diff --git a/infra/__init__.py b/infra/__init__.py index f3706fe..e69de29 100644 --- a/infra/__init__.py +++ b/infra/__init__.py @@ -1,852 +0,0 @@ -import logging -import random -import sys -import time - -from eventbus import EventBus - -from pymongo import IndexModel, DESCENDING -from aviso.settings import sec_context, global_cache -from aviso.events import create_payload -from redis.exceptions import ConnectionError, BusyLoadingError, TimeoutError -EVENT_BUS = EventBus() -logger = logging.getLogger('gnana.%s' % __name__) - - - - -FM_COLL = 'fm_data' -EDW_DATA = 'edw_data' -EDW_PROCESS_UPDATE = 'edw_process_update' -DEALS_COLL = 'deals' -NEW_DEALS_COLL = 'deals_new' -QUARTER_COLL = 'quarterinfo' -FAVS_COLL = 'favorites' -AUDIT_COLL = 'audit_log' - -DEALS_CI_COLL = 'deals_ci' -AI_DEALS_COLL = 'ai_deals' -FIELDS_COLL = 'fields' -DEALS_HISTORY_COLL = 'deals_history' -COMMS_COLL = 'comments' -COMMS_CRR_COLL = 'comments_crr' -MILESTONES_COLL = 'milestones' -FILTS_COLL = 'filters' -ACCOUNT_FILTS_COLL = 'account_filters' -DTFO_COLL = 'dtfo' -DLF_COLL = 'dlf' -PIPE_COLL = 'pipe_dev' -DLF_FCST_COLL = 'dlf_fcst' -INSIGHT_COLL = 'insights' -INSIGHTS_HISTORY_COLL = 'insights_history' -ACTIONABLE_INSIGHTS = 'actionable_insights' -WB_CREATE_COLL = 'wb_creates' -PIPELINE_PROJECTION_COL = 'pipeline_projection' -LEADERBOARD_DATA = 'leaderboard_calculated_data' -ADAPTIVE_METRICS_DATA = 'adaptive_metrics_calculated_data' -TOTALS_CACHED = 'filter_totals' -GBM_DEALS = "gbm_deals_results" -NUDGE_STATS = "nudge_stats" -AI_FORECAST_DIFF = "ai_forecast_diff" -GBM_CRR_COLL = 'gbm_crr_results' -GBM_CRR_COLL_INACTIVE = 'gbm_crr_results_inactive' -PLAN_OVERVIEW = "plan_overview" -PLAN_OBJECTIVES = "plan_objectives" -MEDDIC_COLL = 'meddic' -DLF_PREV_TIMESTAMP = 'dlf_prev_timestamp' -SHARED_USERS_COLL = 'shared_users' -AUTH_TOKENS_COLL = 'auth_tokens' -MEDDIC_FILES_COLL = 'meddic_files' - - -# --special pivot snapshot api collections -- -SNAPSHOT_CACHE_COLL = "accounts_snapshot_data" # primary -SNAPSHOT_CACHE_PREV_COLL = "accounts_snapshot_data_prev" # prev cache -SNAPSHOT_NODE_FORECAST_COLL = "accounts_node_level_forecast" # primary -# -x-special pivot snapshot api collections -x- - -ACCOUNTS_COLL = 'account_details' -DEALS_UE_FIELDS = "deals_ue_fields" -DEALS_STAGE_MAP = "deals_stage_map" -DEAL_CHANGES_CACHE = "deal_changes_cache" -PREMEETING_SCHEDULE_COL = 'premeeting_schedule' -FM_DATA = "fm_data" -INSIGHT_NUGGS= 'insight_nuggs' -DEALS_VLOOKUP = 'deals_vlookup' -ACCOUNTS_FILTERS_TOTAL_COLL = 'account_filter_totals' -ACCOUNTS_GRID_COLL = 'account_grid' - -CURRENCY_CONV_COLL = 'currency_conversion_rates' -ACCOUNT_PLAN = 'account_plan' - -PERIOD_OPP = 'period_opp' -ROLLUP_NODES = 'rollup_nodes' - -fm_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('field', DESCENDING), - ('segment', DESCENDING), - ('timestamp', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('field', DESCENDING), - ('segment', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('how', DESCENDING)]), - ] - -FM_LATEST_COLL = 'fm_latest_dr' - -fm_latest_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('field', DESCENDING), - ('segment', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('how', DESCENDING)]), - ] - - -FM_LATEST_DATA_COLL = 'fm_latest_data' - -FM_FORECAST_INSIGHTS_COLL = 'forecast_insights' - -fm_forecast_insights_indexes = [IndexModel([ - ('timestamp', DESCENDING), - ('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING)], unique=True), - IndexModel([ - ('timestamp', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING)]), - ] - -FM_FORECAST_EXPLANATION_INSIGHTS_COLL = 'forecast_explanation_insights' - -fm_forecast_explanation_insights_indexes = [IndexModel([ - ('timestamp', DESCENDING), - ('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING)], unique=True), - IndexModel([ - ('timestamp', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING)]), - ] - -SNAPSHOT_COLL = 'snapshot_data' - -snapshot_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('last_updated_time', DESCENDING)]), - ] - -SNAPSHOT_HIST_COLL = 'snapshot_historical_data' - -snapshot_hist_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('as_of_date', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('as_of_date', DESCENDING), - ('last_updated_time', DESCENDING)]), - ] - -PERFORMANCE_DASHBOARD_COLL = 'performance_dashboard' - -performance_dashboard_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('field', DESCENDING), - ('timestamp', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('field', DESCENDING)]) - ] - -ROLE_SUFFIX = 'roles' -SNAPSHOT_ROLE_COLL = '_'.join([SNAPSHOT_COLL, ROLE_SUFFIX]) - -snapshot_role_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('role', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('role', DESCENDING), - ('last_updated_time', DESCENDING)]), - ] - -SNAPSHOT_HIST_ROLE_COLL = '_'.join([SNAPSHOT_HIST_COLL, ROLE_SUFFIX]) - -snapshot_hist_role_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('role', DESCENDING), - ('as_of_date', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('role', DESCENDING), - ('as_of_date', DESCENDING), - ('last_updated_time', DESCENDING)]), - ] - -MOBILE_SNAPSHOT_COLL = 'mobile_snapshot_data' - -mobile_snapshot_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('last_updated_time', DESCENDING)]), - ] - -MOBILE_SNAPSHOT_ROLE_COLL = MOBILE_SNAPSHOT_COLL + ROLE_SUFFIX - -mobile_snapshot_role_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('role', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('role', DESCENDING), - ('last_updated_time', DESCENDING)]), - ] -accounts_snapshot_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment_id', DESCENDING)], unique=True)] - -NEXTQ_COLL = 'nextq_dashboard' - -nextq_dashboard_indexes = [IndexModel([('node', DESCENDING), - ('quarter', DESCENDING)]), - IndexModel([('node', DESCENDING), - ('quarter', DESCENDING), - ('timestamp', DESCENDING)], unique=True), - IndexModel([('node', DESCENDING), - ('timestamp', DESCENDING)]), - ] - -WATERFALL_COLL = 'waterfall' -#TODO: Commenting for now as Index we need to check which is optimised to use. -# waterfall_indexes = [IndexModel([('node', DESCENDING), -# ('quarter', DESCENDING)]), -# IndexModel([('node', DESCENDING), -# ('quarter', DESCENDING), -# ('timestamp', DESCENDING)], unique=True), -# IndexModel([('node', DESCENDING), -# ('timestamp', DESCENDING)]), -# ] - -WATERFALL_HISTORY_COLL = 'waterfall_history' - - -WEEKLY_FORECAST_FM_COLL = 'weekly_forecast_fm_data' - -weekly_forecast_fm_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('segment', DESCENDING), - ('field', DESCENDING) - ], unique=True) - ] - -WEEKLY_FORECAST_TREND_COLL = 'weekly_forecast_trend_data' - -WEEKLY_FORECAST_EXPORT_COLL = 'weekly_forecast_fm_export_data' - -WEEKLY_EDW_DATA = 'weekly_edw_data' -WEEKLY_EDW_PROCESS_STATUS = 'weekly_edw_process_status' -WEEKLY_EDW_PROCESS_START_TIME = 'weekly_edw_process_start_time' -WEEKLY_FORECAST_EXPORT_ALL = 'weekly_forecast_export_all_{}' - - -FORECAST_SCHEDULE_COLL = 'fm_schedule' -FORECAST_UNLOCK_REQUESTS = 'fm_unlock_requests' -USER_LEVEL_SCHEDULE = 'user_schedule' -CRM_SCHEDULE = 'crm_schedule' -ADMIN_MAPPING = 'admin_mapping' -EXPORT_ALL = 'export_all' - -export_all_indexes = [IndexModel([('export_date', DESCENDING)], unique=True)] - - -user_schedule_indexes = [IndexModel([('user_id', DESCENDING), - ('node_id', DESCENDING)], unique=True)] - -HIER_COLL = 'hierarchy' -DRILLDOWN_COLL = 'drilldowns' - -HIER_LEADS_COLL = 'hierarchy_leads' -DRILLDOWN_LEADS_COLL = 'drilldowns_leads' - -hier_indexes = [IndexModel([('node', DESCENDING), - ('parent', DESCENDING), - ('from', DESCENDING), - ('to', DESCENDING)], unique=True), - IndexModel([('node', DESCENDING), - ('hidden_from', DESCENDING), - ('hidden_to', DESCENDING), - ('from', DESCENDING), - ('to', DESCENDING)]), - IndexModel([('parent', DESCENDING), - ('hidden_from', DESCENDING), - ('hidden_to', DESCENDING), - ('from', DESCENDING), - ('to', DESCENDING)]), - IndexModel([('from', DESCENDING), - ('to', DESCENDING)]) - ] - -drilldown_indexes = [IndexModel([('node', DESCENDING), - ('parent', DESCENDING), - ('from', DESCENDING), - ('to', DESCENDING)], unique=True), - IndexModel([('node', DESCENDING), - ('hidden_from', DESCENDING), - ('hidden_to', DESCENDING), - ('from', DESCENDING), - ('to', DESCENDING)]), - IndexModel([('parent', DESCENDING), - ('hidden_from', DESCENDING), - ('hidden_to', DESCENDING), - ('from', DESCENDING), - ('to', DESCENDING)]), - IndexModel([('normal_segs', DESCENDING), - ('from', DESCENDING), - ('to', DESCENDING)]), - IndexModel([('from', DESCENDING), - ('to', DESCENDING)]) - ] - -def valid_hier_records(records, _ignore_versioning=False): - required_fields = ['node', 'parent', 'from', 'to', 'label'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - - -def valid_drilldown_records(records, _ignore_versioning=False): - required_fields = ['node', 'parent', 'from', 'to', 'label'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - - -HIER_SERVICE_INDEXES = {HIER_COLL: hier_indexes, - DRILLDOWN_COLL: drilldown_indexes, - HIER_LEADS_COLL: hier_indexes, - DRILLDOWN_LEADS_COLL: drilldown_indexes - } - -HIER_SERVICE_VALIDATORS = {HIER_COLL: valid_hier_records, - DRILLDOWN_COLL: valid_drilldown_records, - HIER_LEADS_COLL: valid_hier_records, - DRILLDOWN_LEADS_COLL: valid_drilldown_records - } - - - -gbm_crr_indexes = [IndexModel([('monthly_period', DESCENDING), - ('RPM_ID', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('RPM_ID', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('RPM_ID', DESCENDING), - ('AccountID', DESCENDING)]), - IndexModel([('BUYING_PROGRAM', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('BUYING_PROGRAM', DESCENDING)]), - IndexModel([('monthly_period', DESCENDING), - ('BUYING_PROGRAM', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('RPM_ID', DESCENDING), - ('OwnerID', DESCENDING)] ), - IndexModel([('monthly_period', DESCENDING), - ('RPM_ID', DESCENDING), - ('__id__', DESCENDING)]), - IndexModel([('monthly_period', DESCENDING), - ('__segs', DESCENDING), - ('forecast', DESCENDING)]), - IndexModel([('monthly_period', DESCENDING), - ('__segs', DESCENDING), - ('forecast', DESCENDING), - ('CRR_BAND_DESCR', DESCENDING)])] - -deal_changes_cache_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('since', DESCENDING), - ('segment', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('node', DESCENDING), - ('since', DESCENDING), - ('segment', DESCENDING), - ('last_updated_time', DESCENDING)])] - -accounts_node_level_forecast_indexes = [IndexModel([('period', DESCENDING), - ('node_key', DESCENDING)], unique=True)] - -pipeline_projection_indexes = [IndexModel([('node', DESCENDING), - ('quarter', DESCENDING)]), - IndexModel([('node', DESCENDING), - ('quarter', DESCENDING), - ('timestamp', DESCENDING)], unique=True), - IndexModel([('node', DESCENDING), - ('timestamp', DESCENDING)]), - IndexModel([('timestamp', DESCENDING)]), - IndexModel([('quarter', DESCENDING), - ('timestamp', DESCENDING)]), - ] - -account_details_indexes = [IndexModel([('account_id', DESCENDING)], unique=True)] -meddic_indexes = [IndexModel([('opp_id', DESCENDING)], unique=True)] - -shared_users_indexes = [IndexModel([('opp_id',DESCENDING)], unique=True)] -auth_tokens_indexes = [IndexModel([('token', DESCENDING)], unique=True), - IndexModel([('opp_id',DESCENDING),('email',DESCENDING)])] -meddic_file_indexes = [IndexModel([('opp_id',DESCENDING),('qId',DESCENDING)]), - IndexModel([('qId',DESCENDING),('opp_id',DESCENDING),('file_name',DESCENDING)]), - IndexModel([('s3_key',DESCENDING)],unique=True)] - - -plan_overview_indexes = [IndexModel([('account_id', DESCENDING)], unique=True)] - -plan_objectives_indexes = [IndexModel([('account_id', DESCENDING)], unique=True)] - -account_filter_totals_indexes = [IndexModel([('node', DESCENDING),('period', DESCENDING)], unique=True)] -account_grid_indexes = [IndexModel([('node', DESCENDING),('period', DESCENDING)], unique=True)] - - -deals_ci_indexes = [IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('hierarchy_list', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('drilldown_list', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('drilldown_list', DESCENDING), - ('is_deleted', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('hierarchy_list', DESCENDING), - ('is_deleted', DESCENDING)]), - ] - -deals_indexes = [IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('hierarchy_list', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('drilldown_list', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('drilldown_list', DESCENDING), - ('is_deleted', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('hierarchy_list', DESCENDING), - ('is_deleted', DESCENDING)]), - IndexModel([('Amount', DESCENDING)]), - IndexModel([('drilldown_list', DESCENDING),('created_date_adj',DESCENDING)])] - -wb_creates_indexes = [IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('timestamp', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('node', DESCENDING)])] - -deals_history_indexes = [IndexModel([('as_of', DESCENDING), - ('period', DESCENDING), - ('opp_id', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('hierarchy_list', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('drilldown_list', DESCENDING), - ('update_date', DESCENDING)])] - -insights_indexes = [IndexModel([('opp_id', DESCENDING), - ('period', DESCENDING), - ('day_timestamp', DESCENDING)], unique=True), - IndexModel([('opp_id', DESCENDING), - ('day_timestamp', DESCENDING)])] - -insights_history_indexes = [IndexModel([('type', DESCENDING), - ('period', DESCENDING), - ('as_of', DESCENDING), - ('opp_id', DESCENDING)], unique=True), - IndexModel([('opp_id', DESCENDING), - ('as_of', DESCENDING)])] - -actionable_insights_indexes = [IndexModel([('opp_id', DESCENDING), - ('period', DESCENDING), - ('node', DESCENDING)], unique=True), - IndexModel([('opp_id', DESCENDING), - ('node', DESCENDING)]), - IndexModel([('opp_id', DESCENDING)])] - -filter_totals_indexes = [IndexModel([('node', DESCENDING), - ('period', DESCENDING), - ('segment', DESCENDING)], unique=True), - IndexModel([('node', DESCENDING), - ('period', DESCENDING)]), - IndexModel([('period', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('stale', DESCENDING)]), - ] - -gbm_deals_results_indexes = [IndexModel([('opp_id', DESCENDING), - ('timestamp', DESCENDING)], unique=True), - IndexModel([('timestamp', DESCENDING), - ('drilldown_list', DESCENDING)]), - IndexModel([('timestamp', DESCENDING)])] - -ai_forecast_diff_indexes = [IndexModel([('opp_id', DESCENDING), - ('begin', DESCENDING), - ('end', DESCENDING), - ('node', DESCENDING), - ('segment_id', DESCENDING)], unique=True), - IndexModel([('begin', DESCENDING), - ('end', DESCENDING), - ('node', DESCENDING), - ('segment_id', DESCENDING)]), - IndexModel([('begin', DESCENDING), - ('end', DESCENDING), - ('node', DESCENDING), - ('segment_id', DESCENDING), - ('diff.trans_category', DESCENDING)])] - -# Nudge history storage -# nudge_history_indexes = [IndexModel([('opp_id', DESCENDING), -# ('period', DESCENDING), -# ('as_of', DESCENDING)], unique=True), -# IndexModel([('opp_id', DESCENDING), -# ('as_of', DESCENDING)])] - -deals_migrator = {'uniqueness_fields': ['period', 'opp_id'], - 'tiebreaker_fields': ['update_date']} - - -def valid_deal_records(records): - required_fields = ['period', 'close_period', 'opp_id', 'hierarchy_list', 'update_date', 'drilldown_list'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - - -favs_indexes = [IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('user', DESCENDING)], unique=True), - IndexModel([('user', DESCENDING), - ('fav', DESCENDING)])] - - -def valid_fav_records(records): - required_fields = ['period', 'opp_id', 'user', 'fav'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - - -comms_indexes = [IndexModel([('timestamp', DESCENDING), - ('opp_id', DESCENDING), - ('user', DESCENDING)], unique=True), - IndexModel([('opp_id', DESCENDING), - ('node', DESCENDING)]), - IndexModel([('opp_id', DESCENDING), - ('node', DESCENDING), - ('period', DESCENDING)])] - -comms_crr_indexes = [IndexModel([('timestamp', DESCENDING), - ('opp_id', DESCENDING), - ('user', DESCENDING)], unique=True), - IndexModel([('opp_id', DESCENDING), - ('node', DESCENDING)]), - IndexModel([('opp_id', DESCENDING), - ('node', DESCENDING), - ('period', DESCENDING)])] - - -def valid_comm_records(records): - required_fields = ['opp_id', 'user', 'node', 'comment', 'timestamp'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - - -def valid_insights_records(records): - required_fields = ['opp_id', 'period', 'day_timestamp', 'timestamp', 'win_prob'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - - -def valid_insights_history_records(records): - required_fields = ['type', 'period',] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - - -filts_indexes = [IndexModel([('filter_id', DESCENDING)], unique=True), - IndexModel([('is_default', DESCENDING)]), - IndexModel([('is_open', DESCENDING)])] - -account_filters_indexes = [IndexModel([('filter_id', DESCENDING)], unique=True), - IndexModel([('is_default', DESCENDING)]), - IndexModel([('is_open', DESCENDING)])] - -dtfo_indexes = [IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('close_period', DESCENDING), - ('as_of', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('as_of', DESCENDING), - ('hierarchy_list_hash', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('as_of', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('hierarchy_list', DESCENDING), - ('as_of', DESCENDING), - ('update_date', DESCENDING)])] - - -pipe_dev_indexes = [IndexModel([('period', DESCENDING), - ('as_of', DESCENDING), - ('segment',DESCENDING), - ('node', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('as_of', DESCENDING), - ('node', DESCENDING)])] - -dlf_indexes = [IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('node', DESCENDING), - ('field', DESCENDING), - ('timestamp', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING), - ('node', DESCENDING), - ('field', DESCENDING)])] - -rollup_nodes_indexes = [IndexModel([('node', DESCENDING)], unique=True)] - - -def valid_dlf_records(records): - required_fields = ['period', 'opp_id', 'node', 'field', 'timestamp', 'option', 'amount', 'deal_amount'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - - - -dlf_fcst_indexes = [IndexModel([('period', DESCENDING), - ('opp_id', DESCENDING)], unique=True), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('hierarchy_list', DESCENDING), - ('update_date', DESCENDING)]), - IndexModel([('period', DESCENDING), - ('close_period', DESCENDING), - ('drilldown_list', DESCENDING), - ('update_date', DESCENDING)])] - -def valid_dlf_fcst_records(records): - required_fields = ['period', 'opp_id', 'close_period'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - -def valid_pp_records(records): - required_fields = ['node', 'quarter', 'timestamp', 'data'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - - -leaderboard_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING)], unique=True), - IndexModel([('node', DESCENDING), - ('period', DESCENDING)])] - -adaptive_metrics_indexes = [IndexModel([('period', DESCENDING), - ('node', DESCENDING)], unique=True), - IndexModel([('node', DESCENDING), - ('period', DESCENDING)])] - -mileston_indexes = [ - IndexModel([('opp_id', DESCENDING)]), -] - - -def valid_mileston_records(records): - required_fields = ['opp_id', 'items', 'name', 'start_date', 'end_date', 'color'] - for record in records: - if any((field not in record for field in required_fields)): - return False - return True - -DEAL_SERVICE_INDEXES = {DEALS_COLL: deals_indexes, - NEW_DEALS_COLL: deals_indexes, - DEALS_CI_COLL: deals_ci_indexes, - AI_DEALS_COLL: deals_indexes, - DEALS_HISTORY_COLL: deals_history_indexes, - FAVS_COLL: favs_indexes, - COMMS_COLL: comms_indexes, - COMMS_CRR_COLL: comms_crr_indexes, - FILTS_COLL: filts_indexes, - ACCOUNT_FILTS_COLL:account_filters_indexes, - DTFO_COLL: dtfo_indexes, - PIPE_COLL: pipe_dev_indexes, - DLF_COLL: dlf_indexes, - DLF_FCST_COLL: dlf_fcst_indexes, - INSIGHT_COLL: insights_indexes, - INSIGHTS_HISTORY_COLL: insights_history_indexes, - WB_CREATE_COLL: wb_creates_indexes, - LEADERBOARD_DATA: leaderboard_indexes, - ADAPTIVE_METRICS_DATA: adaptive_metrics_indexes, - TOTALS_CACHED: filter_totals_indexes, - PIPELINE_PROJECTION_COL: pipeline_projection_indexes, - MILESTONES_COLL: mileston_indexes, - ACCOUNTS_COLL: account_details_indexes, - GBM_CRR_COLL: gbm_crr_indexes, - GBM_CRR_COLL_INACTIVE: gbm_crr_indexes, - PLAN_OVERVIEW: plan_overview_indexes, - PLAN_OBJECTIVES: plan_objectives_indexes, - MEDDIC_COLL: meddic_indexes, - ACCOUNTS_FILTERS_TOTAL_COLL:account_filter_totals_indexes, - ROLLUP_NODES: rollup_nodes_indexes, - ACCOUNTS_GRID_COLL:account_grid_indexes, - SHARED_USERS_COLL: shared_users_indexes, - AUTH_TOKENS_COLL: auth_tokens_indexes, - MEDDIC_FILES_COLL:meddic_file_indexes - } - - -DEAL_SERVICE_VALIDATORS = {DEALS_COLL: valid_deal_records, - NEW_DEALS_COLL: valid_deal_records, - AI_DEALS_COLL: valid_deal_records, - FAVS_COLL: valid_fav_records, - COMMS_COLL: valid_comm_records, - DLF_COLL: valid_dlf_records, - DLF_FCST_COLL: valid_dlf_fcst_records, - INSIGHT_COLL: valid_insights_records, - INSIGHTS_HISTORY_COLL: valid_insights_history_records, - PIPELINE_PROJECTION_COL: valid_pp_records, - MILESTONES_COLL: valid_mileston_records, - } - -DEAL_SERVICE_MIGRATORS = {DEALS_COLL: deals_migrator, - NEW_DEALS_COLL: deals_migrator, - AI_DEALS_COLL: deals_migrator - } - - -CRR_PIVOT = 'CRR' - - -def is_redis_ready(max_retries=6, retry_initial_delay=0.1, backoff_factor=2): - """Checks if the Redis server is ready to accept commands. - - Args: - max_retries: Maximum number of connection attempts (defaults to 5). - retry_initial_delay: Initial delay between retries in seconds (defaults to 0.1). - backoff_factor: Factor by which to increase delay between retries (defaults to 2). - - Returns: - True if the connection is successful and server is responsive, False otherwise. - """ - delay = retry_initial_delay - for attempt in range(1, max_retries): - try: - client = global_cache - if client.info().get('loading') != '1': - client.ping() - return True - except (BusyLoadingError, ConnectionError, TimeoutError) as e: - logger.warning("Redis connection issue on attempt {}: {}".format(attempt, e)) - - delay = min(delay * backoff_factor + random.uniform(0, delay), 30) - time.sleep(delay) - - logger.error("Failed to connect to Redis after {} retries.".format(max_retries)) - return False - - -def send_notifications(service_name, notification_data, notification_time=None, tonkean=False): - """ - Send notifications to an Event Bus - - Args: - service_name (str): Name of the service requesting notification. - notification_data (list): List of dictionaries containing notification information. - notification_time (datetime.datetime, optional): Time to set the notification flag (default: None). - tonkean (bool, optional): Flag to include tokenized payload (default: False). - """ - - tenant_details = sec_context.details - if notification_time: - tenant_details.set_flag('notification', service_name, notification_time) - tenant_details.save() - - published_count = 0 - total_size = sum(sys.getsizeof(notification) for notification in notification_data) - - if is_redis_ready(): - for _notification in notification_data: - EVENT_BUS.publish(service_name, **create_payload(**_notification)) - published_count += 1 - - if tonkean: - try: - EVENT_BUS.publish('tonkean_payload', **create_payload(**_notification)) - return True - except Exception as e: - logger.exception("Unable to send tonkean_payload %s" % e.msg) - logger.info("Published an event {} to eventbus for notifying {} users, total_size={}".format(service_name, published_count, total_size)) - return True - else: - logger.exception("Failed sending {} {} notifications due to redis server busy state, contact administrator asap.".format( - len(notification_data), service_name)) diff --git a/infra/constants.py b/infra/constants.py new file mode 100644 index 0000000..f7c5236 --- /dev/null +++ b/infra/constants.py @@ -0,0 +1,642 @@ +from pymongo import IndexModel, DESCENDING + + +FM_COLL = 'fm_data' +EDW_DATA = 'edw_data' +EDW_PROCESS_UPDATE = 'edw_process_update' +DEALS_COLL = 'deals' +NEW_DEALS_COLL = 'deals_new' +QUARTER_COLL = 'quarterinfo' +FAVS_COLL = 'favorites' +AUDIT_COLL = 'audit_log' + +DEALS_CI_COLL = 'deals_ci' +AI_DEALS_COLL = 'ai_deals' +FIELDS_COLL = 'fields' +DEALS_HISTORY_COLL = 'deals_history' +COMMS_COLL = 'comments' +COMMS_CRR_COLL = 'comments_crr' +MILESTONES_COLL = 'milestones' +FILTS_COLL = 'filters' +ACCOUNT_FILTS_COLL = 'account_filters' +DTFO_COLL = 'dtfo' +DLF_COLL = 'dlf' +PIPE_COLL = 'pipe_dev' +DLF_FCST_COLL = 'dlf_fcst' +INSIGHT_COLL = 'insights' +INSIGHTS_HISTORY_COLL = 'insights_history' +ACTIONABLE_INSIGHTS = 'actionable_insights' +WB_CREATE_COLL = 'wb_creates' +PIPELINE_PROJECTION_COL = 'pipeline_projection' +LEADERBOARD_DATA = 'leaderboard_calculated_data' +ADAPTIVE_METRICS_DATA = 'adaptive_metrics_calculated_data' +TOTALS_CACHED = 'filter_totals' +GBM_DEALS = "gbm_deals_results" +NUDGE_STATS = "nudge_stats" +AI_FORECAST_DIFF = "ai_forecast_diff" +GBM_CRR_COLL = 'gbm_crr_results' +GBM_CRR_COLL_INACTIVE = 'gbm_crr_results_inactive' +PLAN_OVERVIEW = "plan_overview" +PLAN_OBJECTIVES = "plan_objectives" +MEDDIC_COLL = 'meddic' +DLF_PREV_TIMESTAMP = 'dlf_prev_timestamp' +SHARED_USERS_COLL = 'shared_users' +AUTH_TOKENS_COLL = 'auth_tokens' +MEDDIC_FILES_COLL = 'meddic_files' + + +# --special pivot snapshot api collections -- +SNAPSHOT_CACHE_COLL = "accounts_snapshot_data" # primary +SNAPSHOT_CACHE_PREV_COLL = "accounts_snapshot_data_prev" # prev cache +SNAPSHOT_NODE_FORECAST_COLL = "accounts_node_level_forecast" # primary +# -x-special pivot snapshot api collections -x- + +ACCOUNTS_COLL = 'account_details' +DEALS_UE_FIELDS = "deals_ue_fields" +DEALS_STAGE_MAP = "deals_stage_map" +DEAL_CHANGES_CACHE = "deal_changes_cache" +PREMEETING_SCHEDULE_COL = 'premeeting_schedule' +FM_DATA = "fm_data" +INSIGHT_NUGGS= 'insight_nuggs' +DEALS_VLOOKUP = 'deals_vlookup' +ACCOUNTS_FILTERS_TOTAL_COLL = 'account_filter_totals' +ACCOUNTS_GRID_COLL = 'account_grid' + +CURRENCY_CONV_COLL = 'currency_conversion_rates' +ACCOUNT_PLAN = 'account_plan' + +PERIOD_OPP = 'period_opp' +ROLLUP_NODES = 'rollup_nodes' + +fm_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('field', DESCENDING), + ('segment', DESCENDING), + ('timestamp', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('field', DESCENDING), + ('segment', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('how', DESCENDING)]), + ] + +FM_LATEST_COLL = 'fm_latest_dr' + +fm_latest_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('field', DESCENDING), + ('segment', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('how', DESCENDING)]), + ] + + +FM_LATEST_DATA_COLL = 'fm_latest_data' + +FM_FORECAST_INSIGHTS_COLL = 'forecast_insights' + +fm_forecast_insights_indexes = [IndexModel([ + ('timestamp', DESCENDING), + ('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING)], unique=True), + IndexModel([ + ('timestamp', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING)]), + ] + +FM_FORECAST_EXPLANATION_INSIGHTS_COLL = 'forecast_explanation_insights' + +fm_forecast_explanation_insights_indexes = [IndexModel([ + ('timestamp', DESCENDING), + ('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING)], unique=True), + IndexModel([ + ('timestamp', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING)]), + ] + +SNAPSHOT_COLL = 'snapshot_data' + +snapshot_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING), + ('last_updated_time', DESCENDING)]), + ] + +SNAPSHOT_HIST_COLL = 'snapshot_historical_data' + +snapshot_hist_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING), + ('as_of_date', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING), + ('as_of_date', DESCENDING), + ('last_updated_time', DESCENDING)]), + ] + +PERFORMANCE_DASHBOARD_COLL = 'performance_dashboard' + +performance_dashboard_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('field', DESCENDING), + ('timestamp', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('field', DESCENDING)]) + ] + +ROLE_SUFFIX = 'roles' +SNAPSHOT_ROLE_COLL = '_'.join([SNAPSHOT_COLL, ROLE_SUFFIX]) + +snapshot_role_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING), + ('role', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING), + ('role', DESCENDING), + ('last_updated_time', DESCENDING)]), + ] + +SNAPSHOT_HIST_ROLE_COLL = '_'.join([SNAPSHOT_HIST_COLL, ROLE_SUFFIX]) + +snapshot_hist_role_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING), + ('role', DESCENDING), + ('as_of_date', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING), + ('role', DESCENDING), + ('as_of_date', DESCENDING), + ('last_updated_time', DESCENDING)]), + ] + +MOBILE_SNAPSHOT_COLL = 'mobile_snapshot_data' + +mobile_snapshot_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING), + ('last_updated_time', DESCENDING)]), + ] + +MOBILE_SNAPSHOT_ROLE_COLL = MOBILE_SNAPSHOT_COLL + ROLE_SUFFIX + +accounts_snapshot_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment_id', DESCENDING)], unique=True)] + +NEXTQ_COLL = 'nextq_dashboard' + +nextq_dashboard_indexes = [IndexModel([('node', DESCENDING), + ('quarter', DESCENDING)]), + IndexModel([('node', DESCENDING), + ('quarter', DESCENDING), + ('timestamp', DESCENDING)], unique=True), + IndexModel([('node', DESCENDING), + ('timestamp', DESCENDING)]), + ] + +WATERFALL_COLL = 'waterfall' +#TODO: Commenting for now as Index we need to check which is optimised to use. +# waterfall_indexes = [IndexModel([('node', DESCENDING), +# ('quarter', DESCENDING)]), +# IndexModel([('node', DESCENDING), +# ('quarter', DESCENDING), +# ('timestamp', DESCENDING)], unique=True), +# IndexModel([('node', DESCENDING), +# ('timestamp', DESCENDING)]), +# ] + +WATERFALL_HISTORY_COLL = 'waterfall_history' + + +WEEKLY_FORECAST_FM_COLL = 'weekly_forecast_fm_data' + +weekly_forecast_fm_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('segment', DESCENDING), + ('field', DESCENDING) + ], unique=True) + ] + +WEEKLY_FORECAST_TREND_COLL = 'weekly_forecast_trend_data' + +WEEKLY_FORECAST_EXPORT_COLL = 'weekly_forecast_fm_export_data' + +WEEKLY_EDW_DATA = 'weekly_edw_data' +WEEKLY_EDW_PROCESS_STATUS = 'weekly_edw_process_status' +WEEKLY_EDW_PROCESS_START_TIME = 'weekly_edw_process_start_time' +WEEKLY_FORECAST_EXPORT_ALL = 'weekly_forecast_export_all_{}' + + +FORECAST_SCHEDULE_COLL = 'fm_schedule' +FORECAST_UNLOCK_REQUESTS = 'fm_unlock_requests' +USER_LEVEL_SCHEDULE = 'user_schedule' +CRM_SCHEDULE = 'crm_schedule' +ADMIN_MAPPING = 'admin_mapping' +EXPORT_ALL = 'export_all' + +export_all_indexes = [IndexModel([('export_date', DESCENDING)], unique=True)] + + +user_schedule_indexes = [IndexModel([('user_id', DESCENDING), + ('node_id', DESCENDING)], unique=True)] + +HIER_COLL = 'hierarchy' +DRILLDOWN_COLL = 'drilldowns' + +HIER_LEADS_COLL = 'hierarchy_leads' +DRILLDOWN_LEADS_COLL = 'drilldowns_leads' + +hier_indexes = [IndexModel([('node', DESCENDING), + ('parent', DESCENDING), + ('from', DESCENDING), + ('to', DESCENDING)], unique=True), + IndexModel([('node', DESCENDING), + ('hidden_from', DESCENDING), + ('hidden_to', DESCENDING), + ('from', DESCENDING), + ('to', DESCENDING)]), + IndexModel([('parent', DESCENDING), + ('hidden_from', DESCENDING), + ('hidden_to', DESCENDING), + ('from', DESCENDING), + ('to', DESCENDING)]), + IndexModel([('from', DESCENDING), + ('to', DESCENDING)]) + ] + +drilldown_indexes = [IndexModel([('node', DESCENDING), + ('parent', DESCENDING), + ('from', DESCENDING), + ('to', DESCENDING)], unique=True), + IndexModel([('node', DESCENDING), + ('hidden_from', DESCENDING), + ('hidden_to', DESCENDING), + ('from', DESCENDING), + ('to', DESCENDING)]), + IndexModel([('parent', DESCENDING), + ('hidden_from', DESCENDING), + ('hidden_to', DESCENDING), + ('from', DESCENDING), + ('to', DESCENDING)]), + IndexModel([('normal_segs', DESCENDING), + ('from', DESCENDING), + ('to', DESCENDING)]), + IndexModel([('from', DESCENDING), + ('to', DESCENDING)]) + ] +HIER_SERVICE_INDEXES = {HIER_COLL: hier_indexes, + DRILLDOWN_COLL: drilldown_indexes, + HIER_LEADS_COLL: hier_indexes, + DRILLDOWN_LEADS_COLL: drilldown_indexes + } + + + +gbm_crr_indexes = [IndexModel([('monthly_period', DESCENDING), + ('RPM_ID', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('RPM_ID', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('RPM_ID', DESCENDING), + ('AccountID', DESCENDING)]), + IndexModel([('BUYING_PROGRAM', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('BUYING_PROGRAM', DESCENDING)]), + IndexModel([('monthly_period', DESCENDING), + ('BUYING_PROGRAM', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('RPM_ID', DESCENDING), + ('OwnerID', DESCENDING)] ), + IndexModel([('monthly_period', DESCENDING), + ('RPM_ID', DESCENDING), + ('__id__', DESCENDING)]), + IndexModel([('monthly_period', DESCENDING), + ('__segs', DESCENDING), + ('forecast', DESCENDING)]), + IndexModel([('monthly_period', DESCENDING), + ('__segs', DESCENDING), + ('forecast', DESCENDING), + ('CRR_BAND_DESCR', DESCENDING)])] + +deal_changes_cache_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('since', DESCENDING), + ('segment', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('node', DESCENDING), + ('since', DESCENDING), + ('segment', DESCENDING), + ('last_updated_time', DESCENDING)])] + +accounts_node_level_forecast_indexes = [IndexModel([('period', DESCENDING), + ('node_key', DESCENDING)], unique=True)] + +pipeline_projection_indexes = [IndexModel([('node', DESCENDING), + ('quarter', DESCENDING)]), + IndexModel([('node', DESCENDING), + ('quarter', DESCENDING), + ('timestamp', DESCENDING)], unique=True), + IndexModel([('node', DESCENDING), + ('timestamp', DESCENDING)]), + IndexModel([('timestamp', DESCENDING)]), + IndexModel([('quarter', DESCENDING), + ('timestamp', DESCENDING)]), + ] + +account_details_indexes = [IndexModel([('account_id', DESCENDING)], unique=True)] +meddic_indexes = [IndexModel([('opp_id', DESCENDING)], unique=True)] + +shared_users_indexes = [IndexModel([('opp_id',DESCENDING)], unique=True)] +auth_tokens_indexes = [IndexModel([('token', DESCENDING)], unique=True), + IndexModel([('opp_id',DESCENDING),('email',DESCENDING)])] +meddic_file_indexes = [IndexModel([('opp_id',DESCENDING),('qId',DESCENDING)]), + IndexModel([('qId',DESCENDING),('opp_id',DESCENDING),('file_name',DESCENDING)]), + IndexModel([('s3_key',DESCENDING)],unique=True)] + + +plan_overview_indexes = [IndexModel([('account_id', DESCENDING)], unique=True)] + +plan_objectives_indexes = [IndexModel([('account_id', DESCENDING)], unique=True)] + +account_filter_totals_indexes = [IndexModel([('node', DESCENDING),('period', DESCENDING)], unique=True)] +account_grid_indexes = [IndexModel([('node', DESCENDING),('period', DESCENDING)], unique=True)] + + +deals_ci_indexes = [IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('hierarchy_list', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('drilldown_list', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('drilldown_list', DESCENDING), + ('is_deleted', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('hierarchy_list', DESCENDING), + ('is_deleted', DESCENDING)]), + ] + +deals_indexes = [IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('hierarchy_list', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('drilldown_list', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('drilldown_list', DESCENDING), + ('is_deleted', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('hierarchy_list', DESCENDING), + ('is_deleted', DESCENDING)]), + IndexModel([('Amount', DESCENDING)]), + IndexModel([('drilldown_list', DESCENDING),('created_date_adj',DESCENDING)])] + +wb_creates_indexes = [IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('timestamp', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('node', DESCENDING)])] + +deals_history_indexes = [IndexModel([('as_of', DESCENDING), + ('period', DESCENDING), + ('opp_id', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('hierarchy_list', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('drilldown_list', DESCENDING), + ('update_date', DESCENDING)])] + +insights_indexes = [IndexModel([('opp_id', DESCENDING), + ('period', DESCENDING), + ('day_timestamp', DESCENDING)], unique=True), + IndexModel([('opp_id', DESCENDING), + ('day_timestamp', DESCENDING)])] + +insights_history_indexes = [IndexModel([('type', DESCENDING), + ('period', DESCENDING), + ('as_of', DESCENDING), + ('opp_id', DESCENDING)], unique=True), + IndexModel([('opp_id', DESCENDING), + ('as_of', DESCENDING)])] + +actionable_insights_indexes = [IndexModel([('opp_id', DESCENDING), + ('period', DESCENDING), + ('node', DESCENDING)], unique=True), + IndexModel([('opp_id', DESCENDING), + ('node', DESCENDING)]), + IndexModel([('opp_id', DESCENDING)])] + +filter_totals_indexes = [IndexModel([('node', DESCENDING), + ('period', DESCENDING), + ('segment', DESCENDING)], unique=True), + IndexModel([('node', DESCENDING), + ('period', DESCENDING)]), + IndexModel([('period', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('stale', DESCENDING)]), + ] + +gbm_deals_results_indexes = [IndexModel([('opp_id', DESCENDING), + ('timestamp', DESCENDING)], unique=True), + IndexModel([('timestamp', DESCENDING), + ('drilldown_list', DESCENDING)]), + IndexModel([('timestamp', DESCENDING)])] + +ai_forecast_diff_indexes = [IndexModel([('opp_id', DESCENDING), + ('begin', DESCENDING), + ('end', DESCENDING), + ('node', DESCENDING), + ('segment_id', DESCENDING)], unique=True), + IndexModel([('begin', DESCENDING), + ('end', DESCENDING), + ('node', DESCENDING), + ('segment_id', DESCENDING)]), + IndexModel([('begin', DESCENDING), + ('end', DESCENDING), + ('node', DESCENDING), + ('segment_id', DESCENDING), + ('diff.trans_category', DESCENDING)])] + +deals_migrator = {'uniqueness_fields': ['period', 'opp_id'], + 'tiebreaker_fields': ['update_date']} +favs_indexes = [IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('user', DESCENDING)], unique=True), + IndexModel([('user', DESCENDING), + ('fav', DESCENDING)])] +comms_indexes = [IndexModel([('timestamp', DESCENDING), + ('opp_id', DESCENDING), + ('user', DESCENDING)], unique=True), + IndexModel([('opp_id', DESCENDING), + ('node', DESCENDING)]), + IndexModel([('opp_id', DESCENDING), + ('node', DESCENDING), + ('period', DESCENDING)])] + +comms_crr_indexes = [IndexModel([('timestamp', DESCENDING), + ('opp_id', DESCENDING), + ('user', DESCENDING)], unique=True), + IndexModel([('opp_id', DESCENDING), + ('node', DESCENDING)]), + IndexModel([('opp_id', DESCENDING), + ('node', DESCENDING), + ('period', DESCENDING)])] + + +filts_indexes = [IndexModel([('filter_id', DESCENDING)], unique=True), + IndexModel([('is_default', DESCENDING)]), + IndexModel([('is_open', DESCENDING)])] + +account_filters_indexes = [IndexModel([('filter_id', DESCENDING)], unique=True), + IndexModel([('is_default', DESCENDING)]), + IndexModel([('is_open', DESCENDING)])] + +dtfo_indexes = [IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('close_period', DESCENDING), + ('as_of', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('as_of', DESCENDING), + ('hierarchy_list_hash', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('as_of', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('hierarchy_list', DESCENDING), + ('as_of', DESCENDING), + ('update_date', DESCENDING)])] + + +pipe_dev_indexes = [IndexModel([('period', DESCENDING), + ('as_of', DESCENDING), + ('segment',DESCENDING), + ('node', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('as_of', DESCENDING), + ('node', DESCENDING)])] + +dlf_indexes = [IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('node', DESCENDING), + ('field', DESCENDING), + ('timestamp', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING), + ('node', DESCENDING), + ('field', DESCENDING)])] + +rollup_nodes_indexes = [IndexModel([('node', DESCENDING)], unique=True)] +dlf_fcst_indexes = [IndexModel([('period', DESCENDING), + ('opp_id', DESCENDING)], unique=True), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('hierarchy_list', DESCENDING), + ('update_date', DESCENDING)]), + IndexModel([('period', DESCENDING), + ('close_period', DESCENDING), + ('drilldown_list', DESCENDING), + ('update_date', DESCENDING)])] + + +leaderboard_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING)], unique=True), + IndexModel([('node', DESCENDING), + ('period', DESCENDING)])] + +adaptive_metrics_indexes = [IndexModel([('period', DESCENDING), + ('node', DESCENDING)], unique=True), + IndexModel([('node', DESCENDING), + ('period', DESCENDING)])] + +mileston_indexes = [ + IndexModel([('opp_id', DESCENDING)]), +] + +DEAL_SERVICE_INDEXES = {DEALS_COLL: deals_indexes, + NEW_DEALS_COLL: deals_indexes, + DEALS_CI_COLL: deals_ci_indexes, + AI_DEALS_COLL: deals_indexes, + DEALS_HISTORY_COLL: deals_history_indexes, + FAVS_COLL: favs_indexes, + COMMS_COLL: comms_indexes, + COMMS_CRR_COLL: comms_crr_indexes, + FILTS_COLL: filts_indexes, + ACCOUNT_FILTS_COLL:account_filters_indexes, + DTFO_COLL: dtfo_indexes, + PIPE_COLL: pipe_dev_indexes, + DLF_COLL: dlf_indexes, + DLF_FCST_COLL: dlf_fcst_indexes, + INSIGHT_COLL: insights_indexes, + INSIGHTS_HISTORY_COLL: insights_history_indexes, + WB_CREATE_COLL: wb_creates_indexes, + LEADERBOARD_DATA: leaderboard_indexes, + ADAPTIVE_METRICS_DATA: adaptive_metrics_indexes, + TOTALS_CACHED: filter_totals_indexes, + PIPELINE_PROJECTION_COL: pipeline_projection_indexes, + MILESTONES_COLL: mileston_indexes, + ACCOUNTS_COLL: account_details_indexes, + GBM_CRR_COLL: gbm_crr_indexes, + GBM_CRR_COLL_INACTIVE: gbm_crr_indexes, + PLAN_OVERVIEW: plan_overview_indexes, + PLAN_OBJECTIVES: plan_objectives_indexes, + MEDDIC_COLL: meddic_indexes, + ACCOUNTS_FILTERS_TOTAL_COLL:account_filter_totals_indexes, + ROLLUP_NODES: rollup_nodes_indexes, + ACCOUNTS_GRID_COLL:account_grid_indexes, + SHARED_USERS_COLL: shared_users_indexes, + AUTH_TOKENS_COLL: auth_tokens_indexes, + MEDDIC_FILES_COLL:meddic_file_indexes + } + +DEAL_SERVICE_MIGRATORS = {DEALS_COLL: deals_migrator, + NEW_DEALS_COLL: deals_migrator, + AI_DEALS_COLL: deals_migrator + } + + +CRR_PIVOT = 'CRR' diff --git a/infra/fetch_helper.py b/infra/fetch_helper.py deleted file mode 100644 index bd071ac..0000000 --- a/infra/fetch_helper.py +++ /dev/null @@ -1,8 +0,0 @@ - -def modify_hint_field(close_periods, hint_field): - """ - Modify MongoDB hint field based on close periods. - """ - if all(['W' in close_period for close_period in close_periods]) and hint_field: - hint_field = hint_field.replace('close_period', 'weekly_period') - return hint_field diff --git a/infra/filters.py b/infra/filters.py index a9bc3c1..5285d8d 100644 --- a/infra/filters.py +++ b/infra/filters.py @@ -9,44 +9,7 @@ from utils.date_utils import epoch, rng_from_prd_str from utils.misc_utils import get_nested, inrange, try_float, try_index -from . import AUDIT_COLL, FILTS_COLL - - -def _sanitize_string(string): - # let us never speak of this - set_pattern, set_replace = r"\'(set)\(\[.*?\]\)\'", r"(?<=set\()\[.*?\]" - if re.findall(set_pattern, string): - string = re.sub(set_pattern, re.findall(set_replace, string)[0], string) - list_pattern, list_replace = r"\'\[.*?\]\'", r"\[.*?\]" - if re.findall(list_pattern, string): - string = re.sub(list_pattern, re.findall(list_replace, string)[0], string) - return string - -def contextualize_filter(filt, - node, - favs, - period, - ): - """ - inject app context into filter - - Arguments: - filt {dict} -- mongodb filter description {'opp_id': {'$in': '%(favs)s'}} - node {str} -- hierarchy node '0050000FLN2C9I2' - favs {set} -- opp_ids of favorited deals {'0060b00000nRtMS', } - period {str} -- period mnemonic '2020Q2' - - Returns: - dict -- mongodb filter - """ - # puke - #adding a replace for node to add escape characters for single quotes in a node value - #ex: "Global#Becky O'Brien#!" - str_filt = str(filt).replace("%s", "*pers*").replace("%(", "*perb*").replace("%", "*perc*").replace("*pers*", "%s").replace("*perb*", "%(") - resp = eval(_sanitize_string(str_filt % {'node': node.replace("'","\\'") if node else None, 'favs': favs, 'period': period})) - if "*perc*" in str_filt: - resp = eval(str(resp).replace("*perc*", "%")) - return resp +from .constants import AUDIT_COLL, FILTS_COLL NEGATION_MAP = {'$gt': '$lte', '$gte': '$lt', '$lte': '$gt', '$lt': '$gte'} @@ -274,244 +237,6 @@ def fetch_all_filters(config, return all_filters -def fetch_user_levels_filters(config, - filter_type='mongo', - db=None, - collection=FILTS_COLL - ): - """ - fetch all the user level filters from the db - - Arguments: - config {DealsConfig} -- instance of DealsConfig ...? - - Keyword Arguments: - grid_only {bool} -- only return filters that display in deals grid - (default: {False}) - filter_type {str} -- flag to switch return signature (default: {'mongo'}) - if mongo, returns mongo db filter criteria - else, returns python func(deal, favs, dlf) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - (pass in to avoid overhead when fetching many times) - - Returns: - dict -- {(filt ids): (filter name, mongo db filter criteria OR python func)} - """ - all_filters = {} - filters_collection = db[collection] if db else sec_context.tenant_db[collection] - final_criteria = {"user_level": True, "user": sec_context.login_user_name, "$or": [{"deleted": {"$exists": False}}, {"deleted": False}]} - filters = filters_collection.find(final_criteria, {'desc': 1, 'name': 1, 'filter_id': 1, "tenant_level":1, "user_level":1, "group_name":1}) - for filt in filters: - org_filt = deepcopy(filt) - tenant_level = False if filt.get("user_level") else True - all_filters[filt['filter_id']] = (filt['name'], parse_filters(filt['desc'], config, filter_type), org_filt['desc'], tenant_level) - - return all_filters - - -def fetch_filters_by_filter_id(config, - filter_type='mongo', - db=None, - filter_ids=None, - collection=FILTS_COLL, - is_pivot_special=False - ): - """ - fetch the filter details by filter_id from the db - - Arguments: - config {DealsConfig} -- instance of DealsConfig ...? - - Keyword Arguments: - grid_only {bool} -- only return filters that display in deals grid - (default: {False}) - filter_type {str} -- flag to switch return signature (default: {'mongo'}) - if mongo, returns mongo db filter criteria - else, returns python func(deal, favs, dlf) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - (pass in to avoid overhead when fetching many times) - - Returns: - dict -- {(filt ids): (filter name, mongo db filter criteria OR python func)} - """ - all_filters = {} - filters_collection = db[collection] if db else sec_context.tenant_db[collection] - if not isinstance(filter_ids, list): - filter_ids = [filter_ids] - criteria = {'filter_id': {"$in": filter_ids}} - filters = filters_collection.find(criteria, {'desc': 1, 'name': 1, 'filter_id': 1, "tenant_level":1, "user_level":1, "group_name":1}) - for filt in filters: - org_filt = deepcopy(filt) - tenant_level = False if filt.get("user_level") else True - all_filters[filt['filter_id']] = (filt['name'], parse_filters(filt['desc'], config, filter_type, is_pivot_special=is_pivot_special), org_filt['desc'], tenant_level) - - return all_filters - -def fetch_filter_names(filter_ids, - db=None, - collection=FILTS_COLL - ): - """ - fetch human readable name and ids of filters - - Arguments: - filter_ids {list} -- filter ids to compose into a single AND filter ['id1', 'id2'] - - Keyword Arguments: - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - (pass in to avoid overhead when fetching many times) - - Returns: - list -- [{'name': name, 'filter_id': filter_id}] - """ - filters_collection = db[collection] if db else sec_context.tenant_db[collection] - if filter_ids: - criteria = {'filter_id': {'$in': filter_ids}} - else: - criteria = {'is_default': True} - return filters_collection.find(criteria, {'name': 1, 'filter_id': 1, 'is_segment_filter': 1}) - - -def upload_user_level_filter(filter_data, db=None,collection=FILTS_COLL): - action = filter_data.pop("action", "create") - filters_collection = db[collection] if db else sec_context.tenant_db[collection] - errors = [] - if filter_data['name'].strip() == '': - errors.append({'filter': filter_data, 'error': 'You cannot save an empty name'}) - return errors - try: - _valid_filter(filter_data['desc']) - record = {'name': filter_data['name'], - 'desc': filter_data['desc'], - 'user_level': True, - 'user': sec_context.login_user_name - } - if action == "create": - user_filter = filters_collection.find({"name": record['name'], "user": sec_context.login_user_name}) - if user_filter.count() > 0: - if user_filter[0] and user_filter[0].get("deleted"): - filters_collection.update_one({"filter_id": user_filter[0]['filter_id'], "user": sec_context.login_user_name}, - {'$set': record, '$unset': {"deleted": 1, "in_grid": 1}}) - else: - return [{"filter": filter_data, "error": "Filter Already Exists with the name `%s`"%record['name']}] - else: - record["filter_id"] = str(uuid.uuid4()) - filters_collection.insert(record) - else: - filter_id = filter_data['filter_id'] - filters_collection.update_one({"filter_id": filter_id, "user": sec_context.login_user_name}, {'$set': record}) - except Exception as _err: - errors.append({'filter': filter_data, 'error': _err}) - return errors - - -def upload_filter(name, - desc, - filter_id=None, - is_default=False, - is_open=False, - in_grid=True, - db=None, - is_segment_filter=False, - custom_limit=None, - collection=FILTS_COLL, - segment_id=None - ): - """ - upload a filter to db - - Arguments: - name {str} -- display name of filter 'Open Deals' - desc {list} -- our insane filter language format [{'op': 'has', 'key': 'amt'}] - - Keyword Arguments: - filter_id {str} -- unique id for filter (default: {None}) - is_default {bool} -- makes filter default (default: {False}) - is_open {bool} -- makes filter open (default: {False}) - in_grid {bool} -- make filter appear in deals grid (default: {True}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - """ - filters_collection = db[collection] if db else sec_context.tenant_db[collection] - audit_collection = db[AUDIT_COLL] if db else sec_context.tenant_db[AUDIT_COLL] - _valid_filter(desc) - filter_id = filter_id or str(uuid.uuid4()) - - record = {'name': name, - 'desc': desc, - 'filter_id': filter_id, - 'is_default': is_default, - 'in_grid': in_grid, - 'is_open': is_open, - 'segment_id': segment_id, - 'is_segment_filter': is_segment_filter} - if custom_limit is not None: - record['custom_limit'] = int(custom_limit) - - filters_collection.update_one({'filter_id': filter_id}, {'$set': record}, upsert=True) - - # audit = {'service': 'deal_svc', - # 'action': 'save filter', - # 'user': sec_context.login_user_name, - # 'timestamp': epoch().as_epoch(), - # 'coll': FILTS_COLL, - # 'record': record} - # audit_collection.insert_one(audit) - - -def delete_filter(filter_id, - db=None, - user_level=False, - collection=FILTS_COLL, - ): - """ - delete filter from db - - Arguments: - filter_id {str} -- unique id of filter 'id1' - - Keyword Arguments: - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - """ - filters_collection = db[collection] if db else sec_context.tenant_db[collection] - audit_collection = db[AUDIT_COLL] if db else sec_context.tenant_db[AUDIT_COLL] - if user_level: - record = filters_collection.find_one_and_update({'filter_id': filter_id, 'user_level': True, 'user': sec_context.login_user_name}, - {'$set': {'deleted': True, - 'in_grid': False}}) - else: - record = filters_collection.find_one_and_update({'filter_id': filter_id, "$or": [{"user_level": {"$exists": False}}, {"user_level": False}]}, - {'$set': {'deleted': True, - 'in_grid': False}}) - # audit = {'service': 'deal_svc', - # 'action': 'delete filter', - # 'user': sec_context.login_user_name, - # 'timestamp': epoch().as_epoch(), - # 'coll': FILTS_COLL, - # 'record': record} - # audit_collection.insert_one(audit) - - -def negate_filter(desc): - """ - negate a filter - - Arguments: - desc {list} -- filter description to negate [{'op': 'has', 'key': 'amt'}] - - Returns: - list -- negated filter - """ - new_filt = [clause.copy() for clause in desc] - for clause in new_filt: - clause['negate'] = not clause.pop('negate', False) - return new_filt - - def parse_filters(filters, config, filter_type='mongo', safe_mode=True, hier_aware=True, root_node=None, is_pivot_special=False): """ @@ -1037,43 +762,5 @@ def _valid_filter(desc): return True -def passes_new_filter(deal_data, fltr_desc=None, favs_set=None, dlf_svc=None): - """ Filter description: for example - [{ - "key": "as_of_Amount_USD", - "op": "range", - "val": [500, 2000, false, true] - }] - To pass the filter, a deal must match the filter on the appropriate field, - including one of its prefixes. - """ - if not fltr_desc: - return True - - for filt in fltr_desc: - if "or" in filt or "and" in filt: - result = [] - filter_desc = filt["or"] if "or" in filt else filt["and"] - for filtnew in filter_desc: - filter_exp = OP_MAP[filtnew['op']] - if filter_exp is not None: - if not filter_exp(filtnew, 'raw')(deal_data, favs_set, dlf_svc): - result.append(False) - else: - result.append(True) - if "or" in filt: - return True if any(result) else False - else: - return True if all(result) else False - filter_exp = OP_MAP[filt['op']] - if filter_exp is not None: - if not filter_exp(filt, 'raw')(deal_data, favs_set, dlf_svc): - return False - else: - return True - - return True - - class FilterError(Exception): pass diff --git a/infra/mongo_utils.py b/infra/mongo_utils.py index 7a89f58..fb3c25e 100644 --- a/infra/mongo_utils.py +++ b/infra/mongo_utils.py @@ -3,30 +3,6 @@ from aviso.settings import sec_context - -def _determine_period_field_name(close_periods): - """ - Determines the MongoDB field name for filtering based on the type of periods provided. - - Args: - close_periods (list): List of periods to analyze (e.g., weekly or non-weekly). - - Returns: - str: The field name ('weekly_period' if all periods contain 'W', else 'close_period'). - - Raises: - TypeError: If periods is not a list of strings. - ValueError: If periods is empty. - """ - if not isinstance(close_periods, list): - raise TypeError("close_periods must be a list") - if not close_periods: - raise ValueError("close_periods list cannot be empty") - if not all(isinstance(period, str) for period in close_periods): - raise TypeError("all elements in close_periods must be strings") - - return 'weekly_period' if all('W' in period for period in close_periods) else 'close_period' - def create_collection_checksum(collection_name, db=None, ): diff --git a/infra/read.py b/infra/read.py index 279477a..d436cc4 100644 --- a/infra/read.py +++ b/infra/read.py @@ -1,42 +1,23 @@ import copy import logging -import re -import threading -from collections import OrderedDict, namedtuple -from datetime import datetime +from collections import namedtuple from functools import wraps -from itertools import product import numpy as np -import pandas as pd -from aviso.framework.metric_logger import NOOPMetricSet from aviso.settings import sec_context -from bson.objectid import ObjectId -from dateutil.rrule import WEEKLY, rrule - -from config.fm_config import DealConfig -from infra import (DEALS_COLL, FAVS_COLL, GBM_CRR_COLL, NEW_DEALS_COLL, - QUARTER_COLL) -from infra.fetch_helper import modify_hint_field -from infra.filters import contextualize_filter, parse_filters -from infra.mongo_utils import \ - _determine_period_field_name as get_key_for_close_periods +from infra.constants import DEALS_COLL, NEW_DEALS_COLL from tasks.hierarchy.hierarchy_utils import get_user_permissions from utils.date_utils import (current_period, epoch, get_bom, get_boq, get_bow, - get_eod, get_nested_with_placeholder, + get_eod, get_nextq_mnem_safe, get_prevq_mnem_safe, get_week_ago, get_yest, monthly_periods, next_period, period_details, period_details_range, period_rng_from_mnem, - prev_period, prev_periods_allowed_in_deals, - weekly_periods) -from utils.misc_utils import (contextualize_field, flatten_to_list, get_nested, - is_lead_service, iter_chunks, merge_dicts, - try_float, use_df_for_dlf_rollup, - use_dlf_fcst_coll_for_rollups) -from utils.relativedelta import relativedelta + prev_period, prev_periods_allowed_in_deals) +from utils.misc_utils import (flatten_to_list, + is_lead_service, merge_dicts) -from . import DRILLDOWN_COLL, DRILLDOWN_LEADS_COLL, HIER_COLL, HIER_LEADS_COLL +from infra.constants import DRILLDOWN_COLL, DRILLDOWN_LEADS_COLL, HIER_COLL, HIER_LEADS_COLL logger = logging.getLogger('gnana.%s' % __name__) @@ -100,7 +81,7 @@ def period_has_forecast(period, # TODO need to understand and implement accordingly for weekly forecast if 'W' in period: return not future_period - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() if not future_period: prd_begin, prd_end = [epoch(x) for x in period_rng_from_mnem(period)] @@ -130,27 +111,6 @@ def relative_period(now, return 'f' if now < begin else 'h' -def future_period(period, - config=None): - """ - check if period is in the future - - Arguments: - period {str} -- period mnemonic '2020Q2' - config {PeriodsConfig} -- instance of PeriodsConfig ...? - - Returns: - bool -- period is future - """ - # totes cheating to get across services - from config.periods_config import PeriodsConfig - config = config if config is not None else PeriodsConfig() - - prd_begin, prd_end = [epoch(x) for x in period_rng_from_mnem(period)] - now = get_now(config) - return relative_period(now, prd_begin, prd_end) == 'f' - - def component_periods(period, config=None, period_type='Q' ): @@ -165,94 +125,13 @@ def component_periods(period, list -- list of component mnemonic """ # totes cheating to get across services - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() if period[-2] != 'Q' or not config.monthly_fm: return [period] return [mnemonic for (mnemonic, _beg_dt, _end_dt) in monthly_periods(period, period_type)] -def active_periods(config, quarter_editable=False, component_periods_editable=False, - past_quarter_editable=0, future_quarter_editable=1, week_period_editability='W'): - """ - get active forecasting periods (current, and future) - - Arguments: - config {PeriodsConfig} -- instance of PeriodsConfig ...? - - Returns: - list -- list of current and future periods - """ - now_dt = get_now(config).as_datetime() - custom_quarter_editable = config.is_custom_quarter_editable - curr_mnem, next_mnem = current_period(now_dt).mnemonic, next_period(now_dt).mnemonic - active_qtrs = [] - if past_quarter_editable or future_quarter_editable > 1: - avail_prds = get_periods_editable(config, - future_qtr_editable=future_quarter_editable, - past_qtr_editable=past_quarter_editable) - active_qtrs = [prd for prd in avail_prds] - if not config.monthly_fm: - if active_qtrs: - return active_qtrs - return [curr_mnem, next_mnem] - - weeks = [] - if config.weekly_fm: - if week_period_editability == 'Q': - weekly_begin_boundary = current_period(now_dt).begin - elif week_period_editability == 'M': - weekly_begin_boundary = get_current_month(config).begin - else: - weekly_begin_boundary = get_current_week(config).begin - weeks = [week_prd.mnemonic for prd in (active_qtrs if active_qtrs else [curr_mnem, next_mnem]) - for week_prd in weekly_periods(prd) if week_prd.begin >= weekly_begin_boundary] - - if component_periods_editable: - if active_qtrs: - months = [mnth_prd.mnemonic for prd in active_qtrs for mnth_prd in monthly_periods(prd)] - return months + (active_qtrs if (quarter_editable or custom_quarter_editable) else []) + weeks - return [mnem for period in [curr_mnem, next_mnem] for (mnem, _, end_dt) - in monthly_periods(period)] + ([curr_mnem, next_mnem] if (quarter_editable or custom_quarter_editable) else []) + weeks - - return [mnem for period in [curr_mnem, next_mnem] for (mnem, _, end_dt) - in monthly_periods(period) if end_dt >= now_dt] + ([curr_mnem, next_mnem] if (quarter_editable or custom_quarter_editable) else []) + weeks - - -def period_is_active(period, - config=None, - quarter_editable=False, - component_periods_editable=False, - past_quarter_editable=0, - future_quarter_editable=1, - week_period_editability='W' - ): - """ - determine if period is active - - Arguments: - period {str} -- period mnemonic '2020Q2' - config {PeriodsConfig} -- instance of PeriodsConfig ...? - - Optional: - quarter_editable -- If fm_monthly tenant has quarter editability option True/False Deafults to False - Value and impacts for forecast monthly tenants(True : Considers Current and Next Quarters and thier months as active periods, - False : Considers only Months of current and next quarters but not quarters themselves) - - Returns: - bool -- True if active, False if not - """ - # totes cheating to get across services - quarter_editable = quarter_editable and 'Q' in period - from config.periods_config import PeriodsConfig - config = config if config is not None else PeriodsConfig() - return period in active_periods(config, quarter_editable=quarter_editable, - component_periods_editable=component_periods_editable, - past_quarter_editable=past_quarter_editable, - future_quarter_editable=future_quarter_editable, - week_period_editability=week_period_editability) - - def get_now(config=None): """ get time of now in app @@ -264,7 +143,7 @@ def get_now(config=None): epoch -- epoch of apps current time """ - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() if config.use_sys_time: @@ -276,112 +155,6 @@ def get_now(config=None): except: return epoch() - -def validate_period(period, - config=None): - """ - check mnemonic is valid - - Arguments: - period {str} -- period mnemonic '2020Q2' - config {PeriodsConfig} -- instance of PeriodsConfig ...? - - Returns: - bool -- True if valid - """ - # Returns true if the period is a valid period name. - # Should start with 4 numbers, should end with a number. - # totes cheating to get across services - #from config.periods_config import PeriodsConfig - #config = config if config is not None else PeriodsConfig() - - try: - int(period[:4]) - if len(period) == 9: - return 'w' in period.lower() - return len(period) == 6 or len(period) == 4 - except ValueError: - return False - - -def special_pivot_latest_refresh_date(config): - try: - col = sec_context.tenant_db[GBM_CRR_COLL] - last_executed_timestamp = list(col.find({'updated_by': 'gbm_crr_results_task'})) - if last_executed_timestamp: - return last_executed_timestamp[0].get('last_execution_time') - return get_now(config).as_epoch() - except Exception as _err: - logger.warning("Exception in special_pivot_latest_refresh_date {}".format(_err)) - return get_now(config).as_epoch() - -def latest_refresh_timestamp_leads(config): - if config.use_sys_time: - try: - #This is the time when the chipotle was completed - return int(sec_context.details.get_flag('leads', 'last_execution_time')) - except (ValueError, TypeError): - pass - - return get_now(config).as_epoch() - -def latest_refresh_date(config): - """ - get epoch timestamp of last model run - - Arguments: - config {PeriodsConfig} -- instance of PeriodsConfig ...? - - Returns: - int -- epoch timestamp - """ - - if config.use_sys_time: - try: - #This is the time when the chipotle was completed - return int(sec_context.details.get_flag('molecule_status', - 'rtfm', {}).get('last_execution_time', sec_context.details.get_flag('molecule_status', - 'rtfm', {}).get('ui_display_time'))) - except (ValueError, TypeError): - pass - - return get_now(config).as_epoch() - - -def latest_eoq_refresh_date(config, period, now_dt, relative_period): - """ - get epoch timestamp of last model run - - Arguments: - config {PeriodsConfig} -- instance of PeriodsConfig ...? - - Returns: - int -- epoch timestamp - """ - - if config.use_sys_time: - if relative_period == 'h': - last_minute_of_quarter = render_period(current_period(period.begin), now_dt, config).get('end') - try: - # This is the time when the eoq was completed - return int(sec_context.details.get_flag('molecule_status', - 'rtfm', {}).get(period.mnemonic + '_eoq_time', - last_minute_of_quarter)) - except: - pass - else: - prev_prd = prev_period(period.begin) - last_minute_of_quarter = render_period(prev_prd, now_dt, config).get('end') - try: - #This is the time when the chipotle was completed - return int(sec_context.details.get_flag('molecule_status', - 'rtfm', {}).get('ui_display_time', last_minute_of_quarter)) - except: - pass - - return get_now(config).as_epoch() - - def get_available_periods(config): """ available periods in app @@ -420,29 +193,8 @@ def make_cumulative_close_periods(close_period): return close_periods - -def get_periods_editable(config=None, future_qtr_editable=1, past_qtr_editable=0): - """ - get_periods_editable in app - - Arguments: - config {PeriodsConfig} -- instance of PeriodsConfig ...? - - Returns: - set -- {period mnemonics} - """ - from config.periods_config import PeriodsConfig - config = config if config is not None else PeriodsConfig() - - now_dt = get_now(config).as_datetime() - required_periods = {current_period(now_dt)[0]} - if past_qtr_editable: - required_periods |= {prev_period(now_dt, skip=i+1)[0] for i in range(past_qtr_editable)} - required_periods |= {next_period(now_dt, skip=i+1)[0] for i in range(future_qtr_editable)} - return required_periods - def get_quarter_period(period): - from config.periods_config import PeriodsConfig + from config import PeriodsConfig period_config = PeriodsConfig() available_periods = get_available_quarters_and_months(period_config) if period and len(period) == 9 and 'w' in period.lower(): @@ -478,7 +230,7 @@ def get_period_and_close_periods(period, tuple -- (period, [close periods]) """ # totes cheating to get across services - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() prd_begin, prd_end = [epoch(x) for x in period_rng_from_mnem(period)] @@ -533,29 +285,6 @@ def get_period_and_close_periods(period, return fetch_period, [period] - -def get_crr_period_and_monthly_period(period, is_dlf=False): - from config.periods_config import PeriodsConfig - period_config = PeriodsConfig() - available_periods = get_available_quarters_and_months(period_config) - period_and_monthly_period = [] - if period: - if 'Q' in list(period): - quarter_months = available_periods.get(period) or [] - if is_dlf and quarter_months: - return period, [max(quarter_months)] - return period, quarter_months - elif len(period) == 4: - quarters = [period + 'Q' + str(x) for x in range(1, 5)] - for quarter in quarters: - quarter_months = available_periods.get(quarter) or [] - period_and_monthly_period.append((quarter, quarter_months)) - else: - for quarter, months in available_periods.items(): - if period in months: - return quarter, [period] - return period_and_monthly_period - def get_period_and_close_periods_yearly(period, out_of_period=False, config=None, @@ -574,7 +303,7 @@ def get_period_and_close_periods_yearly(period, Returns: tuple -- (period, [close periods]) """ - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() prd_begin, prd_end = [epoch(x) for x in period_rng_from_mnem(period)] @@ -631,7 +360,7 @@ def get_period_and_component_periods(period, tuple -- (period, [close periods]) """ # totes cheating to get across services - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() if len(period) == 4: @@ -678,67 +407,6 @@ def get_period_range(period, period_range[-1] = prd_end # HACK to get end of period timestamp at midnight return period_range -def get_period_range_with_day_of_start(period, - range_type='weeks', - end_timestamp=None, - adjustment=True, - day_of_start=None): - """ - get date ranges in epoch timestamps for a period with user-defined day of start - - Arguments: - period {str} -- period mnemonic '2020Q2' - - Keyword Arguments: - range_type {str} -- range type to fetch (default: 'weeks') 'weeks' - supports days and weeks - end_timestamp {int} -- get range up to this timestamp - day_of_start {int} -- mention from which day you want to start the week '0' - 0-Monday, 1-Tuesday etc. - - Returns: - list -- epoch timestamps for range - """ - prd_begin, prd_end = period_rng_from_mnem(period) - if adjustment: - prd_begin += SIX_HOURS # HACK to deal with results timestamps being 3AM - step_size = ONE_WEEK if range_type == 'weeks' else ONE_DAY - period_range = [] - if day_of_start != datetime.fromtimestamp(epoch(prd_begin).as_epoch() / 1000).weekday(): - period_range.append(prd_begin) - prd_begin = epoch(rrule(freq=WEEKLY, dtstart=epoch(prd_begin).as_datetime(), byweekday=day_of_start, count=1)[0]).as_epoch() - - for ts in np.arange(prd_begin, prd_end, step_size): - if not end_timestamp or ts <= end_timestamp: - period_range.append(ts) - - if not end_timestamp and range_type == 'days': - period_range[-1] = prd_end # HACK to get end of period timestamp at midnight - return period_range - -def get_period_eod_timestamps(period): - - prd_begin, prd_end = period_rng_from_mnem(period) - prd_begin = get_eod(epoch(prd_begin)) - prd_end = get_eod(epoch(prd_end)) - e = prd_begin - timestamps = [] - while e <= prd_end: - timestamps.append(e.as_epoch()) - e += relativedelta(days=1) - return timestamps - -def get_timestamps_by_start_end_prd_ts(start_prd_ts,end_prd_ts): - - prd_begin = get_eod(epoch(start_prd_ts)) - prd_end = get_eod(epoch(end_prd_ts)) - e = prd_begin - timestamps= [] - while e <= prd_end: - timestamps.append(e.as_epoch()) - e += relativedelta(days=1) - return timestamps - def get_period_as_of(period, as_of=None, config=None, @@ -759,7 +427,7 @@ def get_period_as_of(period, epoch -- epoch of apps current time for period, or valid as_of """ # totes cheating to get across services - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() if as_of: @@ -797,7 +465,7 @@ def get_period_infos(period, list -- [{prd_dtls}] """ # totes cheating to get across services - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() # TODO: this may be wrong... @@ -850,7 +518,7 @@ def get_as_of_dates(period, dict -- {as_of_str: {'ep': epoch, 'xl': xldate, 'str': string description}} """ # totes cheating to get across services - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() # TODO: this may be wrong... @@ -902,86 +570,6 @@ def get_as_of_dates(period, return as_ofs -def get_as_of_dates_for_load_changes(period, - as_of=None, - include_yest=False, - config=None, - include_bmon=False): - """ - get as of dates for period in multiple date formats - - Arguments: - period {str} -- period mnemonic '2020Q2' - - Keyword Arguments: - as_of {int} -- epoch timestamp - include_yest {bool} -- include yest as_of if True (default: False) - config {PeriodsConfig} -- instance of PeriodsConfig ...? - - Returns: - dict -- {as_of_str: {'ep': epoch, 'xl': xldate, 'str': string description}} - """ - # totes cheating to get across services - from config.periods_config import PeriodsConfig - config = config if config is not None else PeriodsConfig() - - # TODO: this may be wrong... - now = epoch(get_period_as_of(period, as_of, config=config)) - bow = get_bow(now) - bom = get_bom(now) - week_ago = get_week_ago(now) - boq = get_boq(now) - if boq > week_ago: - week_ago = boq - - as_ofs = {'now': {'ep': now.as_epoch() + SIX_HOURS, - 'xl': int(now.as_xldate()), - 'str': 'Now', - 'date': now.as_datetime().strftime("%Y-%m-%d") - }, - 'bow': {'ep': week_ago.as_epoch() + SIX_HOURS, - 'xl': int(week_ago.as_xldate()), - 'str': 'Since Last Week', - 'date': week_ago.as_datetime().strftime("%Y-%m-%d")}, - 'bom': {'ep': bom.as_epoch() + SIX_HOURS, - 'xl': int(bom.as_xldate()), - 'str': 'Since Beginning of Month', - 'date': bom.as_datetime().strftime("%Y-%m-%d")}} - if include_bmon: - bmon = get_eod(get_bow(now)) - as_ofs['bmon'] = {'ep': bmon.as_epoch(), - 'xl': int(bmon.as_xldate()), - 'str': 'Since Monday', - 'date': bmon.as_datetime().strftime("%Y-%m-%d")} - if include_yest: - yest = get_yest(now) - if boq > yest: - yest = boq - as_ofs['yest'] = {'ep': yest.as_epoch() + SIX_HOURS, - 'xl': int(yest.as_xldate()), - 'str': 'Since Yesterday', - 'date': yest.as_datetime().strftime("%Y-%m-%d")} - - # monthly | is period quarter | should add quarter - # True | True | True - # True | False | False - # False | False | False - # False | False | True - if config.monthly_predictions: - if 'Q' not in period: - return as_ofs - elif 'Q' not in period: - return as_ofs - - as_ofs['boq'] = {'ep': boq.as_epoch() + SIX_HOURS, - 'xl': int(boq.as_xldate()), - 'str': 'Since Beginning of Quarter', - 'date': boq.as_datetime().strftime("%Y-%m-%d")} - - return as_ofs - - - def get_period_boundaries(period, config=None): """ @@ -997,7 +585,7 @@ def get_period_boundaries(period, list -- [(prd beg ts, prd end ts, prd mnem)] sorted by prd beg ts """ # totes cheating to get across services - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() backwards = 24 @@ -1090,7 +678,7 @@ def get_time_context(period, close_periods, deal_timestamp, relative_periods) """ # totes cheating to get across services - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() fm_period, comp_periods = get_period_and_component_periods(period, config) period_and_close_periods_for_year = [] @@ -1125,35 +713,6 @@ def get_time_context(period, period_and_close_periods_for_year) -def get_snapshot_range(period, - config=None): - """ - return available snapshot range for a period in ISO-8601 format - snapshot range is beginning of prev period to end of this period - - Arguments: - period {str} -- period mnemonic '2020Q2' - - Keyword Arguments: - config {PeriodsConfig} -- instance of PeriodsConfig ...? - - Returns: - tuple -- begin date, end date - """ - # totes cheating to get across services - # TODO: bad, this is fm logic in the periods service ... - from config.periods_config import PeriodsConfig - config = config if config is not None else PeriodsConfig() - if period[-2] != 'Q': - prd_begin, prd_end = [epoch(x) for x in period_rng_from_mnem(period)] - period = period_details(prd_begin.as_datetime(),period_type = "Y").mnemonic if len(period) == 4 else period_details(prd_begin.as_datetime(),period_type = "Y").mnemonic - prev_period = get_prevq_mnem_safe(period) - - _, end = period_rng_from_mnem(period) - begin, _ = period_rng_from_mnem(prev_period) - return epoch(begin).as_datetime().date().isoformat(), epoch(end).as_datetime().date().isoformat() - - def get_current_period(config=None): """ get mnemonic of current app period @@ -1165,7 +724,7 @@ def get_current_period(config=None): str -- period mnemonic """ # totes cheating to get across services - from config.periods_config import PeriodsConfig + from config import PeriodsConfig config = config if config is not None else PeriodsConfig() # TODO: should there be months?? @@ -1198,95 +757,6 @@ def get_available_quarters_and_months(config): return quarters_and_months -def adjust_current_period(periods_data, now_dt): - """ - - iterate over all periods and check if the current period has - - Arguments: - periods {obj} -- periods obj - Example - { - "2020Q1": { - "has_forecast": true, - "begin": 1577865600000, - "relative_period": "h", - "end": 1585724399999 - }, - "2020Q3": { - "has_forecast": true, - "begin": 1593586800000, - "relative_period": "c", - "end": 1601535599999 - } - } - """ - - try: - current_prd = current_period(now_dt).mnemonic - current_period_update_date = try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(current_prd), 0)) - if current_period_update_date == 0: - prev_prd = prev_period(now_dt) - - periods_data[prev_prd.mnemonic]["relative_period"] = "c" - periods_data[current_prd]["relative_period"] = "f" - - # considering previous period's end as now_dt for further processing - new_now = epoch(periods_data[prev_prd.mnemonic]["end"]).as_datetime() - return periods_data, new_now - except Exception: - logger.info("previous period does not exist, current period {}".format(now_dt)) - - return periods_data, now_dt - -def get_current_month(config): - now_dt = get_now(config).as_datetime() - curr_mnem = current_period(now_dt).mnemonic - for month in monthly_periods(curr_mnem): - if month.begin <= now_dt <= month.end: - return month - - return now_dt - -def get_current_week(config): - now_dt = get_now(config).as_datetime() - curr_mnem = current_period(now_dt).mnemonic - for week in weekly_periods(curr_mnem): - if week.begin <= now_dt <= week.end: - return week - - return now_dt - -def get_current_weeks(period): - peirod_begin,period_end = get_period_begin_end(period) - peirod_begin, period_end = epoch(peirod_begin).as_datetime(),epoch(period_end).as_datetime() - curr_mnem = current_period(peirod_begin).mnemonic - weeks = [] - for week in weekly_periods(curr_mnem): - if peirod_begin <= week.begin <= period_end: - weeks.append(week) - return weeks - -def is_period_in_past(period): - """ - Determines if a given period is in the past. - - Args: - period (str): The period to check. - - Returns: - bool: True if the period is in the past, False otherwise. - """ - - from config.periods_config import PeriodsConfig - period_in_past = False - if PeriodsConfig().monthly_fm: - if 'Q' not in period and len(period) != 4: - time_context = get_time_context(period) - if 'h' in time_context.relative_periods: - period_in_past = True - return period_in_past - - def versioned_func_switch(fn): @wraps(fn) def switched_fn(*args, **kwargs): @@ -1359,217 +829,15 @@ def fetch_node(as_of, return hier_collection.find(criteria, projection).next() - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_node_versioned(as_of, - node, - include_hidden=False, - drilldown=True, - fields=[], - db=None, - period=None, - boundary_dates=None - ): +@versioned_func_switch +def fetch_eligible_nodes_for_segment(as_of, + segment, + drilldown=True, + db=None, + period=None, + boundary_dates=None): """ - fetch node record for single node - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - node {str} -- node '0050000FLN2C9I2' - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - dict -- node record - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - if boundary_dates: - from_date, to_date = boundary_dates - else: - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - criteria = {'node': node, - '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - projection = {} - if fields: - for field in fields: - projection[field] = 1 - else: - projection['_id'] = 0 - - return hier_collection.find(criteria, projection).next() - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_many_nodes(as_of, - nodes, - include_hidden=False, - drilldown=True, - db=None, - period=None - ): - """ - fetch node records for many nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - nodes {list} -- list of nodes to find records for ['0050000FLN2C9I2', ] - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - generator -- generator of dicts of node records - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - criteria = {'$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - if nodes: - criteria['node'] = {'$in': nodes} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ]}, # Checking for the node which is not marked as hidden earlier - {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state - {'hidden_to': 0}] # Already Node has been unblocked from hidden state - } - criteria = {'$and': [criteria, hide_criteria]} - - return hier_collection.find(criteria, {'_id': 0}) - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_many_nodes_versioned(as_of, - nodes, - include_hidden=False, - drilldown=True, - db=None, - period=None - ): - """ - fetch node records for many nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - nodes {list} -- list of nodes to find records for ['0050000FLN2C9I2', ] - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - generator -- generator of dicts of node records - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - criteria = {'$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if nodes: - criteria['node'] = {'$in': nodes} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - return hier_collection.find(criteria, {'_id': 0}) - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_eligible_nodes_for_segment(as_of, - segment, - drilldown=True, - db=None, - period=None, - boundary_dates=None): - """ - fetches the nodes which are eligible for given segment + fetches the nodes which are eligible for given segment returns nodes which are eligible """ @@ -1587,54 +855,6 @@ def fetch_eligible_nodes_for_segment(as_of, return hier_collection.find(criteria, {'_id':0, 'node': 1}) -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_eligible_nodes_for_segment_versioned(as_of, - segment, - drilldown=True, - db=None, - period=None, - boundary_dates = None): - """ - fetches the nodes which are eligible for given segment - - returns nodes which are eligible - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - hier_collection.ensure_index([('normal_segs', -1), ('from', -1), ('to', -1)]) - - if boundary_dates: - from_date, to_date = boundary_dates - else: - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - criteria = {'normal_segs': {'$in': [segment]}, - '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - - return hier_collection.find(criteria, {'_id':0, 'node': 1}) - def check_children(node, drilldown=True, db=None): coll = DRILLDOWN_COLL if drilldown else HIER_COLL hier_collection = db[coll] if db else sec_context.tenant_db[coll] @@ -1693,96 +913,6 @@ def fetch_hidden_nodes(as_of, return hier_collection.find(criteria, {'_id': 0}) -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_hidden_nodes_versioned(as_of, - drilldown=True, - signature='', - db=None, - period=None, - service=None): - """ - fetch nodes that are hidden - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - - Keyword Arguments: - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - signature {str} -- if provided, fetch only hidden nodes with how - matching signature, (default: {''}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - list -- list of dicts of node records - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if period: - as_of = get_period_as_of(period) - from_date, to_date = fetch_boundry(as_of, drilldown=drilldown, service=service) - else: - from_date = to_date = as_of - - if 'Q' in period: - prev_prd = prev_period(period_type='Q')[0] - prev_as_of = get_period_as_of(prev_prd) - prev_timestamp, prev_end_timestamp = fetch_boundry(prev_as_of, drilldown=drilldown, service=service) - else: - prev_prd = str(int(period[:4]) - 1) + str(int(period[4:]) + 11) if int(period[4:]) - 1 == 0 else str(int(period) - 1) - prev_as_of = get_period_as_of(prev_prd) - prev_timestamp, prev_end_timestamp = fetch_boundry(prev_as_of, drilldown=drilldown, service=service) - - how_criteria = {'$or': [{'how': {'$regex': '_hide'}}, - {'how': {'$regex': '^hide', '$options': 'm'}}]} - - previous_period_hidden_criteria = {'from': {'$lt': from_date}} - - previous_period_hidden_criteria.update(how_criteria) - - #logger.info("previous_period_hidden_criteria: %s",previous_period_hidden_criteria) - - current_period_nodes_criteria = {'from': {'$gte': prev_timestamp}} - - if to_date is None: - current_period_nodes_criteria['$or'] = [{"to": None}, - {"to": {'$lte': prev_end_timestamp}}] - else: - current_period_nodes_criteria['to'] = {'$lte':to_date} - - current_period_hidden_criteria= copy.deepcopy(current_period_nodes_criteria) - current_period_hidden_criteria.update(how_criteria) - - #logger.info("current_period_hidden_criteria: %s", current_period_hidden_criteria) - - hidden_nodes = {} - - previous_period_hidden_nodes = hier_collection.find(previous_period_hidden_criteria, {'_id': 0}) - current_period_hidden_nodes = hier_collection.find(current_period_hidden_criteria, {'_id': 0}) - current_period_nodes = hier_collection.find(current_period_nodes_criteria, {'_id': 0}) - - for node in previous_period_hidden_nodes: - hidden_nodes[node['node']] = node - - for node in current_period_hidden_nodes: - hidden_nodes[node['node']] = node - - for node in current_period_nodes: - if "_hide" in node['how'] or node['how'].startswith('hide'): - continue - - if node['node'] in hidden_nodes and node['from'] > hidden_nodes[node['node']]['from']: - del hidden_nodes[node['node']] - - #logger.info("%s hidden nodes found",len(hidden_nodes)) - - return hidden_nodes.values() - # This function is for a non-versioned tenant # TODO: Any changes to this function should be made in versioned function as well if required @versioned_func_switch @@ -1868,275 +998,62 @@ def fetch_root_nodes(as_of, # This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_root_nodes_versioned(as_of, - include_hidden=False, - drilldown=True, - true_roots=False, - db=None, - period=None, - service=None - ): +# TODO: Any changes to this function should be made in versioned function as well if required +@versioned_func_switch +def fetch_children(as_of, + nodes, + include_hidden=False, + drilldown=True, + db=None, + period=None, + boundary_dates=None, + service=None + ): """ - fetch node records for root nodes + fetch node records for children of given nodes Arguments: as_of {int} -- epoch timestamp to fetch data as of 1556074024910 + nodes {list} -- node to get children on ['0050000FLN2C9I2', ] Keyword Arguments: include_hidden {bool} -- if True, include hidden nodes (default: {False}) drilldown {bool} -- if True, fetch drilldown node instead of hierarchy (default: {False}) - true_roots {bool} -- if True, fetch the true admin root nodes - otherwise fetch the visible root node in app - (default: {False}) db {object} -- instance of tenant_db (default: {None}) if None, will create one Returns: - list -- list of dicts of node records + generator -- generator of node records """ coll = HIER_COLL if not drilldown else DRILLDOWN_COLL if is_lead_service(service): coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL hier_collection = db[coll] if db else sec_context.tenant_db[coll] - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - criteria = {'parent': None, - '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - pipeline = [{'$match': criteria}, - {'$project': {'_id':0}}] - - roots = hier_collection.aggregate(pipeline) - if true_roots or not drilldown: - return roots + as_of = 0 - # TODO: ok 1, this is dumb and can probably do done in a single query - # but also 2, is this safe and does it make sense? - criteria = {'parent': {'$in': [x['node'] for x in roots]}, + criteria = {'parent': {'$in': nodes}, '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} + {'$or': [{'from': None}, + {'from': {'$lte': as_of}}]}, + {'$or': [{'to': None}, + {'to': {'$gte': as_of}}]} ]} - if not include_hidden: hide_criteria = {'$or': [ {"$and": [ {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] + ]}, # Checking for the node which is not marked as hidden earlier + {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state + {'hidden_to': 0}] # Already Node has been unblocked from hidden state } criteria = {'$and': [criteria, hide_criteria]} - pipeline = [{'$match': criteria}, - {'$project': {'_id':0, 'from':0, 'to':0}}] - - return [node for node in hier_collection.aggregate(pipeline) if 'not_in_hier' not in node['node']] - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_children(as_of, - nodes, - include_hidden=False, - drilldown=True, - db=None, - period=None, - boundary_dates=None, - service=None - ): - """ - fetch node records for children of given nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - nodes {list} -- node to get children on ['0050000FLN2C9I2', ] - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - generator -- generator of node records - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - as_of = 0 - - criteria = {'parent': {'$in': nodes}, - '$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ]}, # Checking for the node which is not marked as hidden earlier - {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state - {'hidden_to': 0}] # Already Node has been unblocked from hidden state - } - criteria = {'$and': [criteria, hide_criteria]} - - # return sorted from highest priority to lowest priority, then break ties alphabetically by node id - return hier_collection.find(criteria, {'_id': 0}).sort([('priority', -1), ('node', 1)]) - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_children_versioned(as_of, - nodes, - include_hidden=False, - drilldown=True, - db=None, - period=None, - boundary_dates=None, - service=None - ): - """ - fetch node records for children of given nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - nodes {list} -- node to get children on ['0050000FLN2C9I2', ] - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - generator -- generator of node records - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if boundary_dates: - from_date, to_date = boundary_dates - else: - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown, service=service) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - criteria = {'parent': {'$in': nodes}, - '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - group = {'_id': - {'node': '$node', - 'parent': '$parent', - 'label': '$label', - 'how': '$how'}, - 'node':{'$last':'$node'}, - 'parent':{'$last':'$parent'}, - 'label':{'$last':'$label'}, - 'how':{'$last':'$how'}, - 'priority':{'$last':'$priority'}} - - project = {'_id': 0} - - sort = {'priority': -1, 'node': 1} - - pipeline = [{'$match':criteria}, {'$group': group}, {'$project': project}, {'$sort':sort}] - - # return sorted from highest priority to lowest priority, then break ties alphabetically by node id - return hier_collection.aggregate(pipeline) + # return sorted from highest priority to lowest priority, then break ties alphabetically by node id + return hier_collection.find(criteria, {'_id': 0}).sort([('priority', -1), ('node', 1)]) # This function is for a non-versioned tenant @@ -2247,144 +1164,6 @@ def fetch_ancestors(as_of, return _anc_yielder(hier_collection.aggregate(pipeline, allowDiskUse=True), exclude_admin_root) - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_ancestors_versioned(as_of, - nodes=None, - include_hidden=False, - include_children=False, - drilldown=True, - limit_to_visible=False, - db=None, - period=None, - boundary_dates=None, - service=None, - unattached=False, - is_edw_write=False - ): - """ - fetch ancestors of many hierarchy nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - - Keyword Arguments: - nodes {list} -- nodes to find ancestors for, if None grabs all ['0050000FLN2C9I2', ] - (default: {None}) - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - include_children {bool} -- if True, fetch children of nodes in nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - limit_to_visible {bool} if True, only return ancestors up to visible root if not admin user - if admin user, return ancestors up to admin root - if False, return all ancestors regardless of visibility - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - generator -- generator of ({node: node, parent: parent, ancestors: [ancestors from root to bottom]}) - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if boundary_dates: - from_date, to_date = boundary_dates - else: - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown, service=service) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - if limit_to_visible and is_edw_write == False: - user_access = get_user_permissions(sec_context.get_effective_user(), 'results') - exclude_admin_root = '!' not in user_access - else: - exclude_admin_root = False - - criteria = {'$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - if nodes: - node_criteria = {'node': {'$in': list(nodes)}} - if include_children: - node_criteria = {'$or': [node_criteria, {'parent': {'$in': list(nodes)}}]} - match = merge_dicts(criteria, node_criteria) - else: - match = criteria - - anc_criteria = copy.deepcopy(criteria) - if unattached: - anc_criteria.update({ - 'unattached': True - }) - lookup = {'from': coll, - 'startWith': '$parent', - 'connectFromField': 'parent', - 'connectToField': 'node', - 'depthField': 'level', - 'restrictSearchWithMatch': anc_criteria, - 'as': 'ancestors'} - project = {'node': 1, - 'parent': 1, - 'label': 1, - '_id': 0, - 'ancestors': {'$map': {'input': '$ancestors', - 'as': 'ancestor', - 'in': {'node': '$$ancestor.node', - 'level': '$$ancestor.level'}}}} - - group = {'_id': - {'node': '$node', - 'parent': '$parent', - 'label': '$label', - 'how': '$how'}, - 'node':{'$last':'$node'}, - 'parent':{'$last':'$parent'}, - 'label':{'$last':'$label'}, - 'how':{'$last':'$how'}} - - pipeline = [{'$match': match}, - {'$group': group}, - {'$graphLookup': lookup}, - {'$project': project}] - - return _anc_yielder_versioned(hier_collection.aggregate(pipeline), exclude_admin_root) - - # This function is for a non-versioned tenant # TODO: Any changes to this function should be made in versioned function as well if required @versioned_func_switch @@ -2478,246 +1257,62 @@ def fetch_descendants(as_of, # This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_descendants_versioned(as_of, - nodes=None, - levels=1, - include_hidden=False, - include_children=False, - drilldown=True, - db=None, - period=None, - boundary_dates=None, - leads=False, - ): +# TODO: Any changes to this function should be made in versioned function as well if required +@versioned_func_switch +def fetch_users_nodes_and_root_nodes(user, + as_of, + include_hidden=False, + drilldown=True, + db=None, + period=None, + service=None + ): """ - fetch descendants of many hierarchy nodes + fetch the top level nodes that a user has access to, and the hierarchy roots of those nodes Arguments: + user {dict} -- user object from sec_context ?? as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - nodes {list} -- nodes to find descendants for ['0050000FLN2C9I2', ] Keyword Arguments: - nodes {list} -- nodes to find ancestors for, if None grabs all - (default: {None}) - levels {int} -- how many levels of tree to traverse down - (default: {1}) include_hidden {bool} -- if True, include hidden nodes (default: {False}) - include_children {bool} -- if True, fetch children of nodes in nodes - (default: {False}) drilldown {bool} -- if True, fetch drilldown node instead of hierarchy (default: {False}) db {object} -- instance of tenant_db (default: {None}) if None, will create one Returns: - generator -- generator of ({node: node, parent: parent, descendants: ({children}, {grandchildren}, ... ,)}) + tuple -- ([users top level node dicts], [users root dicts]) """ - if leads: coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - else: coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - if boundary_dates: - from_date, to_date = boundary_dates + if service=='leads': + user_access = get_user_permissions(user, 'leads_results') else: - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of + user_access = get_user_permissions(user, 'results') + logger.info("User Access is data for nodes:" + str(user_access)) + # admin access, users nodes are the admin level roots, usually invisible in app + if '!' in user_access: + root_nodes = [merge_dicts(node, {'root': node['node']}) + for node in fetch_root_nodes(as_of, include_hidden, drilldown, true_roots=True, db=db, service=service)] + return root_nodes, root_nodes - criteria = {'$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} + root_nodes = [merge_dicts(node, {'root': node['node']}) + for node in fetch_root_nodes(as_of, include_hidden, drilldown, true_roots=False, db=db, service=service)] + # global access, users nodes are the hierarchies roots + if '*' in user_access: + if not is_lead_service(service): + return root_nodes, root_nodes + user_access.remove('*') - if nodes: - node_criteria = {'node': {'$in': list(nodes)}} - if include_children: - node_criteria = {'$or': [node_criteria, {'parent': {'$in': list(nodes)}}]} - match = merge_dicts(criteria, node_criteria) - else: - match = criteria - - lookup = {'from': coll, - 'startWith': '$node', - 'connectFromField': 'node', - 'connectToField': 'parent', - 'depthField': 'level', - 'restrictSearchWithMatch': criteria, - 'maxDepth': levels - 1, - 'as': 'descendants'} - project = {'node': 1, - 'parent': 1, - 'label': 1, - '_id': 0, - 'descendants': {'$map': {'input': '$descendants', - 'as': 'descendant', - 'in': {'node': '$$descendant.node', - 'parent': '$$descendant.parent', - 'level': '$$descendant.level'}}}} - pipeline = [{'$match': match}, - {'$graphLookup': lookup}, - {'$project': project}, - ] - - return _desc_yielder(hier_collection.aggregate(pipeline), levels) - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_users_nodes_and_root_nodes(user, - as_of, - include_hidden=False, - drilldown=True, - db=None, - period=None, - service=None - ): - """ - fetch the top level nodes that a user has access to, and the hierarchy roots of those nodes - - Arguments: - user {dict} -- user object from sec_context ?? - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - tuple -- ([users top level node dicts], [users root dicts]) - """ - - if service=='leads': - user_access = get_user_permissions(user, 'leads_results') - else: - user_access = get_user_permissions(user, 'results') - logger.info("User Access is data for nodes:" + str(user_access)) - - # admin access, users nodes are the admin level roots, usually invisible in app - if '!' in user_access: - root_nodes = [merge_dicts(node, {'root': node['node']}) - for node in fetch_root_nodes(as_of, include_hidden, drilldown, true_roots=True, db=db, service=service)] - return root_nodes, root_nodes - - root_nodes = [merge_dicts(node, {'root': node['node']}) - for node in fetch_root_nodes(as_of, include_hidden, drilldown, true_roots=False, db=db, service=service)] - # global access, users nodes are the hierarchies roots - if '*' in user_access: - if not is_lead_service(service): - return root_nodes, root_nodes - user_access.remove('*') - - user_nodes, nodes, user_roots = [], set(), set() - for node in fetch_ancestors(as_of, user_access, include_hidden, drilldown=drilldown, limit_to_visible=True, db=db, service=service): - try: - node['root'] = node['ancestors'][0] - except IndexError: - node['root'] = node['node'] - node['level'] = len(node['ancestors']) - - user_nodes.append(node) - nodes.add(node['node']) - user_roots.add(node['root']) - - user_nodes = sorted((node for node in user_nodes if not any(ancestor in nodes for ancestor in node['ancestors'])), - key=lambda x: x['level']) - - return user_nodes, [x for x in root_nodes if x['node'] in user_roots] - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_users_nodes_and_root_nodes_versioned(user, - as_of, - include_hidden=False, - drilldown=True, - db=None, - period=None, - service=None - ): - """ - fetch the top level nodes that a user has access to, and the hierarchy roots of those nodes - - Arguments: - user {dict} -- user object from sec_context ?? - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - tuple -- ([users top level node dicts], [users root dicts]) - """ - if service == 'leads': - user_access = get_user_permissions(user, 'leads_results') - else: - user_access = get_user_permissions(user, 'results') - logger.info("User Access is data for nodes:" + str(user_access)) - - # admin access, users nodes are the admin level roots, usually invisible in app - if '!' in user_access: - root_nodes = [merge_dicts(node, {'root': node['node']}) - for node in - fetch_root_nodes(as_of, include_hidden, drilldown, true_roots=True, db=db, period=period, - service=service)] - return root_nodes, root_nodes - - root_nodes = [merge_dicts(node, {'root': node['node']}) - for node in fetch_root_nodes(as_of, include_hidden, drilldown, true_roots=False, db=db, period=period, - service=service)] - # global access, users nodes are the hierarchies roots - if '*' in user_access: - if not is_lead_service(service): - return root_nodes, root_nodes - user_access.remove('*') - - user_nodes, nodes, user_roots = [], set(), set() - for node in fetch_ancestors(as_of, user_access, include_hidden, drilldown=drilldown, limit_to_visible=True, db=db, - period=period, service=service): - try: - node['root'] = node['ancestors'][0] - except IndexError: - node['root'] = node['node'] - node['level'] = len(node['ancestors']) + user_nodes, nodes, user_roots = [], set(), set() + for node in fetch_ancestors(as_of, user_access, include_hidden, drilldown=drilldown, limit_to_visible=True, db=db, service=service): + try: + node['root'] = node['ancestors'][0] + except IndexError: + node['root'] = node['node'] + node['level'] = len(node['ancestors']) user_nodes.append(node) nodes.add(node['node']) @@ -2728,7 +1323,6 @@ def fetch_users_nodes_and_root_nodes_versioned(user, return user_nodes, [x for x in root_nodes if x['node'] in user_roots] - # This function is for a non-versioned tenant # TODO: Any changes to this function should be made in versioned function as well if required @versioned_func_switch @@ -2796,104 +1390,27 @@ def fetch_node_to_parent_mapping_and_labels(as_of, return node_to_parent, labels - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_node_to_parent_mapping_and_labels_versioned(as_of, - include_hidden=False, - drilldown=True, - db=None, - action=None, - period=None, - boundary_dates=None, - service=None - ): - """ - fetch map of node : parent for all nodes in hierarchy - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - tuple -- ({node: parent}, {node: label}) - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if boundary_dates: - from_date, to_date = boundary_dates - else: - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown, service=service) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - criteria = {'$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if not include_hidden: - hide_criteria = {'$nor':[{'how': {'$regex':'_hide'}}, - {'how': {'$regex':'^hide','$options':'m'}}]} - criteria = {'$and': [criteria, hide_criteria]} - - hier_records = hier_collection.find(criteria, {'node': 1, 'parent': 1, 'label': 1, '_id': 0, 'is_team':1}) - - node_to_parent, labels = {}, {} - if action: - for hier_record in hier_records: - node_to_parent[hier_record['node']] = {"parent": hier_record['parent'], "is_team": hier_record.get("is_team", False)} - labels[hier_record['node']] = hier_record['label'] - else: - for hier_record in hier_records: - node_to_parent[hier_record['node']] = hier_record['parent'] - labels[hier_record['node']] = hier_record['label'] - - return node_to_parent, labels - - # This function is for a non-versioned tenant # TODO: Any changes to this function should be made in versioned function as well if required @versioned_func_switch -def search_nodes(as_of, - search_terms, - include_hidden=False, - drilldown=True, - db=None, - period=None): +def fetch_descendant_ids(as_of, + node, + levels=None, + include_hidden=False, + drilldown=True, + db=None, + period=None, + service=None): """ - search for nodes fuzzy matching search terms + fetch unordered list of node ids of descendants of node Arguments: as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - search_terms {list} -- list of terms to search for, OR not AND ['alex', 'megan'] + node {str} -- node '0050000FLN2C9I2' Keyword Arguments: + levels {int} -- how many levels of tree to traverse down + if None, traverses entire tree (default: {None}) include_hidden {bool} -- if True, include hidden nodes (default: {False}) drilldown {bool} -- if True, fetch drilldown node instead of hierarchy @@ -2902,9 +1419,11 @@ def search_nodes(as_of, if None, will create one Returns: - generator -- generator of dicts of node records + list -- [node ids] """ coll = HIER_COLL if not drilldown else DRILLDOWN_COLL + if is_lead_service(service): + coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL hier_collection = db[coll] if db else sec_context.tenant_db[coll] as_of = 0 @@ -2925,1277 +1444,110 @@ def search_nodes(as_of, {'hidden_to': 0}] # Already Node has been unblocked from hidden state } criteria = {'$and': [criteria, hide_criteria]} - # TODO: this match could be looser, not super kind to typos - criteria['$or'] = [{field: {'$regex': '|'.join(search_terms), '$options': 'i'}} for field in ['label', 'node']] - return hier_collection.find(criteria, {'node': 1, 'label': 1, '_id': 0}) + if node: + match = merge_dicts(criteria, {'node': node}) + else: + roots = [x['node'] for x in fetch_root_nodes(as_of, drilldown=drilldown, service=service)] + match = merge_dicts(criteria, {'node': {'$in': roots}}) + + lookup = {'from': coll, + 'startWith': '$node', + 'connectFromField': 'node', + 'connectToField': 'parent', + 'depthField': 'level', + 'restrictSearchWithMatch': criteria, + 'as': 'descendants'} + if levels: + lookup['maxDepth'] = levels - 1 + + project = {'_id': 0, + 'descendants': '$descendants.node'} + pipeline = [{'$match': match}, + {'$graphLookup': lookup}, + {'$project': project}, + ] + try: + return hier_collection.aggregate(pipeline).next()['descendants'] + except StopIteration: + return [] # This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def search_nodes_versioned(as_of, - search_terms, - include_hidden=False, - drilldown=True, - db=None, - period=None): +# TODO: Any changes to this function should be made in versioned function as well if required +@versioned_func_switch +def fetch_top_level_nodes(as_of, + levels=1, + include_hidden=False, + include_children=False, + drilldown=True, + db=None, + period=None + ): """ - search for nodes fuzzy matching search terms + fetch descendants of many hierarchy nodes Arguments: as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - search_terms {list} -- list of terms to search for, OR not AND ['alex', 'megan'] + nodes {list} -- nodes to find descendants for ['0050000FLN2C9I2', ] Keyword Arguments: + levels {int} -- how many levels of tree to traverse down + (default: {1}) include_hidden {bool} -- if True, include hidden nodes (default: {False}) + include_children {bool} -- if True, fetch children of nodes in nodes + (default: {False}) drilldown {bool} -- if True, fetch drilldown node instead of hierarchy (default: {False}) db {object} -- instance of tenant_db (default: {None}) if None, will create one - Returns: - generator -- generator of dicts of node records - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - criteria = {'$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - # TODO: this match could be looser, not super kind to typos - criteria['$or'] = [{field: {'$regex': '|'.join(search_terms), '$options': 'i'}} for field in ['label', 'node']] - - return hier_collection.find(criteria, {'node': 1, 'label': 1, '_id': 0}) - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_descendant_ids(as_of, - node, - levels=None, - include_hidden=False, - drilldown=True, - db=None, - period=None, - service=None): - """ - fetch unordered list of node ids of descendants of node - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - node {str} -- node '0050000FLN2C9I2' - - Keyword Arguments: - levels {int} -- how many levels of tree to traverse down - if None, traverses entire tree (default: {None}) - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - list -- [node ids] - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - as_of = 0 - - criteria = {'$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ]}, # Checking for the node which is not marked as hidden earlier - {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state - {'hidden_to': 0}] # Already Node has been unblocked from hidden state - } - criteria = {'$and': [criteria, hide_criteria]} - - if node: - match = merge_dicts(criteria, {'node': node}) - else: - roots = [x['node'] for x in fetch_root_nodes(as_of, drilldown=drilldown, service=service)] - match = merge_dicts(criteria, {'node': {'$in': roots}}) - - lookup = {'from': coll, - 'startWith': '$node', - 'connectFromField': 'node', - 'connectToField': 'parent', - 'depthField': 'level', - 'restrictSearchWithMatch': criteria, - 'as': 'descendants'} - if levels: - lookup['maxDepth'] = levels - 1 - - project = {'_id': 0, - 'descendants': '$descendants.node'} - pipeline = [{'$match': match}, - {'$graphLookup': lookup}, - {'$project': project}, - ] - - try: - return hier_collection.aggregate(pipeline).next()['descendants'] - except StopIteration: - return [] - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_descendant_ids_versioned(as_of, - node, - levels=None, - include_hidden=False, - drilldown=True, - db=None, - period=None, service=None): - """ - fetch unordered list of node ids of descendants of node - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - node {str} -- node '0050000FLN2C9I2' - - Keyword Arguments: - levels {int} -- how many levels of tree to traverse down - if None, traverses entire tree (default: {None}) - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - list -- [node ids] - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown, service=service) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - criteria = {'$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - if node: - match = merge_dicts(criteria, {'node': node}) - else: - roots = [x['node'] for x in fetch_root_nodes(as_of, drilldown=drilldown, period=period, service=service)] - match = merge_dicts(criteria, {'node': {'$in': roots}}) - - lookup = {'from': coll, - 'startWith': '$node', - 'connectFromField': 'node', - 'connectToField': 'parent', - 'depthField': 'level', - 'restrictSearchWithMatch': criteria, - 'as': 'descendants'} - if levels: - lookup['maxDepth'] = levels - 1 - - project = {'_id': 0, - 'descendants': '$descendants.node'} - pipeline = [{'$match': match}, - {'$graphLookup': lookup}, - {'$project': project}, - ] - - try: - return hier_collection.aggregate(pipeline).next()['descendants'] - except StopIteration: - return [] - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_ancestor_ids(as_of, - nodes=None, - include_hidden=False, - hidden_only=False, - drilldown=True, - signature='', - db=None, - period=None, - service=None): - """ - fetch set of node ids of ancestors of nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - - Keyword Arguments: - nodes {list} -- nodes ['0050000FLN2C9I2'] - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - hidden_only {bool} -- if True, include only hidden nodes and their ancestors - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - signature {str} -- if provided, fetch only hidden nodes with how - matching signature, (default: {''}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - set -- {node ids} - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - as_of = 0 - - criteria = {'$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ]}, # Checking for the node which is not marked as hidden earlier - {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state - {'hidden_to': 0}] # Already Node has been unblocked from hidden state - } - criteria = {'$and': [criteria, hide_criteria]} - - if nodes: - node_criteria = {'node': {'$in': list(nodes)}} - match = merge_dicts(criteria, node_criteria) - else: - match = {k: v for k, v in criteria.items()} - - if signature: - match['how'] = signature - - if hidden_only: - hidden_only_criteria = {'$or': [ - {'$and': [{'hidden_from': 0}, - {"$or": [{"hidden_to": {"$exists": False}}, {"hidden_to": None}]} - ] - }] - } - match = {'$and': [criteria, hidden_only_criteria]} - - lookup = {'from': coll, - 'startWith': '$parent', - 'connectFromField': 'parent', - 'connectToField': 'node', - 'depthField': 'level', - 'restrictSearchWithMatch': criteria, - 'as': 'ancestors'} - project = {'_id': 0, - 'ancestors': '$ancestors.node', - 'node': 1} - - pipeline = [{'$match': match}, - {'$graphLookup': lookup}, - {'$project': project}] - - ids = set() - for rec in hier_collection.aggregate(pipeline): - ids.add(rec['node']) - ids |= set(rec['ancestors']) - return ids - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_ancestor_ids_versioned(as_of, - nodes=None, - include_hidden=False, - hidden_only=False, - drilldown=True, - signature='', - db=None, - period=None, - service=None): - """ - fetch set of node ids of ancestors of nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - - Keyword Arguments: - nodes {list} -- nodes ['0050000FLN2C9I2'] - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - hidden_only {bool} -- if True, include only hidden nodes and their ancestors - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - signature {str} -- if provided, fetch only hidden nodes with how - matching signature, (default: {''}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - set -- {node ids} - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown, service=service) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - criteria = {'$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - if nodes: - node_criteria = {'node': {'$in': list(nodes)}} - match = merge_dicts(criteria, node_criteria) - else: - match = {k: v for k, v in criteria.items()} - - if signature: - match['how'] = signature - - if hidden_only: - hidden_only_criteria = {'$or': [ - {'$and': [{'hidden_from': {'$lte': from_date}}, - {"$or": [{"hidden_to": {"$exists": False}}, {"hidden_to": None}, - {'hidden_to': {'$gte': to_date}} - ]}, - ] - }] - } - match = {'$and': [criteria, hidden_only_criteria]} - - lookup = {'from': coll, - 'startWith': '$parent', - 'connectFromField': 'parent', - 'connectToField': 'node', - 'depthField': 'level', - 'restrictSearchWithMatch': criteria, - 'as': 'ancestors'} - project = {'_id': 0, - 'ancestors': '$ancestors.node', - 'node': 1} - - pipeline = [{'$match': match}, - {'$graphLookup': lookup}, - {'$project': project}] - - ids = set() - for rec in hier_collection.aggregate(pipeline): - ids.add(rec['node']) - ids |= set(rec['ancestors']) - return ids - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_labels(as_of, - nodes, - include_hidden=False, - drilldown=True, - db=None, - period=None, - boundary_dates=None, - service=None - ): - """ - fetch labels of nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - node {list} -- list of nodes to find names for ['0050000FLN2C9I2', ] - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - dict -- mapping from node to label - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - as_of = 0 - - criteria = {'node': {'$in': nodes}, - '$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ]}, # Checking for the node which is not marked as hidden earlier - {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state - {'hidden_to': 0}] # Already Node has been unblocked from hidden state - } - criteria = {'$and': [criteria, hide_criteria]} - - return {node['node']: node['label'] for node in hier_collection.find(criteria, {'node': 1, 'label': 1, '_id': 0})} - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_labels_versioned(as_of, - nodes, - include_hidden=False, - drilldown=True, - db=None, - period=None, - boundary_dates=None, - service=None - ): - """ - fetch labels of nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - node {list} -- list of nodes to find names for ['0050000FLN2C9I2', ] - - Keyword Arguments: - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - dict -- mapping from node to label - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - if is_lead_service(service): - coll = HIER_LEADS_COLL if not drilldown else DRILLDOWN_LEADS_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if boundary_dates: - from_date, to_date = boundary_dates - else: - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown, service=service) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - criteria = {'node': {'$in': nodes}, - '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - return {node['node']: node['label'] for node in hier_collection.aggregate([{'$match':criteria}, {'$project':{'node': 1, 'label': 1, '_id': 0}}])} - - -def find_closest_node(node, - period, - drilldown=True, - db=None - ): - """ - find the node nearest to the given node that a user has access to - - Args: - node {str} -- node '0050000FLN2C9I2' - period {str} -- period mnemonic '2020Q2' - - Keyword Arguments: - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - str: hierarchy node - """ - user_nodes = get_user_permissions(sec_context.get_effective_user(), 'results') - - # global access, can see all nodes - if '*' in user_nodes or node in user_nodes: - return node - - as_of = get_period_as_of(period) - ancestors = fetch_ancestors(as_of, [node], drilldown=drilldown, db=db, period=period).next()['ancestors'] - - # user has access to an ancestor of the node, can see given node - if any(user_node in ancestors for user_node in user_nodes): - return node - - for nodes in fetch_descendants(as_of, [node], levels=10, drilldown=drilldown, db=db, period=period).next()['descendants']: - # breadth first traverse down subtree of given node - # return descendant of given node if user has access to it - try: - return next(desc_node for desc_node, user_node in product(nodes, user_nodes) if desc_node == user_node) - except StopIteration: - pass - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def node_is_valid(node, - period, - drilldown=True, - db=None, - ): - """ - check if a node is a real live node - - Arguments: - node {str} -- node '0050000FLN2C9I2' - period {str} -- period mnemonic '2020Q2' - - Keyword Arguments: - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - bool -- True if valid - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - as_of = 0 - - criteria = {'node': node, - '$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ]}, # Checking for the node which is not marked as hidden earlier - {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state - {'hidden_to': 0}] # Already Node has been unblocked from hidden state - } - criteria = {'$and': [criteria, hide_criteria]} - - try: - hier_collection.find(criteria, {'_id': 0}).next() - return True - except StopIteration: - return False - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def node_is_valid_versioned(node, - period, - drilldown=True, - db=None, - ): - """ - check if a node is a real live node - - Arguments: - node {str} -- node '0050000FLN2C9I2' - period {str} -- period mnemonic '2020Q2' - - Keyword Arguments: - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - bool -- True if valid - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - - criteria = {'node': node, - '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - try: - hier_collection.find(criteria, {'_id': 0}).next() - return True - except StopIteration: - return False - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def node_exists(node, - period, - drilldown=True, - db=None, - ): - """ - check if a node exists in the hierarchy, dead or alive - - Arguments: - node {str} -- node '0050000FLN2C9I2' - period {str} -- period mnemonic '2020Q2' - - Keyword Arguments: - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - bool -- True if exists - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - as_of = get_period_as_of(period) - - criteria = {'node': node, - '$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - try: - hier_collection.find(criteria, {'_id': 0}).next() - return True - except StopIteration: - return False - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def node_exists_versioned(node, - period, - drilldown=True, - db=None, - ): - """ - check if a node exists in the hierarchy, dead or alive - - Arguments: - node {str} -- node '0050000FLN2C9I2' - period {str} -- period mnemonic '2020Q2' - - Keyword Arguments: - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - bool -- True if exists - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - - criteria = {'node': node, - '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - try: - hier_collection.find(criteria, {'_id': 0}).next() - return True - except StopIteration: - return False - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_top_level_nodes(as_of, - levels=1, - include_hidden=False, - include_children=False, - drilldown=True, - db=None, - period=None - ): - """ - fetch descendants of many hierarchy nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - nodes {list} -- nodes to find descendants for ['0050000FLN2C9I2', ] - - Keyword Arguments: - levels {int} -- how many levels of tree to traverse down - (default: {1}) - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - include_children {bool} -- if True, fetch children of nodes in nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - generator -- (nodes) - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - as_of = 0 - - criteria = {'$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ]}, # Checking for the node which is not marked as hidden earlier - {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state - {'hidden_to': 0}] # Already Node has been unblocked from hidden state - } - criteria = {'$and': [criteria, hide_criteria]} - - match = merge_dicts(criteria, {'parent': None}) - - lookup = {'from': coll, - 'startWith': '$node', - 'connectFromField': 'node', - 'connectToField': 'parent', - 'restrictSearchWithMatch': criteria, - 'maxDepth': levels - 1, - 'as': 'descendants'} - project = {'node': 1, - '_id': 0, - 'descendants': '$descendants.node'} - pipeline = [{'$match': match}, - {'$graphLookup': lookup}, - {'$project': project}, - ] - for node_rec in hier_collection.aggregate(pipeline): - yield node_rec['node'] - for node in node_rec['descendants']: - yield node - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_top_level_nodes_versioned(as_of, - levels=1, - include_hidden=False, - include_children=False, - drilldown=True, - db=None, - period=None - ): - """ - fetch descendants of many hierarchy nodes - - Arguments: - as_of {int} -- epoch timestamp to fetch data as of 1556074024910 - nodes {list} -- nodes to find descendants for ['0050000FLN2C9I2', ] - - Keyword Arguments: - levels {int} -- how many levels of tree to traverse down - (default: {1}) - include_hidden {bool} -- if True, include hidden nodes - (default: {False}) - include_children {bool} -- if True, fetch children of nodes in nodes - (default: {False}) - drilldown {bool} -- if True, fetch drilldown node instead of hierarchy - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - generator -- (nodes) - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - criteria = {'$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - match = merge_dicts(criteria, {'parent': None}) - - lookup = {'from': coll, - 'startWith': '$node', - 'connectFromField': 'node', - 'connectToField': 'parent', - 'restrictSearchWithMatch': criteria, - 'maxDepth': levels - 1, - 'as': 'descendants'} - project = {'node': 1, - '_id': 0, - 'descendants': '$descendants.node'} - pipeline = [{'$match': match}, - {'$graphLookup': lookup}, - {'$project': project}, - ] - for node_rec in hier_collection.aggregate(pipeline): - yield node_rec['node'] - for node in node_rec['descendants']: - yield node - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_owner_id_nodes(as_of, - include_hidden=False, - drilldown=True, - db=None, - period=None): - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - as_of = 0 - - # TODO: only safe for salesforce right now... - pattern = re.compile("^005") if not drilldown else re.compile("(?<=#)005", re.MULTILINE) - criteria = {'node': {'$regex': pattern}, - '$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ]}, # Checking for the node which is not marked as hidden earlier - {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state - {'hidden_to': 0}] # Already Node has been unblocked from hidden state - } - criteria = {'$and': [criteria, hide_criteria]} - - return hier_collection.find(criteria, {'node': 1, '_id': 0, 'label': 1}) - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_owner_id_nodes_versioned(as_of, - include_hidden=False, - drilldown=True, - db=None, - period=None): - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of - - - # TODO: only safe for salesforce right now... - pattern = re.compile("^005") if not drilldown else re.compile("(?<=#)005", re.MULTILINE) - criteria = {'node': {'$regex': pattern}, - '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] - } - criteria = {'$and': [criteria, hide_criteria]} - - return hier_collection.find(criteria, {'node': 1, '_id': 0, 'label': 1}) - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def fetch_non_owner_id_nodes(as_of, - include_hidden=False, - drilldown=True, - db=None, - period=None): - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - - as_of = 0 - - # TODO: only safe for salesforce right now... - pattern = re.compile("^(?!005).*", re.MULTILINE) if not drilldown else re.compile("#(?!005)(?:.(?!#))+$", re.MULTILINE) - criteria = {'node': {'$regex': pattern}, - '$and': [ - {'$or': [{'from': None}, - {'from': {'$lte': as_of}}]}, - {'$or': [{'to': None}, - {'to': {'$gte': as_of}}]} - ]} - if not include_hidden: - hide_criteria = {'$or': [ - {"$and": [ - {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, - {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ]}, # Checking for the node which is not marked as hidden earlier - {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state - {'hidden_to': 0}] # Already Node has been unblocked from hidden state - } - criteria = {'$and': [criteria, hide_criteria]} - - return hier_collection.find(criteria, {'node': 1, '_id': 0, 'label': 1}) - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def fetch_non_owner_id_nodes_versioned(as_of, - include_hidden=False, - drilldown=True, - db=None, - period=None): + Returns: + generator -- (nodes) + """ coll = HIER_COLL if not drilldown else DRILLDOWN_COLL hier_collection = db[coll] if db else sec_context.tenant_db[coll] - if period: - as_of = get_period_as_of(period) - from_date, _ = fetch_boundry(as_of, drilldown=drilldown) - _, to_date = get_period_begin_end(period) - else: - from_date = to_date = as_of + as_of = 0 - # TODO: only safe for salesforce right now... - pattern = re.compile("^(?!005).*", re.MULTILINE) if not drilldown else re.compile("#(?!005)(?:.(?!#))+$", re.MULTILINE) - criteria = {'node': {'$regex': pattern}, - '$and': [ - {"$or": [ - {"$and": [ - {'from': {'$lte': from_date}}, - {'$or': [ - {'to': None}, - {'to': {'$gte': to_date}} - ] - }] - }, - {"$and": [ - {'from': {'$gt': from_date}}, - {'to': {'$lte': to_date}} - ] - } - ]} + criteria = {'$and': [ + {'$or': [{'from': None}, + {'from': {'$lte': as_of}}]}, + {'$or': [{'to': None}, + {'to': {'$gte': as_of}}]} ]} if not include_hidden: hide_criteria = {'$or': [ {"$and": [ {"$or": [{'hidden_from': {"$exists": False}}, {'hidden_from': None}]}, {"$or": [{'hidden_to': {"$exists": False}}, {'hidden_to': None}]} - ] - }, # Checking for the node which is not marked as hidden earlier - {'hidden_to': {'$lte': to_date}} # Already Node has been unblocked from hidden state - ] + ]}, # Checking for the node which is not marked as hidden earlier + {"$and": [{'hidden_from': 0}, {"hidden_to": 0}]}, # Already Node has been unblocked from hidden state + {'hidden_to': 0}] # Already Node has been unblocked from hidden state } criteria = {'$and': [criteria, hide_criteria]} - return hier_collection.find(criteria, {'node': 1, '_id': 0, 'label': 1}) - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in versioned function as well if required -@versioned_func_switch -def validate_node_for_user(user, - as_of, - node=None, - drilldown=True, - db=None, - period=None - ): - user_nodes = get_user_permissions(user, 'results') - if not node: - if '*' in user_nodes: - return True - raise AccessError() - - try: - ancestors = fetch_ancestors(as_of, [node], drilldown=drilldown, db=db).next()['ancestors'] - if '*' in user_nodes or node in user_nodes: - return True - if any(user_node in ancestors for user_node in user_nodes): - return True - except StopIteration: - # given a totally bogus node - raise NodeDoesntExistError() - - raise AccessError() - - -# This function is for a non-versioned tenant -# TODO: Any changes to this function should be made in non-versioned function as well if required -def validate_node_for_user_versioned(user, - as_of, - node=None, - drilldown=True, - db=None, - period=None - ): - user_nodes = get_user_permissions(user, 'results') - if not node: - if '*' in user_nodes: - return True - raise AccessError() - - try: - ancestors = fetch_ancestors(as_of, [node], drilldown=drilldown, db=db, period=period).next()['ancestors'] - if '*' in user_nodes or node in user_nodes: - return True - if any(user_node in ancestors for user_node in user_nodes): - return True - except StopIteration: - # given a totally bogus node - raise NodeDoesntExistError() + match = merge_dicts(criteria, {'parent': None}) - raise AccessError() + lookup = {'from': coll, + 'startWith': '$node', + 'connectFromField': 'node', + 'connectToField': 'parent', + 'restrictSearchWithMatch': criteria, + 'maxDepth': levels - 1, + 'as': 'descendants'} + project = {'node': 1, + '_id': 0, + 'descendants': '$descendants.node'} + pipeline = [{'$match': match}, + {'$graphLookup': lookup}, + {'$project': project}, + ] + for node_rec in hier_collection.aggregate(pipeline): + yield node_rec['node'] + for node in node_rec['descendants']: + yield node def fetch_closest_boundaries(as_of, @@ -4344,2721 +1696,3 @@ def _anc_yielder(nodes, exclude_admin_root=False): else: node['ancestors'] = [x['node'] for x in sorted(node['ancestors'], key=lambda x: x.get('level'), reverse=True)][1:] yield node - - -def _anc_yielder_versioned(nodes, exclude_admin_root=False): - if not nodes.alive: - yield {} - for node in nodes: - unique_ancestor_nodes = [] - for ancestor in node['ancestors']: - if ancestor not in unique_ancestor_nodes: - unique_ancestor_nodes.append(ancestor) - node['ancestors'] = unique_ancestor_nodes - if not exclude_admin_root: - node['ancestors'] = [x['node'] for x in sorted(node['ancestors'], key=lambda x: x.get('level'), reverse=True)] - else: - node['ancestors'] = [x['node'] for x in sorted(node['ancestors'], key=lambda x: x.get('level'), reverse=True)][1:] - yield node - - -class AccessError(Exception): - pass - - -class NodeDoesntExistError(Exception): - pass - - -# TODO: change implementation by giving level as 20 -def fetch_leaves_of_node(node, period=None): - """ - Note: Not efficient. Just using till a better implementaion comes along - """ - parents = {node} - period = period if period else get_current_period() - as_of = get_period_as_of(period) - leaves = set() - while parents: - all_descendants = list(fetch_descendants(as_of, list(parents), 1, period=period)) - node_parents = set() - children = set() - for node_descendants in all_descendants: - - curr_node_descendants = node_descendants['descendants'][0] - # print(curr_node_descendants) - children = children.union(set(curr_node_descendants.keys())) - node_parents = node_parents.union( - set(curr_node_descendants.values())) - leaves = leaves.union(parents - node_parents) - parents = children - return list(leaves) - - -def get_results_using_query(query, return_fields={}, drilldown=False, db=None): - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - if return_fields: - return hier_collection.find(query, return_fields) - return hier_collection.find(query) - - -def get_distinct_using_query(query, distinct_field, drilldown=False, db=None): - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = db[coll] if db else sec_context.tenant_db[coll] - records = hier_collection.distinct(distinct_field, query) - return records - -def find_all_subtree_height(as_of, root_node, leaf_level): - descendants = fetch_descendants(as_of, [root_node], leaf_level, include_children=False) - descendants = [(rec['node'], rec['descendants']) for rec in descendants] - levels_order_tree = descendants[0][1] - - if isinstance(levels_order_tree, dict): - levels_order_tree = [levels_order_tree] - elif isinstance(levels_order_tree, tuple): - levels_order_tree = list(levels_order_tree) - - levels_order_tree = [d for d in levels_order_tree if d] - height_sub_tree = {descendants[0][0]:0} - - for tree_level in reversed(levels_order_tree): - for node_id, parent_id in tree_level.items(): - height = height_sub_tree.get(node_id) - if height is None : - height_sub_tree[node_id] =0 - height_sub_tree[parent_id] = max(height_sub_tree.get(parent_id, 0), 1 + height_sub_tree[node_id]) - - return height_sub_tree - - -def find_levels_in_tree_from_top_to_down(as_of, root_node, leaf_level = 15): - descendants = fetch_descendants(as_of, [root_node], leaf_level, include_children=False) - descendants = [(rec['node'], rec['descendants']) for rec in descendants] - levels_order_tree = descendants[0][1] - - if isinstance(levels_order_tree, dict): - levels_order_tree = [levels_order_tree] - elif isinstance(levels_order_tree, tuple): - levels_order_tree = list(levels_order_tree) - - levels_order_tree = [d for d in levels_order_tree if d] - node_level_map = {descendants[0][0]:0} - - for tree_level in levels_order_tree: - for node_id, parent_id in tree_level.items(): - parent_height = node_level_map.get(parent_id, 0) - node_level_map[node_id] = parent_height+1 - - return node_level_map - - -def find_map_in_nodes_and_lth_grand_children(as_of, root_node, nodes, lth_grand_children, leaf_level = 15): - descendants = fetch_descendants(as_of, [root_node], leaf_level, include_children=False) - descendants = [(rec['node'], rec['descendants']) for rec in descendants] - levels_order_tree = descendants[0][1] - - if isinstance(levels_order_tree, dict): - levels_order_tree = [levels_order_tree] - elif isinstance(levels_order_tree, tuple): - levels_order_tree = list(levels_order_tree) - - levels_order_tree = [d for d in levels_order_tree if d] - node_parent_map = {descendants[0][0]:descendants[0][0]} - - for tree_level in levels_order_tree: - for node_id, parent_id in tree_level.items(): - if node_id in nodes: - node_parent_map[node_id] = node_id - else: - node_parent_map[node_id] = node_parent_map[parent_id] - - map_data = {} - for lth_grand_child in lth_grand_children: - parent = node_parent_map.get(lth_grand_child) - map_data.setdefault(parent, []).append(lth_grand_child) - - return map_data - - -def fetch_first_line_managers(as_of, root_node, leaf_level): - return fetch_hier_level_line_managers(as_of, root_node, leaf_level, hier_level=1) - -def fetch_second_line_managers(as_of, root_node, leaf_level): - return fetch_hier_level_line_managers(as_of, root_node, leaf_level, hier_level=2) - - -def fetch_hier_level_line_managers(as_of, root_node, leaf_level, hier_level=1): - first_line_managers = [] - height_sub_tree = find_all_subtree_height(as_of, root_node, leaf_level) - for node_id, height in height_sub_tree.items(): - if height == hier_level: - first_line_managers.append(node_id) - return first_line_managers - -def _prev_periods_fallback(config, prev_periods): - if config.config.get('update_new_collection'): - prev_periods = prev_periods or prev_periods_allowed_in_deals() - return prev_periods - - -def is_second_level_manager_and_above(as_of, root_node, leaf_level, node): - first_line_managers = [] - height_sub_tree = find_all_subtree_height(as_of, root_node, leaf_level) - for node_id, height in height_sub_tree.items(): - if height >= 2: - first_line_managers.append(node_id) - return node in first_line_managers - -def _fetch_favorites(period, user, db=None): - favs_collection = db[FAVS_COLL] if db else sec_context.tenant_db[FAVS_COLL] - - criteria = {'user': user, 'fav': True, 'period': period} - - return {x['opp_id'] for x in favs_collection.find(criteria, {'opp_id': 1, '_id': 0})} - -def fetch_deal_totals(period_and_close_periods, - node, - fields_and_operations, - config, - user=None, - filter_criteria=None, - filter_name=None, - favorites=None, - timestamp=None, - db=None, - cache=None, - return_seg_info=False, - dlf_node=None, - search_criteria=None, - criteria=None, - sort_fields=[], - custom_limit=None, - prev_periods=[], - sfdc_view=False, - actual_view=False, - nodes=None - ): - """ - fetch total and count of deals for period and node - - Arguments: - period_and_close_periods {tuple} -- as of mnemonic, [close mnemonics] ('2020Q2', ['2020Q2']) - node {str} -- hierarchy node '0050000FLN2C9I2' - fields_and_operations {list} -- tuples of label, deal field, [('Amount', 'Amount', '$sum')] - mongo op to total - config {DealsConfig} -- instance of DealsConfig ...? - - Keyword Arguments: - user {str} -- user name, if None, will use sec_context 'gnana' - filter_criteria {dict} -- mongo db filter criteria (default: {None}) {'Amount': {'$gte': 100000}} - filter_name {str} -- label to identify filter in cache - favorites {set} -- set of opp_ids that have been favorited by user - if None, queries favs collectionf to find them - (default: {None}) - timestamp {float} -- epoch timestamp for when deal records expire - if None, checks flags to find it - (default: {None}) - db {pymongo.database.Database} -- instance of tenant_db - (default: {None}) - if None, will create one - cache {dict} -- dict to hold records fetched by func (default: {None}) - (used to memoize fetching many recs) - - Returns: - dict -- 'count' key and total for each total field - """ - - # Below line created a problem in threading, this fall back funtions calculation should be passed from outside - quarter_collection = db[QUARTER_COLL] if db else sec_context.tenant_db[QUARTER_COLL] - prev_periods = _prev_periods_fallback(config, prev_periods) - - user = user or sec_context.login_user_name - hint_field = 'period_-1_close_period_-1_drilldown_list_-1_is_deleted_-1' if '#' in node \ - else 'period_-1_close_period_-1_hierarchy_list_-1_update_date_-1' - if not isinstance(period_and_close_periods, list): - period_and_close_periods = [period_and_close_periods] - - period_and_close_periods_map = {"old_deals_coll": [], "new_deals_coll": []} - deals_collection_map = {} - periods = [] - for period_and_close_period_for_db in period_and_close_periods: - period, _ = period_and_close_period_for_db - periods.append(period) - if ((period in prev_periods and config.config.get('update_new_collection')) or - (isinstance(period, list) and not set(period).isdisjoint(prev_periods) and - config.config.get('update_new_collection')) or - period is None): - period_and_close_periods_map['new_deals_coll'].append(period_and_close_period_for_db) - deals_collection = db[NEW_DEALS_COLL] if db else \ - sec_context.tenant_db[NEW_DEALS_COLL] - deals_collection_map['new_deals_coll'] = deals_collection - collection_name = NEW_DEALS_COLL - else: - period_and_close_periods_map['old_deals_coll'].append(period_and_close_period_for_db) - deals_collection = db[DEALS_COLL] if db else sec_context.tenant_db[DEALS_COLL] - deals_collection_map['old_deals_coll'] = deals_collection - collection_name = DEALS_COLL - - moved_deals_in_actual_view = False - if actual_view: - objectids = [] - moveddeals = list(quarter_collection.find({'period': {"$in" : periods}})) - if len(moveddeals) > 0: objectids = list(map(ObjectId, moveddeals[0]['ObjectId'])) - if len(objectids) > 0: moved_deals_in_actual_view = True - - deals_collection_criteria = {} - for period_map in period_and_close_periods_map.keys(): - match_list = [] - if (period_map == 'old_deals_coll' and period_and_close_periods_map['old_deals_coll']) or \ - (period_map == 'new_deals_coll' and period_and_close_periods_map['new_deals_coll']): - pass - else: - continue - - period_and_close_periods = period_and_close_periods_map[period_map] - - for period_and_close_period in period_and_close_periods: - period, close_periods = period_and_close_period - if isinstance(favorites, dict): - favs = favorites.get(period, None) - if not favs: - favs = _fetch_favorites(period, user, db) - else: - favs = favorites or _fetch_favorites(period, user, db) - if isinstance(timestamp, dict): - timestamp = timestamp.get(period, None) - if not timestamp: - timestamp = _get_timestamp(period) - else: - timestamp = timestamp or _get_timestamp(period) # BUG: cant get timestamp when threaded from fm svc - - hier_field = 'drilldown_list' if '#' in node else 'hierarchy_list' - - if not isinstance(period, list): - if not moved_deals_in_actual_view: - match = {'period': period, hier_field: {'$in': [node]}} - else: - match = {hier_field: {'$in': [node]}} - if criteria: - match.update(criteria) - if timestamp: - match['update_date'] = {'$gte': timestamp} - else: - match['is_deleted'] = False - - if close_periods: - period_key = get_key_for_close_periods(close_periods) - hint_field = modify_hint_field(close_periods, hint_field) - if not moved_deals_in_actual_view: - match[period_key] = {'$in': close_periods} - else: - closeperiods_value = close_periods - else: - match = {'$or': []} - for temp_num in range(len(period)): - if not moved_deals_in_actual_view: - match1 = {'period': period[temp_num], hier_field: {'$in': [node]}} - else: - match1 = {hier_field: {'$in': [node]}} - if criteria: - match1.update(criteria) - if timestamp: - match1['update_date'] = {'$gte': timestamp} - else: - match1['is_deleted'] = False - - if close_periods: - period_key = get_key_for_close_periods(close_periods) - if not moved_deals_in_actual_view: - match1[period_key] = {'$in': close_periods[temp_num]} - else: - closeperiods_value = close_periods - match['$or'] += [match1] - - if filter_criteria: - match = { - '$and': [match, contextualize_filter(filter_criteria, dlf_node if dlf_node else node, favs, period)]} - - if search_criteria: - search_terms, search_fields = search_criteria - match['$or'] = [{field: {'$regex': '|'.join(search_terms), '$options': 'i'}} for field in search_fields] - - match_list.append(match) - - if not period_and_close_periods: - period = {} - favs = {} - hier_field = 'drilldown_list' if '#' in node else 'hierarchy_list' - match = {hier_field: {'$in': [node]}} - if criteria: - match.update(criteria) - match['is_deleted'] = False - if filter_criteria: - match = { - '$and': [match, contextualize_filter(filter_criteria, dlf_node if dlf_node else node, favs, period)]} - if search_criteria: - search_terms, search_fields = search_criteria - match['$or'] = [{field: {'$regex': '|'.join(search_terms), '$options': 'i'}} for field in search_fields] - match_list.append(match) - logger.info("Match List %s", match) - - match = {'$or': []} - if len(match_list) > 1: - for m in match_list: - match['$or'].append(m) - else: - match = match_list[0] - - if actual_view: - match['is_moved'] = {'$exists': False} - - if moved_deals_in_actual_view: - actual_view_criteria = [{period_key: {'$in': closeperiods_value}, 'period': period}, - {'_id': {'$in': objectids}}] - if '$or' in match.keys(): - existing_or = match.pop('$or') - match['$and'] = [{'$or': existing_or}, {'$or': actual_view_criteria}] - elif '$and' in match.keys(): - match['$and'].append({'$or': actual_view_criteria}) - else: - match['$or'] = actual_view_criteria - deals_collection_criteria[period_map] = match - - project = {contextualize_field(fld, config, dlf_node if dlf_node else node): 1 for _, fld, _ in - fields_and_operations} - project['update_date'] = 1 - - group = {label: {op: '$' + contextualize_field(fld, config, dlf_node if dlf_node else node)} for label, fld, op in - fields_and_operations} - - group.update({'_id': None, - 'count': {'$sum': 1}, - 'timestamp': {'$last': '$update_date'} - }) - if period_and_close_periods_map['old_deals_coll'] and period_and_close_periods_map['new_deals_coll']: - collection_name = NEW_DEALS_COLL + " & " + DEALS_COLL - deals_collection = deals_collection_map['new_deals_coll'] - match1 = deals_collection_criteria['new_deals_coll'] - match2 = deals_collection_criteria['old_deals_coll'] - pipeline = [{'$match': match1}, - {'$unionWith': {'coll': 'deals', - 'pipeline': [{'$match': match2}]}}, - {'$project': project}, - {'$group': group}] - else: - pipeline = [{'$match': match}, - {'$project': project}, - {'$group': group}] - - if custom_limit: - if sort_fields: - if period_and_close_periods_map['old_deals_coll'] and period_and_close_periods_map['new_deals_coll']: - deals_collection = deals_collection_map['new_deals_coll'] - match1 = deals_collection_criteria['new_deals_coll'] - match2 = deals_collection_criteria['old_deals_coll'] - pipeline = [{'$match': match1}, - {'$unionWith': {'coll': 'deals', - 'pipeline': [{'$match': match2}]}}, - {'$project': project}, - {'$sort': OrderedDict( - [(contextualize_field(field, config, node), direction) for - field, direction in sort_fields])}, - {'$limit': custom_limit}, - {'$group': group}] - else: - pipeline = [{'$match': match}, - {'$project': project}, - {'$sort': OrderedDict( - [(contextualize_field(field, config, node), direction) for - field, direction in sort_fields])}, - {'$limit': custom_limit}, - {'$group': group}] - else: - if period_and_close_periods_map['old_deals_coll'] and period_and_close_periods_map['new_deals_coll']: - deals_collection = deals_collection_map['new_deals_coll'] - match1 = deals_collection_criteria['new_deals_coll'] - match2 = deals_collection_criteria['old_deals_coll'] - pipeline = [{'$match': match1}, - {'$unionWith': {'coll': 'deals', - 'pipeline': [{'$match': match2}]}}, - {'$project': project}, - {'$limit': custom_limit}, - {'$group': group}] - else: - pipeline = [{'$match': match}, - {'$project': project}, - {'$limit': custom_limit}, - {'$group': group}] - - if return_seg_info: - if config.segment_field: - seg_field = config.segment_field - group['_id'] = '$' + str(seg_field) - project.update({seg_field: 1}) - else: - if config.debug: - logger.error("segment_field config not there in deal_svc config") - - if sfdc_view: - sort = {'update_time': 1} - project['opp_id'] = 1 - pregroup = {label: {"$last": '$' + contextualize_field(fld, config, dlf_node if dlf_node else node)} for - label, fld, _ in fields_and_operations} - pregroup.update({'_id': '$opp_id', - 'timestamp': {'$last': '$update_date'} - }) - group = {label: {op: '$' + label} for label, fld, op in fields_and_operations} - group.update({'_id': None, - 'count': {'$sum': 1}, - 'timestamp': {'$last': '$timestamp'} - }) - if period_and_close_periods_map['old_deals_coll'] and period_and_close_periods_map['new_deals_coll']: - deals_collection = deals_collection_map['new_deals_coll'] - match1 = deals_collection_criteria['new_deals_coll'] - match2 = deals_collection_criteria['old_deals_coll'] - pipeline = [{'$match': match1}, - {'$unionWith': {'coll': 'deals', - 'pipeline': [{'$match': match2}]}}, - {'$project': project}, - {'$sort': sort}, - {'$group': pregroup}, - {'$group': group}] - else: - pipeline = [{'$match': match}, - {'$project': project}, - {'$sort': sort}, - {'$group': pregroup}, - {'$group': group}] - - # AV-14059 Commenting below as part of the log fix - # Reason: Cannot process this log everytime as ingestion is high and utility is low - # logger.info('fetch_deal_totals collection_name %s, filter_name: %s, pipeline: %s', collection_name, - # filter_name, pipeline) - - aggs = list(deals_collection.aggregate(pipeline=pipeline, allowDiskUse=True, hint=hint_field)) - - aggregated_val = {} - if return_seg_info: - for res in aggs: - val = {} - for k, v in res.items(): - if k != '_id': - val[k] = v - if k not in aggregated_val: - aggregated_val[k] = v - else: - aggregated_val[k] += v - if config.segment_field: - cache[(filter_name, res['_id'], node)] = val - cache[(filter_name, 'all_deals', node)] = aggregated_val - return cache - else: - try: - val = {k: v for k, v in aggs[0].items() if k != '_id'} - except IndexError: - val = {label: 0 for label, _, _ in fields_and_operations} - val['count'] = 0 - val['timestamp'] = None - - if cache is not None: - cache[(filter_name, node)] = val - - return val - -def fetch_crr_deal_rollup(period_and_close_periods, - nodes, - fields_and_operations, - config, - user=None, - filter_criteria=None, - filter_name=None, - favorites=None, - timestamp=None, - db=None, - cache=None, - return_seg_info=False, - filters=[], - root_node=None, - use_dlf_fcst_coll=False - ): - deals_collection = db[GBM_CRR_COLL] if db else sec_context.tenant_db[GBM_CRR_COLL] - user = user or sec_context.login_user_name - - if not isinstance(period_and_close_periods, list): - period_and_close_periods = [period_and_close_periods] - hier_aware = False - match_list = [] - match_list2 = [] - for _, fld, _ in fields_and_operations: - if fld in config.hier_aware_fields and fld != 'forecast': - hier_aware = True - for period_and_close_period in period_and_close_periods: - period, close_periods = period_and_close_period - if isinstance(favorites, dict): - favs = favorites.get(period, None) - if not favs: - favs = _fetch_favorites(period, user, db) - else: - favs = favorites or _fetch_favorites(period, user, db) - - if isinstance(timestamp, dict): - timestamp = timestamp.get(period, None) - if not timestamp: - timestamp = _get_timestamp(period) - else: - timestamp = timestamp or _get_timestamp(period) # BUG: cant get timestamp when threaded from fm svc - - hier_field = '__segs' - hint_field = 'monthly_period_-1___segs_-1_last_modified_-1' - - match = {'period': period, - hier_field: {'$in': nodes}} - if timestamp: - match['last_modified'] = {'$gte': timestamp} - - if close_periods: - match['monthly_period'] = {'$in': close_periods} - if filter_criteria: - if "%(node)s" in str(filter_criteria): - matches = None - for node in nodes: - if matches is None: - matches = [contextualize_filter(filter_criteria, node, favs, period)] - else: - matches.append(contextualize_filter(filter_criteria, node, favs, period)) - match = {'$and': [match, {'$or': matches}]} - else: - match = {'$and': [match, contextualize_filter(filter_criteria, nodes[0], favs, period)]} - match_list.append(match) - if not hier_aware: - match2 = {hier_field: {'$in': nodes}} - match_list2.append(match2) - - match = {'$or': []} - if len(match_list) > 1: - for m in match_list: - match['$or'].append(m) - else: - match = match_list[0] - - if not hier_aware: - match2 = {'$or': []} - if len(match_list2) > 1: - for m in match_list2: - match2['$or'].append(m) - else: - match2 = match_list2[0] - - - group = {} - regroup = {} - if hier_aware: - project = {contextualize_field(fld, config, node): 1 for node in nodes for _, fld, _ in fields_and_operations} - group['_id'] = None - group.update( - {label + node.replace('.', '$'): {op: '$' + contextualize_field(fld, config, node)} for node in nodes for - label, fld, op in fields_and_operations}) - else: - if len(nodes) == 1 or root_node is None: - root_node = nodes[0] - project = {contextualize_field(fld, config, root_node, hier_aware=False if 'forecast' in fld else True): 1 for - _, fld, _ in fields_and_operations} - - group['_id'] = { - label: '$' + contextualize_field(fld, config, root_node, hier_aware=False if 'forecast' in fld else True) - for label, fld, op in fields_and_operations} - group['_id'][hier_field] = '$' + hier_field - group.update({hier_field: {'$addToSet': '$' + hier_field}}) - regroup = {label: {op: '$_id.' + label} for label, fld, op in fields_and_operations} - regroup['_id'] = {hier_field: '$_id.' + hier_field} - regroup.update({'count': {'$sum': 1}}) - - if hier_aware: - group.update({'count': {'$sum': 1}}) - pipeline = [{'$match': match}, - {'$project': project}, - {'$group': group}] - else: - project.update({'RPM_ID': 1, hier_field: 1}) - group['_id']['RPM_ID'] = '$RPM_ID' - pipeline = [{'$match': match}, - {'$project': project}, - {'$unwind': "$" + hier_field}, - {'$match': match2}, - {'$group': group}, - {'$unwind': "$" + hier_field}, - {'$group': regroup}] - - if config.debug: - logger.info('fetch_deal_rollup filter_name: %s, pipeline: %s', filter_name, pipeline) - - aggs = list(deals_collection.aggregate(pipeline, allowDiskUse=True)) - - for node in nodes: - aggregated_val = {} - for res in aggs: - val = {} - segment = None - drilldown = None - if res['_id']: - segment = res['_id'].get("segment", None) - drilldown = res['_id'].get(hier_field, None) - if not hier_aware: - if node != drilldown: - continue - for k, v in res.items(): - if k != '_id': - if hier_aware: - if k.endswith(node.replace('.', '$')): - k = k.split(node.replace('.', '$'))[0] - else: - continue - val[k] = v - if k not in aggregated_val: - aggregated_val[k] = v - else: - aggregated_val[k] += v - if segment and return_seg_info: - cache[(filter_name, segment, node)] = val - else: - cache[(filter_name, node)] = val - if aggregated_val: - if return_seg_info: - cache[(filter_name, 'all_deals', node)] = aggregated_val - else: - cache[(filter_name, node)] = aggregated_val - else: - val = {label: 0 for label, _, _ in fields_and_operations} - val['count'] = 0 - cache[(filter_name, node)] = val - return cache - -def fetch_deal_rollup_dlf_using_df(period_and_close_periods, - nodes, - fields_and_operations, - config, - user=None, - filter_criteria=None, - filter_name=None, - favorites=None, - timestamp=None, - db=None, - cache=None, - return_seg_info=False, - filters=[], - root_node=None, - use_dlf_fcst_coll=False, - is_pivot_special=False, - deals_coll_name=None, - cumulative_fields=[], - cumulative_close_periods=[], - prev_periods=[], - actual_view=False - ): - if is_pivot_special: - return fetch_crr_deal_rollup(period_and_close_periods, - nodes, - fields_and_operations, - config, - user=user, - filter_criteria=filter_criteria, - filter_name=filter_name, - favorites=favorites, - timestamp=timestamp, - db=db, - cache=cache, - return_seg_info=return_seg_info, - filters=filters, - root_node=root_node, - use_dlf_fcst_coll=use_dlf_fcst_coll) - - quarter_collection = db[QUARTER_COLL] if db else sec_context.tenant_db[QUARTER_COLL] - prev_periods = _prev_periods_fallback(config, prev_periods) - if not isinstance(period_and_close_periods, list): - period_and_close_periods = [period_and_close_periods] - period_and_close_periods_map = {"old_deals_coll": [], "new_deals_coll": []} - periods = [] - deals_collection_map = {} - if deals_coll_name: - deals_collection = db[deals_coll_name] if db else sec_context.tenant_db[deals_coll_name] - deals_collection_map['new_deals_coll'] = deals_collection - else: - for period_and_close_period_for_db in period_and_close_periods: - period, _ = period_and_close_period_for_db - periods.append(period) - if ((period in prev_periods and config.config.get('update_new_collection')) or - (isinstance(period, list) and not set(period).isdisjoint(prev_periods) and - config.config.get('update_new_collection')) or - period is None): - period_and_close_periods_map['new_deals_coll'].append(period_and_close_period_for_db) - deals_collection = db[NEW_DEALS_COLL] if db else \ - sec_context.tenant_db[NEW_DEALS_COLL] - deals_collection_map['new_deals_coll'] = deals_collection - collection_name = NEW_DEALS_COLL - else: - period_and_close_periods_map['old_deals_coll'].append(period_and_close_period_for_db) - deals_collection = db[DEALS_COLL] if db else sec_context.tenant_db[DEALS_COLL] - deals_collection_map['old_deals_coll'] = deals_collection - collection_name = DEALS_COLL - - moved_deals_in_actual_view = False - if actual_view: - objectids = [] - #moveddeals = list(quarter_collection.find({'period': periods})) - moveddeals = list(quarter_collection.find({'period': {"$in" : periods}})) - if len(moveddeals) > 0: objectids = list(map(ObjectId, moveddeals[0]['ObjectId'])) - if len(objectids) > 0: moved_deals_in_actual_view = True - - drilldown_nodes = [] - hierarchy_nodes = [] - hier_aware = False - for _, fld, _ in fields_and_operations: - if fld in config.hier_aware_fields: - hier_aware = True - for node in nodes: - if '#' in node: - drilldown_nodes.append(node) - else: - hierarchy_nodes.append(node) - - user = user or sec_context.login_user_name - - for hier_field in ['drilldown_list', 'hierarchy_list']: - if timestamp: - if hier_field == 'drilldown_list': - nodes = drilldown_nodes - hint_field = 'period_-1_close_period_-1_drilldown_list_-1_update_date_-1' - else: - nodes = hierarchy_nodes - hint_field = 'period_-1_close_period_-1_hierarchy_list_-1_update_date_-1' - else: - if hier_field == 'drilldown_list': - nodes = drilldown_nodes - hint_field = 'period_-1_close_period_-1_drilldown_list_-1_is_deleted_-1' - else: - nodes = hierarchy_nodes - hint_field = 'period_-1_close_period_-1_hierarchy_list_-1_is_deleted_-1' - if not nodes: - continue - deals_collection_criteria = {} - for period_map in period_and_close_periods_map.keys(): - if (period_map == 'old_deals_coll' and period_and_close_periods_map['old_deals_coll']) or \ - (period_map == 'new_deals_coll' and period_and_close_periods_map['new_deals_coll']): - pass - else: - continue - - period_and_close_periods = period_and_close_periods_map[period_map] - match_list = [] - for period_and_close_period in period_and_close_periods: - period, close_periods = period_and_close_period - if isinstance(favorites, dict): - favs = favorites.get(period, None) - if not favs: - favs = _fetch_favorites(period, user, db) - else: - favs = favorites or _fetch_favorites(period, user, db) - - if isinstance(timestamp, dict): - timestamp = timestamp.get(period, None) - if not timestamp: - timestamp = _get_timestamp(period) - else: - timestamp = timestamp or _get_timestamp(period) # BUG: cant get timestamp when threaded from fm svc - - if not moved_deals_in_actual_view: - match = {'period': period, - hier_field: {'$in': nodes}} - else: - match = {hier_field: {'$in': nodes}} - - if timestamp: - match['update_date'] = {'$gte': timestamp} - else: - match['is_deleted'] = False - - if close_periods: - period_key = get_key_for_close_periods(close_periods) - hint_field = modify_hint_field(close_periods, hint_field) - if not moved_deals_in_actual_view: - match[period_key] = {'$in': close_periods} - else: - closeperiods_value = close_periods - matches = {} - if filters: - for filt in filters: - op = filt['op'] - if op != 'dlf': - matches.update(parse_filters(filt, config)) - if matches: - match = {'$and': [match, matches]} - match_list.append(match) - match = {'$or': []} - if len(match_list) > 1: - for m in match_list: - match['$or'].append(m) - else: - match = match_list[0] - - if actual_view: - match['is_moved'] = {'$exists': False} - - if moved_deals_in_actual_view: - actual_view_criteria = [{period_key: {'$in': closeperiods_value}, 'period': period}, - {'_id': {'$in': objectids}}] - if '$or' in match.keys(): - existing_or = match.pop('$or') - match['$and'] = [{'$or': existing_or}, {'$or': actual_view_criteria}] - elif '$and' in match.keys(): - match['$and'].append({'$or': actual_view_criteria}) - else: - match['$or'] = actual_view_criteria - deals_collection_criteria[period_map] = match - - amount_field_hierarchy = [] # Capture the hierarchy of amount field to find in the mongo result - filter_field_project_hierarchy = [] # Capture the hierarchy of filter to use from the mongo result - filter_field_match_hierarchy = [] - filter_field_values = [] # Capture the hierarchy of filter values to check for - projected_seg_field = None - - group = {} - project = {'dlf': 1} - hier_aware = False - for label, fld, op in fields_and_operations: - if "%(node)s" in str(filter_criteria): - amount_field_hierarchy.extend(fld.split('.')) # Convert the '.' notation to a hierarchy of elements - for node in nodes: - if not fld.startswith('dlf'): - project.update({contextualize_field(fld, config, node): 1}) - group.update({label + node.replace('.', '$'): {op: '$' + contextualize_field(fld, config, node)}}) - hier_aware = True - else: - amount_field_hierarchy.extend( - fld.split('.')) # Since this is not hier based, add it directly to the list - group.update({label: {op: '$' + contextualize_field(fld, config, nodes[0])}}) - project.update({contextualize_field(fld, config, nodes[0]): 1}) - if filters: - for filt in filters: - op = filt['op'] - if op == 'dlf': - project['dlf'] = 1 # Always get the entire dlf field - value = filt['val'] - filter_field_project_hierarchy.extend(['dlf', value]) # Build the hierarchy for filtering - # project['in_fcst_array'] = {'$objectToArray': '$dlf.' + value} - - matches = [] - for filt in filters: - op = filt['op'] - negate = filt.get('negate', False) - match_vals = filt.get('match_vals') - match_vals = match_vals if match_vals else True - mongo_op = '$in' if not negate else '$nin' - if op == 'dlf': - filter_field_match_hierarchy.extend( - ['%(node)s', 'state']) # Update the filter hierarchy to the last depth - if isinstance(match_vals, list): - filter_field_values.extend(match_vals) # Build the filter values for checking - # matches.append({"in_fcst_array.v.state": {mongo_op: match_vals}}) - else: - if not negate: - filter_field_values.append(True) # Build the filter values for checking - # matches.append({"in_fcst_array.v.state": True}) - else: - filter_field_values.append(False) # Build the filter values for checking - # matches.append({"in_fcst_array.v.state": False}) - else: - continue - # matches.append({"in_fcst_array.k" : {"$in": nodes}}) - - # match2 = {'$and': matches} - - # group['_id'] = {hier_field: "$in_fcst_array.k"} - if return_seg_info: - if config.segment_field: - seg_field = config.segment_field - # if group['_id']: - # group['_id']['segment'] = '$' + str(seg_field) - # else: - # group['_id'] = '$' + str(seg_field) - project.update({seg_field: 1}) - projected_seg_field = seg_field - else: - if config.debug: - logger.error("segment_field config not there in deal_svc config") - - project.update({'_id': 0}) - # group.update({'count': {'$sum': 1}}) - pipeline = [{'$match': match}, - {'$project': project}] - if period_and_close_periods_map['old_deals_coll'] and period_and_close_periods_map['new_deals_coll']: - collection_name = NEW_DEALS_COLL + " & " + DEALS_COLL - deals_collection = deals_collection_map['new_deals_coll'] - new_deals_match = deals_collection_criteria['new_deals_coll'] - old_deals_match = deals_collection_criteria['old_deals_coll'] - pipeline = [{'$match': new_deals_match}, - {'$unionWith': {'coll': 'deals', - 'pipeline': [{'$match': old_deals_match}]}}, - {'$project': project}] - - if config.debug: - logger.info('fetch_deal_rollup_dlf_using_df filter_name: %s, pipeline: %s', filter_name, pipeline) - - aggs = deals_collection.aggregate(pipeline, allowDiskUse=True, hint=hint_field) - - # aggs = deals_collection.aggregate(pipeline, allowDiskUse=True) - - ## Create a data frame to unwind the data ## - - frames = [] - try: - hier_aware_match_filter = True if filter_field_match_hierarchy.index('%(node)s') >= 0 else False - except: - hier_aware_match_filter = False - for record in aggs: - try: - for dlf_node in get_nested(record, filter_field_project_hierarchy): - filter_val = None - - if hier_aware_match_filter: - filter_val = get_nested_with_placeholder(record, - filter_field_project_hierarchy + filter_field_match_hierarchy, - {'node': dlf_node}) - else: - filter_val = get_nested(record, filter_field_project_hierarchy + filter_field_match_hierarchy) - - if filter_val in filter_field_values: - amount_data = get_nested_with_placeholder(record, amount_field_hierarchy, {'node': dlf_node}) - - node_frame_by_state = { - 'amount': amount_data.get(dlf_node, 0) if isinstance(amount_data, dict) else amount_data, - 'node': dlf_node, - 'count': 1 - } - - if projected_seg_field is not None: - node_frame_by_state['segment'] = record[projected_seg_field] - - frames.append(node_frame_by_state) - except KeyError: - pass - - # If the dataframe is empty then there is no need to calculate the totals, its always 0 - - df = pd.DataFrame(frames, - columns=['node', 'amount', 'count']) - - # Now group the nodes by node and segment(if available) - if projected_seg_field is not None: - df_grouped = df.groupby(['node', 'segment']).sum().reset_index() - else: - df_grouped = df.groupby(['node']).sum().reset_index() - - amount_field_label = amount_field_hierarchy[0] if amount_field_hierarchy[0] != 'dlf' else \ - amount_field_hierarchy[1] - - for node in nodes: - aggregated_val = {} - df_by_node = df_grouped.loc[df_grouped['node'] == node] - for index, res in df_by_node.iterrows(): - val = {} - segment = None - if projected_seg_field is not None: - segment = res['segment'] - - val[amount_field_label] = res['amount'] - val['count'] = res['count'] - - if amount_field_label not in aggregated_val: - aggregated_val[amount_field_label] = res['amount'] - aggregated_val['count'] = res['count'] - else: - aggregated_val[amount_field_label] += res['amount'] - aggregated_val['count'] += res['count'] - - if segment and return_seg_info: - cache[(filter_name, segment, node)] = val - else: - cache[(filter_name, node)] = val - if aggregated_val: - if return_seg_info: - cache[(filter_name, 'all_deals', node)] = aggregated_val - else: - cache[(filter_name, node)] = aggregated_val - else: - val = {label: 0 for label, _, _ in fields_and_operations} - val['count'] = 0 - cache[(filter_name, node)] = val - return cache - -def fetch_crr_deal_rollup_dlf(period_and_close_periods, - nodes, - fields_and_operations, - config, - user=None, - filter_criteria=None, - filter_name=None, - favorites=None, - timestamp=None, - db=None, - cache=None, - return_seg_info=False, - filters=[], - root_node=None, - use_dlf_fcst_coll=False - ): - deals_collection = db[GBM_CRR_COLL] if db else sec_context.tenant_db[GBM_CRR_COLL] - user = user or sec_context.login_user_name - if not isinstance(period_and_close_periods, list): - period_and_close_periods = [period_and_close_periods] - hier_aware = False - match_list = [] - match_list2 = [] - for _, fld, _ in fields_and_operations: - if fld in config.hier_aware_fields and fld != 'forecast': - hier_aware = True - for period_and_close_period in period_and_close_periods: - period, close_periods = period_and_close_period - if isinstance(favorites, dict): - favs = favorites.get(period, None) - if not favs: - favs = _fetch_favorites(period, user, db) - else: - favs = favorites or _fetch_favorites(period, user, db) - - if isinstance(timestamp, dict): - timestamp = timestamp.get(period, None) - if not timestamp: - timestamp = _get_timestamp(period) - else: - timestamp = timestamp or _get_timestamp(period) # BUG: cant get timestamp when threaded from fm svc - - hier_field = '__segs' - hint_field = 'monthly_period_-1___segs_-1_last_modified_-1' - - match = {'period': period} - if timestamp: - match['last_modified'] = {'$gte': timestamp} - - if close_periods: - match['monthly_period'] = {'$in': close_periods} - - if filter_criteria: - if "%(node)s" in str(filter_criteria): - matches = None - for node in nodes: - if matches is None: - matches = [contextualize_filter(filter_criteria, node, favs, period)] - else: - matches.append(contextualize_filter(filter_criteria, node, favs, period)) - match = {'$and': [match, {'$or': matches}]} - else: - match = {'$and': [match, contextualize_filter(filter_criteria, nodes[0], favs, period)]} - - matches = {} - if filters: - for filt in filters: - op = filt['op'] - if op != 'dlf': - matches.update(parse_filters(filt, config, is_pivot_special=True)) - if matches: - match = {'$and': [match, matches]} - match_list.append(match) - matches = [] - for filt in filters: - op = filt['op'] - negate = filt.get('negate', False) - match_vals = filt.get('match_vals') - match_vals = match_vals if match_vals else True - mongo_op = '$in' if not negate else '$nin' - if op == 'dlf': - if isinstance(match_vals, list): - matches.append({"crr_in_fcst_array.v.state": {mongo_op: match_vals}}) - else: - if not negate: - matches.append({"crr_in_fcst_array.v.state": True}) - else: - matches.append({"crr_in_fcst_array.v.state": False}) - else: - continue - matches.append({"crr_in_fcst_array.k": {"$in": nodes}}) - - match2 = {'$and': matches} - match_list2.append(match2) - - group = {} - project = {} - hier_aware = False - con_fields_list = [] - for label, fld, op in fields_and_operations: - if "%(node)s" in str(filter_criteria): - for node in nodes: - con_field = contextualize_field(fld, config, node) - con_fields_list.append(con_field) - project.update({con_field: 1}) - group.update({label + node.replace('.', '$'): {op: '$' + contextualize_field(fld, config, node)}}) - hier_aware = True - else: - group.update({label: {op: '$' + contextualize_field(fld, config, nodes[0])}}) - con_field = contextualize_field(fld, config, nodes[0]) - con_fields_list.append(con_field) - project.update({con_field: 1}) - if filters: - for filt in filters: - op = filt['op'] - if op == 'dlf': - value = filt['val'] - project['crr_in_fcst_array'] = {'$objectToArray': '$dlf.' + value} - - - if not filters and con_fields_list and "crr_in_fcst_array" not in project: - for con_fied in con_fields_list: - con_fied_lst = con_fied.split(".") - if con_fied_lst[0] == "dlf": - dlf_val = con_fied_lst[1] - project['crr_in_fcst_array'] = {'$objectToArray': '$dlf.' + dlf_val} - break - - group['_id'] = {hier_field: "$crr_in_fcst_array.k"} - - group.update({'count': {'$sum': 1}}) - pipeline = [{'$match': match}, - {'$project': project}, - {'$unwind': "$crr_in_fcst_array"}, - {'$match': match2}, - {'$group': group}] - - if config.debug: - logger.info('fetch_deal_rollup_dlf filter_name: %s, pipeline: %s', filter_name, pipeline) - - aggs = list(deals_collection.aggregate(pipeline, allowDiskUse=True)) - - for node in nodes: - aggregated_val = {} - for res in aggs: - val = {} - segment = None - drilldown = None - if res['_id']: - segment = res['_id'].get("segment", None) - drilldown = res['_id'].get(hier_field, None) - if node != drilldown: - continue - for k, v in res.items(): - if k != '_id': - if hier_aware: - if k.endswith(node.replace('.', '$')): - k = k.split(node.replace('.', '$'))[0] - else: - continue - val[k] = v - if k not in aggregated_val: - aggregated_val[k] = v - else: - aggregated_val[k] += v - if segment and return_seg_info: - cache[(filter_name, segment, node)] = val - else: - cache[(filter_name, node)] = val - if aggregated_val: - if return_seg_info: - cache[(filter_name, 'all_deals', node)] = aggregated_val - else: - cache[(filter_name, node)] = aggregated_val - else: - val = {label: 0 for label, _, _ in fields_and_operations} - val['count'] = 0 - cache[(filter_name, node)] = val - return cache - -def fetch_deal_rollup_dlf(period_and_close_periods, - nodes, - fields_and_operations, - config, - user=None, - filter_criteria=None, - filter_name=None, - favorites=None, - timestamp=None, - db=None, - cache=None, - return_seg_info=False, - filters=[], - root_node=None, - use_dlf_fcst_coll=False, - is_pivot_special=False, - deals_coll_name=None, - cumulative_fields=[], - cumulative_close_periods=[], - prev_periods=[], - actual_view=False - ): - if is_pivot_special: - return fetch_crr_deal_rollup_dlf(period_and_close_periods, - nodes, - fields_and_operations, - config, - user=user, - filter_criteria=filter_criteria, - filter_name=filter_name, - favorites=favorites, - timestamp=timestamp, - db=db, - cache=cache, - return_seg_info=return_seg_info, - filters=filters, - root_node=root_node, - use_dlf_fcst_coll=use_dlf_fcst_coll) - quarter_collection = db[QUARTER_COLL] if db else sec_context.tenant_db[QUARTER_COLL] - prev_periods = _prev_periods_fallback(config, prev_periods) - if not isinstance(period_and_close_periods, list): - period_and_close_periods = [period_and_close_periods] - period_and_close_periods_map = {"old_deals_coll": [], "new_deals_coll": []} - periods = [] - deals_collection_map = {} - - for period_and_close_period_for_db in period_and_close_periods: - period, _ = period_and_close_period_for_db - periods.append(period) - if ((period in prev_periods and config.config.get('update_new_collection')) or - (isinstance(period, list) and not set(period).isdisjoint(prev_periods) and - config.config.get('update_new_collection')) or - period is None): - period_and_close_periods_map['new_deals_coll'].append(period_and_close_period_for_db) - deals_collection = db[NEW_DEALS_COLL] if db else \ - sec_context.tenant_db[NEW_DEALS_COLL] - deals_collection_map['new_deals_coll'] = deals_collection - collection_name = NEW_DEALS_COLL - else: - period_and_close_periods_map['old_deals_coll'].append(period_and_close_period_for_db) - deals_collection = db[DEALS_COLL] if db else sec_context.tenant_db[DEALS_COLL] - deals_collection_map['old_deals_coll'] = deals_collection - collection_name = DEALS_COLL - - moved_deals_in_actual_view = False - if actual_view: - objectids = [] - #moveddeals = list(quarter_collection.find({'period': periods})) - moveddeals = list(quarter_collection.find({'period': {"$in" : periods}})) - if len(moveddeals) > 0: objectids = list(map(ObjectId, moveddeals[0]['ObjectId'])) - if len(objectids) > 0: moved_deals_in_actual_view = True - - drilldown_nodes = [] - hierarchy_nodes = [] - hier_aware = False - for _, fld, _ in fields_and_operations: - if fld in config.hier_aware_fields: - hier_aware = True - for node in nodes: - if '#' in node: - drilldown_nodes.append(node) - else: - hierarchy_nodes.append(node) - - user = user or sec_context.login_user_name - - for hier_field in ['drilldown_list', 'hierarchy_list']: - if hier_field == 'drilldown_list': - nodes = drilldown_nodes - hint_field = 'period_-1_close_period_-1_drilldown_list_-1_update_date_-1' - else: - nodes = hierarchy_nodes - hint_field = 'period_-1_close_period_-1_hierarchy_list_-1_update_date_-1' - if not nodes: - continue - deals_collection_criteria = {} - match_list2 = [] - for period_map in period_and_close_periods_map.keys(): - if (period_map == 'old_deals_coll' and period_and_close_periods_map['old_deals_coll']) or \ - (period_map == 'new_deals_coll' and period_and_close_periods_map['new_deals_coll']): - pass - else: - continue - - period_and_close_periods = period_and_close_periods_map[period_map] - match_list = [] - for period_and_close_period in period_and_close_periods: - period, close_periods = period_and_close_period - if isinstance(favorites, dict): - favs = favorites.get(period, None) - if not favs: - favs = _fetch_favorites(period, user, db) - else: - favs = favorites or _fetch_favorites(period, user, db) - - if isinstance(timestamp, dict): - timestamp = timestamp.get(period, None) - if not timestamp: - timestamp = _get_timestamp(period) - else: - timestamp = timestamp or _get_timestamp(period) # BUG: cant get timestamp when threaded from fm svc - if not moved_deals_in_actual_view: - match = {'period': period} - else: - match = {} - - if timestamp: - match['update_date'] = {'$gte': timestamp} - else: - match['is_deleted'] = False - hint_field = hint_field.replace('update_date', 'is_deleted') - - if close_periods: - period_key = get_key_for_close_periods(close_periods) - hint_field = modify_hint_field(close_periods, hint_field) - if not moved_deals_in_actual_view: - match[period_key] = {'$in': close_periods} - else: - closeperiods_value = close_periods - - if filter_criteria: - if "%(node)s" in str(filter_criteria): - matches = None - for node in nodes: - if matches is None: - matches = [contextualize_filter(filter_criteria, node, favs, period)] - else: - matches.append(contextualize_filter(filter_criteria, node, favs, period)) - match = {'$and': [match, {'$or': matches}]} - else: - match = {'$and': [match, contextualize_filter(filter_criteria, nodes[0], favs, period)]} - - matches = {} - if filters: - for filt in filters: - op = filt['op'] - if op != 'dlf': - matches.update(parse_filters(filt, config)) - if matches: - match = {'$and': [match, matches]} - match_list.append(match) - matches = [] - for filt in filters: - op = filt['op'] - negate = filt.get('negate', False) - match_vals = filt.get('match_vals') - match_vals = match_vals if match_vals else True - mongo_op = '$in' if not negate else '$nin' - if op == 'dlf': - if isinstance(match_vals, list): - matches.append({"in_fcst_array.v.state": {mongo_op: match_vals}}) - else: - if not negate: - matches.append({"in_fcst_array.v.state": True}) - else: - matches.append({"in_fcst_array.v.state": False}) - else: - continue - if not filters and con_fields_list and "in_fcst_array" not in project: - for con_fied in con_fields_list: - con_fied_lst = con_fied.split(".") - if con_fied_lst[0] == "dlf": - dlf_val = con_fied_lst[1] - project['in_fcst_array'] = {'$objectToArray': '$dlf.' + dlf_val} - break - matches.append({"in_fcst_array.k": {"$in": nodes}}) - - match2 = {'$and': matches} - match_list2.append(match2) - match = {'$or': []} - if len(match_list) > 1: - for m in match_list: - match['$or'].append(m) - else: - match = match_list[0] - - if actual_view: - match['is_moved'] = {'$exists': False} - - if moved_deals_in_actual_view: - actual_view_criteria = [{period_key: {'$in': closeperiods_value}, 'period': period}, - {'_id': {'$in': objectids}}] - if '$or' in match.keys(): - existing_or = match.pop('$or') - match['$and'] = [{'$or': existing_or}, {'$or': actual_view_criteria}] - elif '$and' in match.keys(): - match['$and'].append({'$or': actual_view_criteria}) - else: - match['$or'] = actual_view_criteria - deals_collection_criteria[period_map] = match - - match2 = {'$or': []} - if len(match_list2) > 1: - for m in match_list2: - match2['$or'].append(m) - else: - if match_list2: - match2 = match_list2[0] - - group = {} - project = {} - hier_aware = False - con_fields_list = [] - for label, fld, op in fields_and_operations: - if "%(node)s" in str(filter_criteria): - for node in nodes: - con_field = contextualize_field(fld, config, node) - con_fields_list.append(con_field) - project.update({con_field: 1}) - group.update({label + node.replace('.', '$'): {op: '$' + contextualize_field(fld, config, node)}}) - hier_aware = True - else: - group.update({label: {op: '$' + contextualize_field(fld, config, nodes[0])}}) - con_field = contextualize_field(fld, config, nodes[0]) - con_fields_list.append(con_field) - project.update({con_field: 1}) - if filters: - for filt in filters: - op = filt['op'] - if op == 'dlf': - value = filt['val'] - project['in_fcst_array'] = {'$objectToArray': '$dlf.' + value} - - group['_id'] = {hier_field: "$in_fcst_array.k"} - if return_seg_info: - if config.segment_field: - seg_field = config.segment_field - if group['_id']: - group['_id']['segment'] = '$' + str(seg_field) - else: - group['_id'] = '$' + str(seg_field) - project.update({seg_field: 1}) - else: - if config.debug: - logger.error("segment_field config not there in deal_svc config") - - group.update({'count': {'$sum': 1}}) - pipeline = [{'$match': match}, - {'$project': project}, - {'$unwind': "$in_fcst_array"}, - {'$match': match2}, - {'$group': group}] - - if period_and_close_periods_map['old_deals_coll'] and period_and_close_periods_map['new_deals_coll']: - collection_name = NEW_DEALS_COLL + " & " + DEALS_COLL - deals_collection = deals_collection_map['new_deals_coll'] - new_deals_match = deals_collection_criteria['new_deals_coll'] - old_deals_match = deals_collection_criteria['old_deals_coll'] - pipeline = [{'$match': new_deals_match}, - {'$unionWith': {'coll': 'deals', - 'pipeline': [{'$match': old_deals_match}]}}, - {'$project': project}, - {'$unwind': "$in_fcst_array"}, - {'$match': match2}, - {'$group': group}] - - - if config.debug: - logger.info('fetch_deal_rollup_dlf filter_name: %s, pipeline: %s', filter_name, pipeline) - - aggs = list(deals_collection.aggregate(pipeline, allowDiskUse=True, hint=hint_field)) - - for node in nodes: - aggregated_val = {} - for res in aggs: - val = {} - segment = None - drilldown = None - if res['_id']: - segment = res['_id'].get("segment", None) - drilldown = res['_id'].get(hier_field, None) - if node != drilldown: - continue - for k, v in res.items(): - if k != '_id': - if hier_aware: - if k.endswith(node.replace('.', '$')): - k = k.split(node.replace('.', '$'))[0] - else: - continue - val[k] = v - if k not in aggregated_val: - aggregated_val[k] = v - else: - aggregated_val[k] += v - if segment and return_seg_info: - cache[(filter_name, segment, node)] = val - else: - cache[(filter_name, node)] = val - if aggregated_val: - if return_seg_info: - cache[(filter_name, 'all_deals', node)] = aggregated_val - else: - cache[(filter_name, node)] = aggregated_val - else: - val = {label: 0 for label, _, _ in fields_and_operations} - val['count'] = 0 - cache[(filter_name, node)] = val - return cache - -def fetch_deal_rollup(period_and_close_periods, - nodes, - fields_and_operations, - config, - user=None, - filter_criteria=None, - filter_name=None, - favorites=None, - timestamp=None, - db=None, - cache=None, - return_seg_info=False, - filters=[], - root_node=None, - use_dlf_fcst_coll=False, - is_pivot_special=False, - deals_coll_name=None, - cumulative_fields=[], - cumulative_close_periods=[], - prev_periods=[], - actual_view=False - ): - if is_pivot_special: - return fetch_crr_deal_rollup(period_and_close_periods, - nodes, - fields_and_operations, - config, - user=user, - filter_criteria=filter_criteria, - filter_name=filter_name, - favorites=favorites, - timestamp=timestamp, - db=db, - cache=cache, - return_seg_info=return_seg_info, - filters=filters, - root_node=root_node, - use_dlf_fcst_coll=use_dlf_fcst_coll) - quarter_collection = db[QUARTER_COLL] if db else sec_context.tenant_db[QUARTER_COLL] - prev_periods = _prev_periods_fallback(config, prev_periods) - if not isinstance(period_and_close_periods, list): - period_and_close_periods = [period_and_close_periods] - period_and_close_periods_map = {"old_deals_coll": [], "new_deals_coll": []} - periods = [] - deals_collection_map = {} - - for period_and_close_period_for_db in period_and_close_periods: - period, _ = period_and_close_period_for_db - periods.append(period) - if ((period in prev_periods and config.config.get('update_new_collection')) or - (isinstance(period, list) and not set(period).isdisjoint(prev_periods) and - config.config.get('update_new_collection')) or - period is None): - period_and_close_periods_map['new_deals_coll'].append(period_and_close_period_for_db) - deals_collection = db[NEW_DEALS_COLL] if db else \ - sec_context.tenant_db[NEW_DEALS_COLL] - deals_collection_map['new_deals_coll'] = deals_collection - collection_name = NEW_DEALS_COLL - else: - period_and_close_periods_map['old_deals_coll'].append(period_and_close_period_for_db) - deals_collection = db[DEALS_COLL] if db else sec_context.tenant_db[DEALS_COLL] - deals_collection_map['old_deals_coll'] = deals_collection - collection_name = DEALS_COLL - - moved_deals_in_actual_view = False - if actual_view: - objectids = [] - moveddeals = list(quarter_collection.find({'period': {"$in" : periods}})) - if len(moveddeals) > 0: objectids = list(map(ObjectId, moveddeals[0]['ObjectId'])) - if len(objectids) > 0: moved_deals_in_actual_view = True - - drilldown_nodes = [] - hierarchy_nodes = [] - hier_aware = False - for _, fld, _ in fields_and_operations: - if fld in config.hier_aware_fields: - hier_aware = True - for node in nodes: - if '#' in node: - drilldown_nodes.append(node) - else: - hierarchy_nodes.append(node) - - user = user or sec_context.login_user_name - - for hier_field in ['drilldown_list', 'hierarchy_list']: - if hier_field == 'drilldown_list': - nodes = drilldown_nodes - hint_field = 'period_-1_close_period_-1_drilldown_list_-1_update_date_-1' - else: - nodes = hierarchy_nodes - hint_field = 'period_-1_close_period_-1_hierarchy_list_-1_update_date_-1' - if not nodes: - continue - deals_collection_criteria = {} - match_list2 = [] - for period_map in period_and_close_periods_map.keys(): - match_list = [] - if (period_map == 'old_deals_coll' and period_and_close_periods_map['old_deals_coll']) or \ - (period_map == 'new_deals_coll' and period_and_close_periods_map['new_deals_coll']): - pass - else: - continue - - period_and_close_periods = period_and_close_periods_map[period_map] - match_list = [] - for period_and_close_period in period_and_close_periods: - period, close_periods = period_and_close_period - if isinstance(favorites, dict): - favs = favorites.get(period, None) - if not favs: - favs = _fetch_favorites(period, user, db) - else: - favs = favorites or _fetch_favorites(period, user, db) - - if isinstance(timestamp, dict): - timestamp = timestamp.get(period, None) - if not timestamp: - timestamp = _get_timestamp(period) - else: - timestamp = timestamp or _get_timestamp(period) # BUG: cant get timestamp when threaded from fm svc - - if not moved_deals_in_actual_view: - match = {'period': period, - hier_field: {'$in': nodes}} - else: - match = {hier_field: {'$in': nodes}} - if timestamp: - match['update_date'] = {'$gte': timestamp} - else: - match['is_deleted'] = False - if close_periods: - if filter_name in cumulative_fields: - close_periods = cumulative_close_periods - period_key = get_key_for_close_periods(close_periods) - hint_field = modify_hint_field(close_periods, hint_field) - if not moved_deals_in_actual_view: - match[period_key] = {'$in': close_periods} - else: - closeperiods_value = close_periods - - if filter_criteria: - if "%(node)s" in str(filter_criteria): - matches = None - for node in nodes: - if matches is None: - matches = [contextualize_filter(filter_criteria, node, favs, period)] - else: - matches.append(contextualize_filter(filter_criteria, node, favs, period)) - match = {'$and': [match, {'$or': matches}]} - else: - match = {'$and': [match, contextualize_filter(filter_criteria, nodes[0], favs, period)]} - - match_list.append(match) - if not hier_aware: - match2 = {hier_field: {'$in': nodes}} - match_list2.append(match2) - match = {'$or': []} - if len(match_list) > 1: - for m in match_list: - match['$or'].append(m) - else: - match = match_list[0] - - if actual_view: - match['is_moved'] = {'$exists': False} - - if moved_deals_in_actual_view: - actual_view_criteria = [{period_key: {'$in': closeperiods_value}, 'period': period}, - {'_id': {'$in': objectids}}] - if '$or' in match.keys(): - existing_or = match.pop('$or') - match['$and'] = [{'$or': existing_or}, {'$or': actual_view_criteria}] - elif '$and' in match.keys(): - match['$and'].append({'$or': actual_view_criteria}) - else: - match['$or'] = actual_view_criteria - deals_collection_criteria[period_map] = match - - if not hier_aware: - match2 = {'$or': []} - if len(match_list2) > 1: - for m in match_list2: - match2['$or'].append(m) - else: - match2 = match_list2[0] - group = {} - regroup = {} - if hier_aware: - project = {contextualize_field(fld, config, node): 1 for node in nodes for _, fld, _ in - fields_and_operations} - group['_id'] = None - group.update( - {label + node.replace('.', '$'): {op: '$' + contextualize_field(fld, config, node)} for node in nodes - for label, fld, op in fields_and_operations}) - else: - if len(nodes) == 1 or root_node is None: - root_node = nodes[0] - project = {contextualize_field(fld, config, root_node): 1 for _, fld, _ in fields_and_operations} - - group['_id'] = {label: '$' + contextualize_field(fld, config, root_node) - for label, fld, op in fields_and_operations} - group['_id'][hier_field] = '$' + hier_field - group.update({hier_field: {'$addToSet': '$' + hier_field}}) - regroup = {label: {op: '$_id.' + label} for label, fld, op in fields_and_operations} - regroup['_id'] = {hier_field: '$_id.' + hier_field} - regroup.update({'count': {'$sum': 1}}) - - if return_seg_info: - if config.segment_field: - seg_field = config.segment_field - if group['_id']: - group['_id']['segment'] = '$' + str(seg_field) - else: - group['_id'] = {'segment': '$' + str(seg_field)} - if regroup.get('_id', None): - regroup['_id']['segment'] = '$_id.segment' - project.update({seg_field: 1}) - else: - if config.debug: - logger.error("segment_field config not there in deal_svc config") - - if hier_aware: - group.update({'count': {'$sum': 1}}) - pipeline = [{'$match': match}, - {'$project': project}, - {'$group': group}] - else: - project.update({'opp_id': 1, hier_field: 1}) - group['_id']['opp_id'] = '$opp_id' - pipeline = [{'$match': match}, - {'$project': project}, - {'$unwind': "$" + hier_field}, - {'$match': match2}, - {'$group': group}, - {'$unwind': "$" + hier_field}, - {'$group': regroup}] - if period_and_close_periods_map['old_deals_coll'] and period_and_close_periods_map['new_deals_coll']: - collection_name = NEW_DEALS_COLL + " & " + DEALS_COLL - deals_collection = deals_collection_map['new_deals_coll'] - new_deals_match = deals_collection_criteria['new_deals_coll'] - old_deals_match = deals_collection_criteria['old_deals_coll'] - if hier_aware: - group.update({'count': {'$sum': 1}}) - pipeline = [{'$match': new_deals_match}, - {'$unionWith': {'coll': 'deals', - 'pipeline': [{'$match': old_deals_match}]}}, - {'$project': project}, - {'$group': group}] - else: - project.update({'opp_id': 1, hier_field: 1}) - group['_id']['opp_id'] = '$opp_id' - pipeline = [{'$match': new_deals_match}, - {'$unionWith': {'coll': 'deals', - 'pipeline': [{'$match': old_deals_match}]}}, - {'$project': project}, - {'$unwind': "$" + hier_field}, - {'$match': match2}, - {'$group': group}, - {'$unwind': "$" + hier_field}, - {'$group': regroup}] - - if config.debug: - logger.info('fetch_deal_rollup filter_name: %s, pipeline: %s', filter_name, pipeline) - - aggs = list(deals_collection.aggregate(pipeline, allowDiskUse=True, hint=hint_field)) - - for node in nodes: - aggregated_val = {} - for res in aggs: - val = {} - segment = None - drilldown = None - if res['_id']: - segment = res['_id'].get("segment", None) - drilldown = res['_id'].get(hier_field, None) - if not hier_aware: - if node != drilldown: - continue - for k, v in res.items(): - if k != '_id': - if hier_aware: - if k.endswith(node.replace('.', '$')): - k = k.split(node.replace('.', '$'))[0] - else: - continue - val[k] = v - if k not in aggregated_val: - aggregated_val[k] = v - else: - aggregated_val[k] += v - if segment and return_seg_info: - cache[(filter_name, segment, node)] = val - else: - cache[(filter_name, node)] = val - if aggregated_val: - if return_seg_info: - cache[(filter_name, 'all_deals', node)] = aggregated_val - else: - cache[(filter_name, node)] = aggregated_val - else: - val = {label: 0 for label, _, _ in fields_and_operations} - val['count'] = 0 - cache[(filter_name, node)] = val - return cache - -def fetch_many_prnt_DR_deal_totals(period_and_close_periods, - nodes, - fields_and_operations, - config, - prnt_node, - user=None, - filter_criteria=None, - filter_name=None, - favorites=None, - timestamp=None, - db=None, - cache=None, - return_seg_info=False, - deals_coll_name=None, - prev_periods=[] - ): - """ - fetch total amount of deals for period and prnt_node for the deals belonging to the - node. It fetches the relevant deal as per the filter. - - Arguments: - period_and_close_periods {tuple} -- as of mnemonic, [close mnemonics] ('2020Q2', ['2020Q2']) - node {str} -- hierarchy node '0050000FLN2C9I2' - fields_and_operations {list} -- tuples of label, deal field, [('Amount', 'Amount', '$sum')] - mongo op to total - config {DealsConfig} -- instance of DealsConfig ...? - prnt_node -- parent of the node. same as node if it is already the parent - - Keyword Arguments: - user {str} -- user name, if None, will use sec_context 'gnana' - filter_criteria {dict} -- mongo db filter criteria (default: {None}) {'Amount': {'$gte': 100000}} - filter_name {str} -- label to identify filter in cache - favorites {set} -- set of opp_ids that have been favorited by user - if None, queries favs collectionf to find them - (default: {None}) - timestamp {float} -- epoch timestamp for when deal records expire - if None, checks flags to find it - (default: {None}) - db {pymongo.database.Database} -- instance of tenant_db - (default: {None}) - if None, will create one - cache {dict} -- dict to hold records fetched by func (default: {None}) - (used to memoize fetching many recs) - - Returns: - dict -- 'total amount of deals - """ - prev_periods = _prev_periods_fallback(config, prev_periods) - deals_coll_name = deals_coll_name if deals_coll_name is not None else DEALS_COLL - deals_collection = db[deals_coll_name] if db else sec_context.tenant_db[deals_coll_name] - if isinstance(period_and_close_periods, list): - for period_and_close_period in period_and_close_periods: - period, close_periods = period_and_close_period - else: - period, close_periods = period_and_close_periods - if deals_coll_name is not None and ((period in prev_periods and config.config.get('update_new_collection')) or - (isinstance(period, list) and not set(period).isdisjoint(prev_periods) and - config.config.get('update_new_collection')) or - period is None): - deals_collection = db[NEW_DEALS_COLL] if db else sec_context.tenant_db[NEW_DEALS_COLL] - user = user or sec_context.login_user_name - hint_field = 'period_-1_close_period_-1_drilldown_list_-1_update_date_-1' if '#' in nodes[0] \ - else 'period_-1_close_period_-1_hierarchy_list_-1_update_date_-1' - match_list = [] - if isinstance(favorites, dict): - favs = favorites.get(period, None) - if not favs: - favs = _fetch_favorites(period, user, db) - else: - favs = favorites or _fetch_favorites(period, user, db) - - if isinstance(timestamp, dict): - timestamp = timestamp.get(period, None) - if not timestamp: - timestamp = _get_timestamp(period) - else: - timestamp = timestamp or _get_timestamp(period) # BUG: cant get timestamp when threaded from fm svc - - hier_field = 'drilldown_list' if '#' in nodes[0] else 'hierarchy_list' - - if not isinstance(period, list): - match = {'period': period, - hier_field: {'$in': nodes}} - if timestamp: - match['update_date'] = {'$gte': timestamp} - else: - match['is_deleted'] = False - - if close_periods: - match['close_period'] = {'$in': close_periods} - else: - match = {'$or': []} - for temp_num in range(len(period)): - match1 = {'period': period[temp_num], - hier_field: {'$in': nodes}} - if timestamp: - match1['update_date'] = {'$gte': timestamp} - else: - match1['is_deleted'] = False - - if close_periods: - period_key = get_key_for_close_periods(close_periods) - hint_field = modify_hint_field(close_periods, hint_field) - match1['close_period'] = {'$in': close_periods[temp_num]} - match['$or'] += [match1] - # Using the prnt_node in the filter criteria to fetch relevant deal - if filter_criteria: - match = { - '$and': [match, contextualize_filter(filter_criteria, prnt_node if prnt_node else nodes[0], favs, period)]} - match_list.append(match) - - match = {'$or': []} - if len(match_list) > 1: - for m in match_list: - match['$or'].append(m) - else: - match = match_list[0] - # Using the prnt_node in the for the projection - project = {contextualize_field(fld, config, prnt_node if prnt_node else nodes[0]): 1 for _, fld, _ in - fields_and_operations} - # Using the prnt_node for grouping - group = {label: {op: '$' + contextualize_field(fld, config, prnt_node if prnt_node else nodes[0])} for - label, fld, op in - fields_and_operations} - - group.update({'_id': {hier_field: "$" + hier_field}, - 'count': {'$sum': 1}, - }) - project[hier_field] = 1 - - if return_seg_info: - if config.segment_field: - seg_field = config.segment_field - group['_id']['segment'] = '$' + str(seg_field) - project.update({seg_field: 1}) - else: - if config.debug: - logger.error("segment_field config not there in deal_svc config") - - pipeline = [{'$match': match}, - {'$project': project}, - {"$unwind": "$" + hier_field}, - {'$match': {hier_field: {'$in': nodes}}}, - {'$group': group}, - {'$unwind': "$_id." + hier_field}] - - # AV-14059 Commenting below as part of the log fix, - # config.debug is not working need to think of better approach - # if config.debug: - # logger.info('fetch_many_prnt_DR_deal_totals filter_name: %s, pipeline: %s', filter_name, pipeline) - - aggs = list(deals_collection.aggregate(pipeline, allowDiskUse=True, hint=hint_field)) - - aggregated_val = {} - if return_seg_info: - for res in aggs: - val = {} - for k, v in res.items(): - if k != '_id': - val[k] = v - if k not in aggregated_val: - aggregated_val[k] = v - else: - aggregated_val[k] += v - node = res['_id'][hier_field] - seg = res['_id']['segment'] - if config.segment_field: - cache[(filter_name, seg, node)] = val - cache[(filter_name, 'all_deals', node)] = aggregated_val - else: - for res in aggs: - try: - val = {} - for k, v in res.items(): - if k != '_id': - val[k] = v - node = res['_id'][hier_field] - except IndexError: - val = {label: 0 for label, _, _ in fields_and_operations} - val['count'] = 0 - cache[(filter_name, node)] = val - return cache - -def fetch_many_dr_from_deals(period_and_close_periods, - nodes, - filter_criteria_and_fields_and_operations, - config, - user=None, - db=None, - return_seg_info=False, - timestamp=None, - metrics=NOOPMetricSet(), - is_pivot_special=False, - prev_periods=[], - rollup_task=False, - parents_wise_nodes={}, - deals_coll_name=None, - cumulative_fields=[], - cumulative_close_periods=[], - actual_view=False - ): - """ - fetch a whole bunch of deal totals for many nodes and many different filters - - Arguments: - period_and_close_periods {tuple} -- as of mnemonic, [close mnemonics] ('2020Q2', ['2020Q2']) - nodes {list} -- list of hierarchy nodes to get totals for ['0050000FLN2C9I2',] - filter_criteria_and_fields_and_operations {list} -- list of tuples of [('big', [{'Amount': {'$gte': 100000}}], - (filt name, [filt criteria], [(label, field, opp),]) [('Amount', 'Amount', '$sum')])] - config {DealsConfig} -- instance of DealsConfig ...? - - Keyword Arguments: - user {str} -- user name, if None, will use sec_context 'gnana' - db {pymongo.database.Database} -- instance of tenant_db - (default: {None}) - if None, will create one - Returns: - dict -- {(filter name, node): {'count': 1, 'amount': 10}} - """ - prev_periods = _prev_periods_fallback(config, prev_periods) - cache = {} - db = db or sec_context.tenant_db - user = user or sec_context.login_user_name - if not isinstance(period_and_close_periods, list): - period_and_close_periods = [period_and_close_periods] - - fm_rollup_dlf_func = fetch_deal_rollup_dlf_using_df if use_df_for_dlf_rollup() else fetch_deal_rollup_dlf - - # doing some work upfront before the threading begins - favs = {} - timestamp_new = {} - for period_and_close_period in period_and_close_periods: - period, _ = period_and_close_period - favs[period] = _fetch_favorites(period, user, db) - timestamp_new[period] = timestamp or _get_timestamp(period) - timestamp = timestamp_new - - from infra.read import get_period_as_of - as_of = get_period_as_of(period) - from infra.read import fetch_root_nodes - root_nodes = fetch_root_nodes(as_of) - root_nodes = [root_node['node'] for root_node in root_nodes] - root_node = root_nodes[0] - if is_pivot_special: - for node in root_nodes: - if node.split('#')[0] in config.not_deals_tenant.get('special_pivot', []): - root_node = node - if config.debug: - if not rollup_task: - for (filter_name, - filter_criteria, - fields_and_operations, - filters, - dlf_field) in filter_criteria_and_fields_and_operations: - if dlf_field: - fm_rollup_dlf_func(period_and_close_periods, - nodes, - fields_and_operations, - config, - user, - filter_criteria, - filter_name, - favs, - timestamp, - db, - cache=cache, - return_seg_info=return_seg_info, - filters=filters, - root_node=root_node, - use_dlf_fcst_coll=use_dlf_fcst_coll_for_rollups(), - is_pivot_special=is_pivot_special, - deals_coll_name=deals_coll_name, - cumulative_fields=cumulative_fields, - cumulative_close_periods=cumulative_close_periods, - prev_periods=prev_periods, - actual_view=actual_view - ) - else: - fetch_deal_rollup(period_and_close_periods, - nodes, - fields_and_operations, - config, - user, - filter_criteria, - filter_name, - favs, - timestamp, - db, - cache=cache, - return_seg_info=return_seg_info, - filters=filters, - use_dlf_fcst_coll=use_dlf_fcst_coll_for_rollups(), - root_node=root_node, - is_pivot_special=is_pivot_special, - deals_coll_name=deals_coll_name, - cumulative_fields=cumulative_fields, - cumulative_close_periods=cumulative_close_periods, - prev_periods=prev_periods, - actual_view=actual_view - ) - else: - for (filter_name, - filter_criteria, - fields_and_operations, - filters, - dlf_field, - dr_type) in filter_criteria_and_fields_and_operations: - if dr_type == 'prnt_dr': - for prnt_node in nodes: - fetch_many_prnt_DR_deal_totals(period_and_close_periods, - parents_wise_nodes[prnt_node], - fields_and_operations, - config, - prnt_node, - user, - filter_criteria, - filter_name, - favs, - timestamp, - db, - cache=cache, - return_seg_info=return_seg_info, - deals_coll_name=deals_coll_name, - prev_periods=prev_periods - ) - elif dlf_field: - fm_rollup_dlf_func(period_and_close_periods, - nodes, - fields_and_operations, - config, - user, - filter_criteria, - filter_name, - favs, - timestamp, - db, - cache=cache, - return_seg_info=return_seg_info, - filters=filters, - root_node=root_node, - use_dlf_fcst_coll=use_dlf_fcst_coll_for_rollups(), - is_pivot_special=is_pivot_special, - prev_periods=prev_periods, - deals_coll_name=deals_coll_name - ) - else: - fetch_deal_rollup(period_and_close_periods, - nodes, - fields_and_operations, - config, - user, - filter_criteria, - filter_name, - favs, - timestamp, - db, - cache=cache, - return_seg_info=return_seg_info, - filters=filters, - root_node=root_node, - use_dlf_fcst_coll=use_dlf_fcst_coll_for_rollups(), - is_pivot_special=is_pivot_special, - deals_coll_name=deals_coll_name, - cumulative_fields=cumulative_fields, - cumulative_close_periods=cumulative_close_periods, - prev_periods=prev_periods - ) - else: - for chunk in iter_chunks(list(filter_criteria_and_fields_and_operations), batch_size=100): - metrics.inc_counter('fmdr_batches') - max_threads = 0 - threads = [] - if not rollup_task: - for (filter_name, - filter_criteria, - fields_and_operations, - filters, - dlf_field, - ) in chunk: - t = threading.Thread(target=fm_rollup_dlf_func if dlf_field else fetch_deal_rollup, - args=(period_and_close_periods, - nodes, - fields_and_operations, - config, - user, - filter_criteria, - filter_name, - favs, - timestamp, - db, - cache, - return_seg_info, - filters, - root_node, - use_dlf_fcst_coll_for_rollups(), - is_pivot_special, - deals_coll_name, - cumulative_fields, - cumulative_close_periods, - prev_periods, - actual_view - )) - threads.append(t) - t.start() - metrics.inc_counter('fmdr_threadcnt', max_threads) - else: - for (filter_name, - filter_criteria, - fields_and_operations, - filters, - dlf_field, - dr_type) in chunk: - if dr_type == 'prnt_dr': - for prnt_node in nodes: - t = threading.Thread(target=fetch_many_prnt_DR_deal_totals, - args=(period_and_close_periods, - parents_wise_nodes[prnt_node], - fields_and_operations, - config, - prnt_node, - user, - filter_criteria, - filter_name, - favs, - timestamp, - db, - cache, - return_seg_info, - deals_coll_name, - prev_periods - )) - threads.append(t) - t.start() - metrics.inc_counter('fmdr_threadcnt', max_threads) - else: - t = threading.Thread(target=fm_rollup_dlf_func if dlf_field else fetch_deal_rollup, - args=(period_and_close_periods, - nodes, - fields_and_operations, - config, - user, - filter_criteria, - filter_name, - favs, - timestamp, - db, - cache, - return_seg_info, - filters, - root_node, - use_dlf_fcst_coll_for_rollups(), - is_pivot_special, - deals_coll_name, - cumulative_fields, - cumulative_close_periods, - prev_periods - )) - - threads.append(t) - t.start() - metrics.inc_counter('fmdr_threadcnt', max_threads) - for t in threads: - t.join() - if len(threads) > max_threads: - max_threads = len(threads) - metrics.set_counter('fmdr_maxthreads', max_threads) - - return cache - -def fetch_prnt_DR_deal_totals(period_and_close_periods, - node, - fields_and_operations, - config, - prnt_node, - user=None, - filter_criteria=None, - filter_name=None, - favorites=None, - timestamp=None, - db=None, - cache=None, - return_seg_info=False, - dlf_node=None, - prev_periods=[] - ): - """ - fetch total amount of deals for period and prnt_node for the deals belonging to the - node. It fetches the relevant deal as per the filter. - - Arguments: - period_and_close_periods {tuple} -- as of mnemonic, [close mnemonics] ('2020Q2', ['2020Q2']) - node {str} -- hierarchy node '0050000FLN2C9I2' - fields_and_operations {list} -- tuples of label, deal field, [('Amount', 'Amount', '$sum')] - mongo op to total - config {DealsConfig} -- instance of DealsConfig ...? - prnt_node -- parent of the node. same as node if it is already the parent - - Keyword Arguments: - user {str} -- user name, if None, will use sec_context 'gnana' - filter_criteria {dict} -- mongo db filter criteria (default: {None}) {'Amount': {'$gte': 100000}} - filter_name {str} -- label to identify filter in cache - favorites {set} -- set of opp_ids that have been favorited by user - if None, queries favs collectionf to find them - (default: {None}) - timestamp {float} -- epoch timestamp for when deal records expire - if None, checks flags to find it - (default: {None}) - db {pymongo.database.Database} -- instance of tenant_db - (default: {None}) - if None, will create one - cache {dict} -- dict to hold records fetched by func (default: {None}) - (used to memoize fetching many recs) - - Returns: - dict -- 'total amount of deals - """ - prev_periods = _prev_periods_fallback(config, prev_periods) - if not isinstance(period_and_close_periods, list): - period_and_close_periods = [period_and_close_periods] - for period_and_close_period_for_db in period_and_close_periods: - period, _ = period_and_close_period_for_db - if (period in prev_periods and config.config.get('update_new_collection')) or ( - isinstance(period, list) and not set(period).isdisjoint(prev_periods) and config.config.get( - 'update_new_collection')) or period is None: - deals_collection = db[NEW_DEALS_COLL] if db else sec_context.tenant_db[NEW_DEALS_COLL] - else: - deals_collection = db[DEALS_COLL] if db else sec_context.tenant_db[DEALS_COLL] - break - user = user or sec_context.login_user_name - hint_field = 'period_-1_close_period_-1_drilldown_list_-1_update_date_-1' if '#' in node \ - else 'period_-1_close_period_-1_hierarchy_list_-1_update_date_-1' - - match_list = [] - for period_and_close_period in period_and_close_periods: - period, close_periods = period_and_close_period - if isinstance(favorites, dict): - favs = favorites.get(period, None) - if not favs: - favs = _fetch_favorites(period, user, db) - else: - favs = favorites or _fetch_favorites(period, user, db) - - if isinstance(timestamp, dict): - timestamp = timestamp.get(period, None) - if not timestamp: - timestamp = _get_timestamp(period) - else: - timestamp = timestamp or _get_timestamp(period) # BUG: cant get timestamp when threaded from fm svc - - hier_field = 'drilldown_list' if '#' in node else 'hierarchy_list' - - if not isinstance(period, list): - match = {'period': period, - hier_field: {'$in': [node]}} - if timestamp: - match['update_date'] = {'$gte': timestamp} - else: - match['is_deleted'] = False - - if close_periods: - period_key = get_key_for_close_periods(close_periods) - hint_field = modify_hint_field(close_periods, hint_field) - match[period_key] = {'$in': close_periods} - else: - match = {'$or': []} - for temp_num in range(len(period)): - match1 = {'period': period[temp_num], - hier_field: {'$in': [node]}} - if timestamp: - match1['update_date'] = {'$gte': timestamp} - else: - match1['is_deleted'] = False - - if close_periods: - period_key = get_key_for_close_periods(close_periods) - match1[period_key] = {'$in': close_periods[temp_num]} - match['$or'] += [match1] - # Using the prnt_node in the filter criteria to fetch relevant deal - if filter_criteria: - match = { - '$and': [match, contextualize_filter(filter_criteria, prnt_node if prnt_node else node, favs, period)]} - match_list.append(match) - - match = {'$or': []} - if len(match_list) > 1: - for m in match_list: - match['$or'].append(m) - else: - match = match_list[0] - # Using the prnt_node in the for the projection - project = {contextualize_field(fld, config, prnt_node if prnt_node else node): 1 for _, fld, _ in - fields_and_operations} - # Using the prnt_node for grouping - group = {label: {op: '$' + contextualize_field(fld, config, prnt_node if prnt_node else node)} for label, fld, op in - fields_and_operations} - - group.update({'_id': None, - 'count': {'$sum': 1}, - }) - - pipeline = [{'$match': match}, - {'$project': project}, - {'$group': group}] - - if return_seg_info: - if config.segment_field: - seg_field = config.segment_field - group['_id'] = '$' + str(seg_field) - project.update({seg_field: 1}) - else: - if config.debug: - logger.error("segment_field config not there in deal_svc config") - - if config.debug: - logger.info('fetch_prnt_DR_deal_totals filter_name: %s, pipeline: %s', filter_name, pipeline) - - aggs = list(deals_collection.aggregate(pipeline, allowDiskUse=True, hint=hint_field)) - - aggregated_val = {} - if return_seg_info: - for res in aggs: - val = {} - for k, v in res.items(): - if k != '_id': - val[k] = v - if k not in aggregated_val: - aggregated_val[k] = v - else: - aggregated_val[k] += v - if config.segment_field: - cache[(filter_name, res['_id'], node)] = val - cache[(filter_name, 'all_deals', node)] = aggregated_val - return cache - else: - try: - val = {k: v for k, v in aggs[0].items() if k != '_id'} - except IndexError: - val = {label: 0 for label, _, _ in fields_and_operations} - val['count'] = 0 - - if cache is not None: - cache[(filter_name, node)] = val - - return val - -def fetch_crr_deal_totals(period_and_close_periods, - node, - fields_and_operations, - config, - user=None, - filter_criteria=None, - filter_name=None, - favorites=None, - timestamp=None, - db=None, - cache=None, - return_seg_info=False, - dlf_node=None, - search_criteria=None, - criteria=None, - sort_fields=[], - custom_limit=None - ): - """ - fetch total and count of deals for period and node - - Arguments: - period_and_close_periods {tuple} -- as of mnemonic, [close mnemonics] ('2020Q2', ['2020Q2']) - node {str} -- hierarchy node '0050000FLN2C9I2' - fields_and_operations {list} -- tuples of label, deal field, [('Amount', 'Amount', '$sum')] - mongo op to total - config {DealsConfig} -- instance of DealsConfig ...? - - Keyword Arguments: - user {str} -- user name, if None, will use sec_context 'gnana' - filter_criteria {dict} -- mongo db filter criteria (default: {None}) {'Amount': {'$gte': 100000}} - filter_name {str} -- label to identify filter in cache - favorites {set} -- set of opp_ids that have been favorited by user - if None, queries favs collectionf to find them - (default: {None}) - timestamp {float} -- epoch timestamp for when deal records expire - if None, checks flags to find it - (default: {None}) - db {pymongo.database.Database} -- instance of tenant_db - (default: {None}) - if None, will create one - cache {dict} -- dict to hold records fetched by func (default: {None}) - (used to memoize fetching many recs) - - Returns: - dict -- 'count' key and total for each total field - """ - deals_collection = db[GBM_CRR_COLL] if db else sec_context.tenant_db[GBM_CRR_COLL] - user = user or sec_context.login_user_name - match_list = [] - if period_and_close_periods: - if not isinstance(period_and_close_periods, list): - period_and_close_periods = [period_and_close_periods] - for period_and_close_period in period_and_close_periods: - period, close_periods = period_and_close_period - if isinstance(favorites, dict): - favs = favorites.get(period, None) - if not favs: - favs = _fetch_favorites(period, user, db) - else: - favs = favorites or _fetch_favorites(period, user, db) - - hier_field = '__segs' - if not isinstance(period, list): - match = {hier_field: {'$in': [node]}} - if criteria: - match.update(criteria) - - if close_periods: - match['monthly_period'] = {'$in': close_periods} - else: - match = {'$or': []} - for temp_num in range(len(period)): - match1 = {hier_field: {'$in': [node]}} - if criteria: - match1.update(criteria) - - if close_periods: - match1['monthly_period'] = {'$in': close_periods[temp_num]} - match['$or'] += [match1] - - if filter_criteria: - match = {'$and': [match, - contextualize_filter(filter_criteria, dlf_node if dlf_node else node, favs, period)]} - - if search_criteria: - search_terms, search_fields = search_criteria - match['$or'] = [{field: {'$regex': '|'.join(search_terms), '$options': 'i'}} for field in search_fields] - - match_list.append(match) - else: - period = {} - favs = favorites - hier_field = '__segs' - match = {hier_field: {'$in': [node]}} - if criteria: - match.update(criteria) - if filter_criteria: - match = { - '$and': [match, contextualize_filter(filter_criteria, dlf_node if dlf_node else node, favs, period)]} - if search_criteria: - search_terms, search_fields = search_criteria - match['$or'] = [{field: {'$regex': '|'.join(search_terms), '$options': 'i'}} for field in search_fields] - match_list.append(match) - - match = {'$or': []} - if len(match_list) > 1: - for m in match_list: - match['$or'].append(m) - else: - match = match_list[0] - - _node = dlf_node if dlf_node else node - project = {contextualize_field(fld, config, _node, hier_aware='forecast' not in fld): 1 - for _, fld, _ in fields_and_operations} - project['update_date'] = 1 - - group = {label: {op: '$' + contextualize_field(fld, config, _node, hier_aware='forecast' not in fld)} - for label, fld, op in fields_and_operations} - - group.update({'_id': None, - 'count': {'$sum': 1}, - 'timestamp': {'$last': '$update_date'} - }) - - pipeline = [{'$match': match}, - {'$project': project}, - {'$group': group}] - if custom_limit: - if sort_fields: - pipeline = [{'$match': match}, - {'$project': project}, - {'$sort': OrderedDict( - [(contextualize_field(field, config, node), direction) for - field, direction in sort_fields])}, - {'$limit': custom_limit}, - {'$group': group}] - else: - pipeline = [{'$match': match}, - {'$project': project}, - {'$limit': custom_limit}, - {'$group': group}] - if return_seg_info: - if config.segment_field: - seg_field = config.segment_field - group['_id'] = '$' + str(seg_field) - project.update({seg_field: 1}) - else: - if config.debug: - logger.error("segment_field config not there in deal_svc config") - - aggregated_results = list(deals_collection.aggregate(pipeline, allowDiskUse=True)) - - total_aggregated_values = {} - if return_seg_info: - for result in aggregated_results: - segment_values = {key: value for key, value in result.items() if key != '_id'} - for key, value in segment_values.items(): - total_aggregated_values[key] = total_aggregated_values.get(key, 0) + value - if config.segment_field: - cache[(filter_name, result['_id'], node)] = value - cache[(filter_name, 'all_deals', node)] = total_aggregated_values - return cache - else: - if aggregated_results: - aggregated_values = {key: value for key, value in aggregated_results[0].items() if key != '_id'} - else: - aggregated_values = {label: 0 for label, _, _ in fields_and_operations} - aggregated_values['count'] = 0 - aggregated_values['timestamp'] = None - - if cache is not None: - cache[(filter_name, node)] = aggregated_values - - return aggregated_values - -def get_waterfall_week_total(period, node, begin, end, close_date_field, sum_field, crit): - config = DealConfig() - if config.config.get('update_new_collection'): - deals_collection = sec_context.tenant_db[NEW_DEALS_COLL] - else: - deals_collection = sec_context.tenant_db[DEALS_COLL] - criteria = { - "period": period, - "drilldown_list": node, - close_date_field: { - "$gte": begin, - "$lte": end - }, - 'is_deleted': False - } - criteria = {'$and': [crit, criteria]} - pipeline = [ - { - "$match": criteria - }, - { - "$group": { - "_id": None, - "val": { - "$sum": '$'+ sum_field - } - } - } - ] - result = list(deals_collection.aggregate(pipeline)) - try: - return result[0]["val"] - except: - return 0 - -def get_waterfall_weekly_totals(period, node, weeks, close_date_field, sum_field, crit, multiplier=1000): - weekly_totals = {} - for week in weeks: - begin = week["begin"] - end = week["end"] - week_label = week["label"] - total = get_waterfall_week_total(period, node, begin, end, close_date_field, sum_field, crit) - weekly_totals[week_label] = total / multiplier - return weekly_totals diff --git a/infra/rules.py b/infra/rules.py index 054956a..9e87149 100644 --- a/infra/rules.py +++ b/infra/rules.py @@ -1,24 +1,4 @@ -from infra.read import fetch_node, fetch_ancestors, fetch_children, fetch_descendants - - -def passes_hierarchy_rules(as_of, - node, - segment, - drilldown=True, - db=None, - exclude_display_specific=False, period=None, boundary_dates=None): - ''' - Check if a segment is valid for node (Cached Version) - exclude_display_specific : Indicator to exclude display specific rules. - ''' - try: - drilldown_record = fetch_node(as_of, node, fields=['normal_segs'], period=period, boundary_dates=boundary_dates) - if exclude_display_specific: - if segment in drilldown_record['normal_segs']: - return True - return False - except: - return False +from infra.read import fetch_ancestors, fetch_children, fetch_descendants def passes_configured_hierarchy_rules(as_of, @@ -75,35 +55,6 @@ def passes_configured_hierarchy_rules(as_of, return True -def node_aligns_to_segment(as_of, - node, - segment, - config, - exclude_display_specific=False, - eligible_nodes_for_segs=None, - period=None, - boundary_dates=None): - ''' - This Function checks if a node satisfies all rules which are configured to a segment - as_of: Time_Stamp as_of - node: drilldown node - segment: segment name - config: FMConfig() - ''' - if exclude_display_specific: - if eligible_nodes_for_segs: - return node in eligible_nodes_for_segs.get(segment, []) - return passes_hierarchy_rules(as_of, - node, - segment, - exclude_display_specific=exclude_display_specific, period=period, boundary_dates=boundary_dates) - else: - return passes_configured_hierarchy_rules(as_of, - node, - config.segments[segment].get('rules', []), - exclude_display_specific=exclude_display_specific, period=period, boundary_dates=boundary_dates) - - def is_ancestor_of(as_of, node, related_nodes, diff --git a/infra/write.py b/infra/write.py index 105c3a7..8a743a1 100644 --- a/infra/write.py +++ b/infra/write.py @@ -3,7 +3,7 @@ from aviso.settings import sec_context from pymongo import UpdateOne -from infra import DEALS_COLL, FORECAST_SCHEDULE_COLL +from infra.constants import FORECAST_SCHEDULE_COLL, DEALS_COLL from utils.mongo_writer import bulk_write logger = logging.getLogger('gnana.%s' % __name__) diff --git a/periods_service/__init__.py b/periods_service/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..2072fde --- /dev/null +++ b/pytest.ini @@ -0,0 +1,8 @@ +[pytest] +minversion = 6.0 +addopts = -ra -q +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +env_files = .env \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index bc0d68a..3418d08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,45 +1,59 @@ amqp==5.3.1 -asgiref==3.8.1 -billiard==4.2.1 +asn1crypto==1.5.1 +async-timeout==4.0.2 +-e git+git@github.com:gnanarepo/aviso-infrastructure.git@276ee584f85a5eb8210b396215a7374ccc631f2a#egg=aviso +Aviso-Schema==0.0.0 +avisosdk==1.0 +billiard==3.6.4.0 boto==2.49.0 -boto3==1.38.28 -botocore==1.38.28 -celery==5.5.3 -click==8.2.1 +boto3==1.17.78 +botocore==1.20.78 +cached-property==2.0 +celery==5.1.2 +cffi==1.15.1 +click==7.1.2 click-didyoumean==0.3.1 click-plugins==1.1.1 click-repl==0.3.0 -Django==5.2.1 -dnspython==2.7.0 -exceptiongroup==1.3.0 -iniconfig==2.1.0 -isort==6.0.1 -Jinja2==3.1.6 -jmespath==1.0.1 -kombu==5.5.4 -line_profiler==4.2.0 -MarkupSafe==3.0.2 -netaddr==1.3.0 -numpy==2.2.6 -packaging==25.0 -pandas==2.2.3 -pluggy==1.6.0 -prompt_toolkit==3.0.51 -psutil==7.0.0 -pycryptodome==3.23.0 -pydevd==3.3.0 -Pygments==2.19.1 -pymongo==4.13.0 -pytest==8.4.0 -python-dateutil==2.9.0.post0 -pytz==2025.2 -s3transfer==0.13.0 -six==1.17.0 -sqlparse==0.5.3 -tomli==2.2.1 -typing_extensions==4.14.0 -tzdata==2025.2 -urllib3==2.4.0 +cryptography==2.4.2 +dj-database-url==0.5.0 +Django==2.2.28 +eventbus==1.0 +idna==3.10 +imbox==0.9.5 +importlib-metadata==4.8.3 +importlib-resources==5.4.0 +Jinja2==2.11.3 +jmespath==0.10.0 +kombu==5.1.0 +lxml==4.9.3 +MarkupSafe==2.0.1 +netaddr==0.10.1 +newrelic==7.16.0.178 +numpy==1.19.5 +packaging==21.3 +prompt-toolkit==3.0.36 +psycopg2-binary==2.8.6 +pycparser==2.21 +pycrypto==2.6.1 +pycryptodome==3.10.1 +PyJWT==1.7.1 +pymongo==3.12.3 +pyparsing==3.1.4 +pyschema==2.4.0 +python-dateutil==2.8.2 +python-memcached==1.59 +pytz==2018.9 +redis==3.5.3 +requests==2.14.2 +requests-toolbelt==0.8.0 +s3transfer==0.4.2 +simplejson==3.20.1 +six==1.12.0 +sqlparse==0.4.4 +typing-extensions==4.1.1 +urllib3==1.24.1 vine==5.1.0 wcwidth==0.2.13 --e git+git@github.com:gnanarepo/eventbus.git@v1.0#egg=eventbus \ No newline at end of file +websocket-client==0.44.0 +zipp==3.6.0 diff --git a/tasks/__init__.py b/tasks/__init__.py index ab305cf..4a2addc 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -1,29 +1,23 @@ import json import logging import os -import shutil import sys import tempfile import threading import time import traceback import uuid -from datetime import datetime, UTC +from datetime import datetime, timezone as UTC -import celery -from _collections import defaultdict from aviso.framework import tracer from aviso.framework.diagnostics import probe_util from aviso.framework.views import GnanaError from aviso.settings import (CNAME, DEBUG, TMP_DIR, event_context, - gnana_cprofile_bucket, gnana_db, gnana_storage, + gnana_cprofile_bucket, gnana_storage, log_config, node, sec_context) from celery import current_task -from celery.result import AsyncResult -from domainmodel.app import (ResultCache, Task, TaskActive, TaskArchive, - V2StatsLog) -from tasks import asynctasks +from domainmodel.app import Task, TaskArchive, V2StatsLog from utils import date_utils, file_utils, memory_usage_resource from utils.common import cached_property @@ -43,91 +37,6 @@ thread_local_tags = threading.local() -def celery_task_handler(tasks, cleanup, kill_siblings_on_fail=False, d=0): - def find_kill_children(traceid): - criteria = {'object.consumers': traceid} - result = Task.getAll(criteria) - for t in result: - res = AsyncResult(t.extid) - if res.state not in ['SUCCESS', 'FAILURE']: - t.status = Task.STATUS_ERROR - t.save(is_partial=True, field_list=['object.status']) - t.current_status_msg = 'terminated' - res.revoke(terminate=True, signal='SIGKILL') - results = [] - failed_task_message = '' - failed = False - if d == 0: - task_list = [asynctasks.subtask_handler( - t[0], kwargs=t[1], queue=t[2]) for t in tasks] - elif d == 1: - task_list = [asynctasks.subtask_handler(t=t, d=d) for t in tasks] - elif d == 2: - task_list = [ - asynctasks.subtask_handler(t=t[0], queue=t[1], d=d) for t in tasks] - dup = [] - copy_list = [] - for task_entry in task_list: - if task_entry in copy_list: - dup.append(task_entry) - else: - copy_list.append(task_entry) - if dup: - failed = True - logger.error("Duplicate task ids in task list %s" % dup) - while task_list and not dup: - remainingtasks = task_list - time.sleep(2) - try: - for t in remainingtasks: - if t.ready(): - # if isinstance(result,list) else results.append(result) - resp = t.get() - if isinstance(resp, dict): - if large_response_key in resp: - resp = load_response(resp) - elif dup_celery_id in resp: - raise Exception("Tasks are having duplicate celery_id") - results.append(resp) - task_list.remove(t) - - except Exception as ex: - failed = True - failed_task_message += t.id + \ - ' failed with the message' + ex.message - task_list.remove(t) - if kill_siblings_on_fail: - break - - if kill_siblings_on_fail and failed: - for t in task_list: - mytask = Task.getByFieldValue('extid', t.task_id) - mytask.status = Task.STATUS_ERROR - mytask.current_status_msg = 'terminated' + failed_task_message - mytask.save(is_partial=True, field_list=['object.status', - 'object.current_status_msg']) - t.revoke(terminate=True, signal='SIGKILL') - find_kill_children(t.task_id) - if failed: - if cleanup is not None: - cleanup() - raise Exception( - 'Failures detected in subtasks:\n ' + failed_task_message) - return results - - -class CacheDBInterface: - def get_cache(self, res_id): - return ResultCache.getByFieldValue('extid', res_id) - - def getBySpecifiedCriteria(self, criteria): - return ResultCache.getBySpecifiedCriteria(criteria) - - def save_cache(self, res, **kwargs): - if 'is_partial' not in kwargs: - kwargs['is_partial'] = True - res.save(**kwargs) - class TaskDBInterface: def fetch_task(self, task_id): @@ -185,7 +94,6 @@ def new_task(user_and_tenant, *args, **options): tenantname = user_and_tenant[1] logintenant = user_and_tenant[2] - cache_db = CacheDBInterface() if 'cache_db' not in options else options['cache_db'] task_db = TaskDBInterface() if 'task_db' not in options else options['task_db'] task_metrics = logger.new_metrics() task_metrics.new_timer('task.time') @@ -498,182 +406,10 @@ def check_response_size(fn_response, user_and_tenant, trace): return fn_response -def load_response(resp_dict): - directory_name = tempfile.mkdtemp(dir=TMP_DIR) - try: - """ We expect that the large_response_key and filename are part of the resp_dict, this will be populated - in case of a large response during task processing """ - filename = resp_dict[filename_key] - key = resp_dict[large_response_key] - logger.info("loading response from %s-%s" % (key, filename)) - file_utils.download_from_s3(directory_name, filename, key) - with open(os.path.join(directory_name, filename), 'r') as f: - return json.load(f) - except Exception as ex: - msg = 're-raising exception %s while loading response of %s - %s' % ( - key, filename, ex) - logger.warning(msg) - raise GnanaError(msg) - finally: - shutil.rmtree(directory_name) - - def get_profile_filename(prefix=""): return "%sprofile_%s_%s.profile" % (prefix, os.getpid(), time.time()) -class UnknownTaskError(Exception): - - """ Raised when the completed task is not found to be pending""" - pass - - -def revokejobs_streaming(criteria, cleanup=False, reset_chipotle_trace=False): - try: - yield '{"affected_tasks":[' - comma = "" - success = "true" - tasklist = defaultdict(dict) - status = '' - status_criteria = {'object.status': {'$in': [Task.STATUS_CREATED, - Task.STATUS_STARTED, - Task.STATUS_SUBMITTED, - Task.STATUS_ERROR]}} - criteria = {'$and': [criteria, status_criteria]} - msg = '' - total_task_count = 0 - field_list = {'object.extid': 1, 'object.tenant': 1, - 'object.framework_version': 1, 'object.trace': 1} - if cleanup: - field_list['object.task_meta'] = 1 - field_list['object.tasktype'] = 1 - for task in gnana_db.findDocuments(Task.getCollectionName(), criteria, - fieldlist=field_list, - read_from_primary=True, - tenant_aware=Task.tenant_aware): - yield '%s"%s"' % (comma, task['object']['extid']) - comma = "," - total_task_count += 1 - if task['object']['framework_version'] == '2' and cleanup: - task_details = (task['object']['extid'], task['object']['task_meta'], - task['object']['tasktype'], task['object']['trace']) - else: - task_details = (task['object']['extid'], task['object']['trace']) - if task['object']['tenant'] in tasklist: - if task['object']['framework_version'] in tasklist[task['object']['tenant']]: - tasklist[task['object']['tenant']][ - task['object']['framework_version']].append(task_details) - else: - tasklist[task['object']['tenant']][task['object']['framework_version']] = [task_details] - else: - tasklist[task['object']['tenant']][task['object']['framework_version']] = [task_details] - progress = 0 - for tenant, framework_versions in tasklist.items(): - user_name = 'revoke' - logintenant = 'administrative.domain' - sec_context.set_context(user_name, tenant, logintenant, user_name, 'tenant', {}) - trace_list = [] - for version, total_tasks in framework_versions.items(): - if msg: - msg += ' and ' - msg += str(len(total_tasks)) + " v" + version + "_tasks" - logger.info("Retrieved %s v%s_tasks for revoking for %s" % (str(len(total_tasks)), version, tenant)) - if version == '1': - total_tasks_ = [] - for i, task_info in enumerate(total_tasks): - task_id, trace = task_info - total_tasks_.append(task_id) - if trace not in trace_list: - trace_list.append(trace) - progress = check_and_log_progress(i, total_task_count, progress) - res = AsyncResult(task_id) - if not res.ready(): - id_ = task_id - logger.error( - "terminating task %s, as one of the sub_tasks failed in this group", task_id) - celery.task.control.revoke(id_, terminate=True, signal='SIGKILL') - set_to_criteria = {'$set': {'object.status': Task.STATUS_ERROR, - 'object.current_status_msg': "revoked"}} - task_criteria = {'object.extid': {'$in': total_tasks_}} - gnana_db.updateAll(Task.getCollectionName(), task_criteria, set_to_criteria) - else: - total_tasks_ = [] - for i, taskdetail in enumerate(total_tasks): - if cleanup: - task_id, task_meta, path, trace = taskdetail - params = task_meta['params'] - context = task_meta['context'] - from aviso.framework.tasker import V2Task - v2task_obj = V2Task.create_task(path, params=params, context=context) - v2task_obj.cleanup_on_revoke() - else: - task_id, trace = taskdetail - total_tasks_.append(task_id) - if trace not in trace_list: - trace_list.append(trace) - task_criteria = {'object.extid': {'$in': total_tasks_}} - total_tasks = total_tasks_ - set_to_criteria = {'$set': {'object.status': Task.STATUS_TERMINATED, - 'object.current_status_msg': "revoked"}} - gnana_db.updateAll(Task.getCollectionName(), task_criteria, set_to_criteria) - TaskActive.truncate_or_drop(task_criteria) - criteria = {'object.requesting_task': {'$in': total_tasks}} - set_to_criteria = {'$set': {'object.status': ResultCache.FAILED}} - gnana_db.updateAll(ResultCache.getCollectionName(), criteria, set_to_criteria) - progress = check_and_log_progress(len(total_tasks), total_task_count, progress) - msg += " for " + tenant - msg = " Retrieved " + msg + " for revoking" - status = ',"status":"%s"' % msg - if reset_chipotle_trace: - chipotle_status = sec_context.details.get_flag('molecule', 'chipotle_trace', 'finished') - if chipotle_status in trace_list: - sec_context.details.set_flag('molecule', 'chipotle_trace', 'finished') - dtfo_trace = sec_context.details.get_flag('molecule', 'dtfo_trace', 'finished') - if dtfo_trace in trace_list: - sec_context.details.set_flag('molecule', 'dtfo_trace', 'finished') - load_activity_trace = sec_context.details.get_flag('molecule', 'load_activity_trace', 'finished') - if load_activity_trace in trace_list: - sec_context.details.set_flag('molecule', 'load_activity_trace', 'finished') - snapshot_trace = sec_context.details.get_flag('molecule', 'snapshot_trace', 'finished') - if snapshot_trace in trace_list: - sec_context.details.set_flag('molecule', 'snapshot_trace', 'finished') - logger.info("Revoke progress: %%100.") - if not tasklist: - msg = "Retrieved 0 tasks for revoking" - status += ',"status":"%s"' % msg - logger.info(msg) - yield '],"success":%s%s}' % (success, status) - except Exception as e: - logger.exception(e) - - -def check_and_log_progress(tasks_revoked, total_task_count, progress): - this_progress = float(tasks_revoked) / total_task_count - if this_progress - progress > 0.1: - logger.info("Revoke progress: %%%.0f", this_progress * 100.) - progress = this_progress - return progress - - -def revokejobs(traceid): - return json.loads(''.join(revokejobs_streaming(traceid))) - -def run_task(task, - args, - ): - """ - helper method to run tasks - Arguments: - task {classobj} -- task class - args {dict} -- task args - Returns: - dict -- success status of task executon - """ - task_instance = task(**args) - task_instance.process() - task_instance.persist() - return task_instance.return_value - class BaseTask(object): """ base class of interface each task in micro app must adhere to diff --git a/tasks/asynctasks.py b/tasks/asynctasks.py deleted file mode 100644 index c706691..0000000 --- a/tasks/asynctasks.py +++ /dev/null @@ -1,65 +0,0 @@ -import datetime -import logging -from datetime import UTC - -from aviso.framework import tracer -from aviso.framework.diagnostics import probe_util -from aviso.settings import POOL_PREFIX, WORKER_POOL, event_context -from celery import current_task - -from domainmodel.app import Task - -logger = logging.getLogger('gnana.%s' % __name__) - - -def subtask_handler(t, kwargs=None, queue=None, args=(), d=0): - if kwargs is None: - kwargs = {} - tid = None - ''' - Adding the event id to kwargs to pass it onto other child tasks for tenant events. - The event id stored in thread local as part of event context. - ''' - te_id = event_context.event_id - if d == 0: - kwargs['event_id'] = te_id - tid = t.apply_async(args=args, kwargs=kwargs, queue=queue) - - elif d == 1: - t.kwargs['event_id'] = te_id - tid = t.apply_async() - - elif d == 2: - t.kwargs['event_id'] = te_id - tid = t.apply_async(queue=queue) - - if not current_task: - ts = Task.getByFieldValue('trace', tracer.trace) - ts.extid = tid.id - ts.celery_id = tid.id - ts.main_id = tid.id - ts.submit_time = datetime.datetime.now(UTC) - ts.status = Task.STATUS_SUBMITTED - ts.pool_name = WORKER_POOL - ts.cname = POOL_PREFIX - ts.save(is_partial=True, field_list=['object.extid', - 'object.celery_id', - 'object.main_id', - 'object.submit_time', - 'object.status', - 'object.pool_name', - 'object.cname']) - else: - pid = current_task.request.id - na = tid.task_name - di = {} - if kwargs: - di['kwargs'] = kwargs - if args: - di['args'] = args - - if not Task.get_mainid(tracer.trace): - logger.info("Main task is None for (%s)", current_task.request.id) - Task.set_task(tracer.trace, Task.get_mainid(tracer.trace), tid.id, pid, Task.STATUS_SUBMITTED, di, na) - - return tid diff --git a/tasks/csv_tasks.py b/tasks/csv_tasks.py deleted file mode 100644 index c7aca9a..0000000 --- a/tasks/csv_tasks.py +++ /dev/null @@ -1,188 +0,0 @@ -import json -import logging -import sys -import traceback -from collections import defaultdict - -from aviso.settings import gnana_storage, sec_context -from pymongo.errors import BulkWriteError - -from domainmodel.csv_data import CSVDataClass, csv_version_decorator - -logger = logging.getLogger('gnana.%s' % __name__) - - -def csv_upload_task(kwargs): - filepath = kwargs['filepath'] - csv_file = gnana_storage.open(filepath) - reader = json.loads(csv_file.readlines()[0]) - return csvdatauploadprocess(reader, **kwargs) - - -@csv_version_decorator -def csvdatauploadprocess(reader, **kwargs): - filepath = kwargs.get('filepath',False) - csv_type = kwargs['csv_type'] - csv_name = kwargs['csv_name'] - CSVClass = CSVDataClass(csv_type, csv_name, used_for_write=True) - dd_type = kwargs['dd_type'] - static_type = kwargs['static_type'] - ret = {'inserts': 0, 'updates': 0} - - from_scratch = not(dd_type=='partial' or static_type=='partial') - - def process_batch(record_map): - bulk_list = CSVClass.bulk_ops() - starting = len(record_map) - inserts = 0 - updates = 0 - criteria_list = record_map.keys() - rec_list = [] - logger.info('-=' * 20) - # csv_record = CSVClass.getByFieldValue('extid', unique_value) - if not from_scratch and criteria_list: - matched_csv_records = CSVClass.getAll( - {'object.extid': {'$in': [sec_context.encrypt(x, CSVClass) if CSVClass.encrypted else x for x in criteria_list]}}) - else: - logger.info('from_scratch mode enabled. Not distinguishing between insert and update.') - matched_csv_records = [] - update_map = defaultdict(list) - matched_keys = set() - matched_count = 0 - for matched_rec in matched_csv_records: - matched_count += 1 - try: - matched_rec_extid = str(matched_rec.extid) if isinstance(matched_rec.extid, str) else matched_rec.extid - except: - matched_rec_extid = matched_rec.extid - if matched_rec_extid in record_map: - update_map[matched_rec_extid] = record_map.pop( - matched_rec_extid) - matched_keys.add(matched_rec_extid) - else: - logger.info( - f'matched record is not found in the record map {matched_rec_extid}') - logger.info(f'Matched records count {matched_count} - total criteria list {len(criteria_list)}') - if not from_scratch: - deals_to_insert = set(criteria_list) - matched_keys - else: - deals_to_insert = [] - for k in deals_to_insert: - try: - recs = record_map.pop(k) if k in record_map else None - if recs: - csv_obj = CSVClass() - c_rec = recs.pop(0) - csv_obj.apply_fields(c_rec) - rec_list.append(csv_obj) - else: - logger.info( - 'Expecting record for insert - Key not found in the map of csv recs %s ' % k) - if recs: - logger.info( - 'Multiple records found for the same cache key %s - update count %s' % (k, len(recs))) - update_map[k] = recs - except: - logger.exception(f"Key not found {k}") - pass - # Insert recs - try: - CSVClass.bulk_insert(rec_list) - inserts += len(rec_list) - except Exception as e: - logger.info(f'bulk_insert failed with msg {e} \nsaving one_by_one') - res = save_one_by_one(rec_list, **kwargs) - updates = res['updates'] - inserts = res['inserts'] - rec_list = [] - # Update recs - recs_to_update = record_map if from_scratch else update_map - for key, csv_records in recs_to_update.iteritems(): - csv_obj = CSVClass() if from_scratch else CSVClass.getByFieldValue('extid', key) - # FIXME: for some collections getByFieldValue returns a list of a single csv object?? - if isinstance(csv_obj, list): - csv_obj = csv_obj[0] - for csv_record in csv_records: - updates += 1 - csv_obj.apply_fields(csv_record, partial_dd=(dd_type == 'partial'), - partial_static=(static_type == 'partial')) - csv_obj.save(bulk_list=bulk_list) - if updates: - try: - bulk_list.execute() - except BulkWriteError as e: - logger.exception(e) - e.message += "| " + str(e.details)[:1000] + " ... " - raise - - logger.info('Updated %s records ' % updates) - logger.info('Inserted %s records ' % inserts) - logger.info('starting %s ' % starting) - ret['inserts'] += inserts - ret['updates'] += updates - return ret - # checking the existance of table if the csv_type is a postgres enabled - csv_obj = CSVClass() - if csv_obj.postgres: - prefix = "%s._csvdata.%s." % (sec_context.name, kwargs['csv_type']) - all_collections = [x for x in CSVClass.list_all_collections(prefix)] - if csv_obj.getCollectionName() not in all_collections: - logger.info("Table not exist trying to create one.") - try: - tc = sec_context.details - config = tc.get_config('csv_data', kwargs['csv_type']) - csv_obj.create_postgres_table(config, tc.is_encrypted) - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - err_msg = "\n".join(traceback.format_exception(exc_type, exc_value, exc_traceback)) - message = "\n".join(["ERROR: custom message", e, err_msg]) - logger.error(message) - raise e - - batch_size = 5000 - rec_count = 0 - record_map = defaultdict(list) - for record in reader: - unique_value = CSVClass.getUniqueValue(record) - record_map[unique_value].append(record) - rec_count += 1 - if rec_count % batch_size == 0: - process_batch(record_map) - record_map = defaultdict(list) - # process Remaining entries - process_batch(record_map) - if ret['inserts'] == 0 and ret['updates'] == 0: - logger.info('Nothing passed to save in the CSV data upload') - logger.info('-=' * 20) - if filepath: - gnana_storage.delete_adhoc_file(filepath) - return ret - return ret - - -def save_one_by_one(rec_list, **kwargs): - csv_type = kwargs['csv_type'] - csv_name = kwargs['csv_name'] - CSVClass = CSVDataClass(csv_type, csv_name, used_for_write=True) - dd_type = kwargs.get('dd_type', 'complete') - static_type = kwargs.get('static_type', 'complete') - res = {'inserts': 0, 'updates': 0} - for record in rec_list: - try: - record.save() - res['inserts'] += 1 - except: - localattrs = {} - record.encode(localattrs) - localattrs.pop('extid') - df = localattrs.pop('dynamic_fields') - localattrs.update(df) - csv_obj = CSVClass.getByFieldValue('extid', record.extid) - if csv_obj: - csv_obj.apply_fields(localattrs, partial_dd=(dd_type == 'partial'), - partial_static=(static_type == 'partial')) - res['updates'] += 1 - else: - logger.info('unable to process %s' % record.extid) - csv_obj.save() - return res diff --git a/tasks/dataload.py b/tasks/dataload.py deleted file mode 100644 index 620e0f4..0000000 --- a/tasks/dataload.py +++ /dev/null @@ -1,98 +0,0 @@ -import datetime -import logging - -import celery -from aviso.settings import gnana_db, sec_context -from aviso.framework import tracer -import pytz - -from domainmodel.datameta import Dataset -from domainmodel.uip import MAX_CREATED_DATE -from tasks import asynctasks, gnana_task_support -from utils import queue_name - - -logger = logging.getLogger('gnana.%s' % __name__) - - -@celery.task -def process_inbox_entries(user_and_tenant, dsname, inboxname, idlist, **kwargs): - return process_inbox_entries_task(user_and_tenant, dsname, inboxname, idlist, **kwargs) - - -@gnana_task_support -def process_inbox_entries_task(user_and_tenant, dsname, inboxname, idlist, **kwargs): - ds = Dataset.getByNameAndStage(dsname, inboxname) - for x in idlist: - if not x: - continue - record = ds.DatasetClass.getByFieldValue('extid', x) - if not record: - record = ds.DatasetClass(extid=x) - docs = gnana_db.findDocuments(ds.InboxEntryClass.getCollectionName(), - {'object.extid': sec_context.encrypt(x)}) - for doc in docs: - if doc['_kind'] == 'domainmodel.uip.InboxFileEntry': - logger.exception("Unexpected Kind of object") - inbox_entry = ds.InboxEntryClass(doc) - file_type = ds.filetype(inbox_entry.filetype) - file_type.apply_inbox_record(record, inbox_entry, fld_config=ds.fields) - record.compress() - if not record.created_date == MAX_CREATED_DATE: - record.save() - else: - logger.warning( - 'Ignoring record with id %s has no fields.', record.ID) - - -def merge_inbox_entries_to_uip(user_and_tenant, datasetname, stagename, - auto_purge=True): - dataset = Dataset.getByNameAndStage(datasetname, stagename, full_config=False) - - # Get the unique ext-id values and pass 1000 - # at a time to workers for processing - distinct_entries = gnana_db.getDistinctValues( - dataset.InboxEntryClass.getCollectionName(), - "object.extid") - record_count = len(distinct_entries) - - tasks = [] - while distinct_entries: - firstslice = distinct_entries[0:1000] - distinct_entries = distinct_entries[1000:] - tasks.append(process_inbox_entries.subtask(args=(user_and_tenant, dataset.name, stagename, firstslice,), - kwargs={ - 'trace': tracer.trace}, - options={'queue': queue_name('worker')})) - - if tasks: - async_res = [asynctasks.subtask_handler( - t, queue=queue_name('worker'), d=2) for t in tasks] - # We have to wait for the response. Otherwise we will drop the - # inboxes while the id's are in process - for t in async_res: - t.get() - - docs = gnana_db.findDocuments(dataset.InboxEntryClass.getCollectionName(), - {u"_kind": u"domainmodel.uip.InboxFileEntry"}) - types_and_dates = {} - for file_entry in docs: - f = dataset.InboxFileEntryClass(file_entry) - if f.filetype in types_and_dates: - if f.when: - types_and_dates[f.filetype] = max( - types_and_dates[f.filetype], f.when) - else: - types_and_dates[f.filetype] = f.when or datetime.datetime( - 1970, 1, 1, tzinfo=pytz.utc) - - for each_type, value_for_type in types_and_dates.items(): - dataset.file_types[each_type].last_approved_time = value_for_type - - dataset.save() - - if auto_purge: - # Clear the inbox, we don't need it anymore - gnana_db.dropCollection(dataset.InboxEntryClass.getCollectionName()) - - return record_count diff --git a/tasks/drilldown/drill_down_tasks.py b/tasks/drilldown/drill_down_tasks.py new file mode 100644 index 0000000..0cac129 --- /dev/null +++ b/tasks/drilldown/drill_down_tasks.py @@ -0,0 +1,143 @@ +import logging +from urllib.parse import quote + +from aviso.settings import sec_context + +from deal_service.tasks import DealsTask +from infra.constants import DEALS_COLL, NEW_DEALS_COLL +from infra.write import _remove_run_full_mode_flag +from tasks.hierarchy.hier_sync_tasks import Sync +from tasks.sync_drilldown import SyncDrilldown +from utils.date_utils import prev_periods_allowed_in_deals + +logger = logging.getLogger('aviso-core.%s' % __name__) + + +class DrilldownSyncTask: + + def execute(self, period): + try: + sync_obj = SyncDrilldown(period=period) + sync_obj.process() + sync_obj.persist() + result = sync_obj.return_value + return {'success': True, 'result': result} + except Exception as e: + logger.exception(e) + return {'success': False, + 'error_msg': e} + +class DrilldownSyncLeadTask: + + def execute(self, period): + try: + sync_obj = SyncDrilldown(period=period, service='leads') + sync_obj.process() + sync_obj.persist() + result = sync_obj.return_value + return {'success': True, 'result': result} + except Exception as e: + logger.exception(e) + return {'success': False, + 'error_msg': e} + + +class CaptureDrilldownsDiffsTask(DealsTask): + + def execute_forcefully(self): + return True + + def execute(self, period, *args, **kwargs): + self.period = period + # fetch the result from gbm results + new_deal_results = self.fetch_deal_results_from_gbm() + new_opp_ids_and_nodes_drilldowns = {} + new_opp_ids_and_nodes_hierachy = {} + # prepare new drilldowns + for p, data in new_deal_results.iteritems(): + if not data: + continue + + as_of = data['timestamp'] + + for opp_id, deal in data['results'].iteritems(): + deal['opp_id'] = opp_id + hierarchy_list, drilldown_list = self.adorn_hierarchy(deal, as_of) + new_opp_ids_and_nodes_drilldowns[opp_id] = drilldown_list + new_opp_ids_and_nodes_hierachy[opp_id] = hierarchy_list + # fetch the current deals in deals collection + old_deal_results = self.fetch_deals_from_deals_results() + old_opp_ids_and_nodes_drilldowns = {} + old_opp_ids_and_nodes_hierarchy = {} + for res in old_deal_results: + old_opp_ids_and_nodes_drilldowns[res['opp_id']] = res.get('drilldown_list', []) + old_opp_ids_and_nodes_hierarchy[res['opp_id']] = res.get('hierarchy_list', []) + + # capture the impacted opp_ids + impacted_opp_ids = [] + impacted_drilldowns = [] + for opp_id in new_opp_ids_and_nodes_drilldowns: + new_drilldowns = new_opp_ids_and_nodes_drilldowns[opp_id] + old_drilldowns = None + if opp_id in old_opp_ids_and_nodes_drilldowns: + old_drilldowns = old_opp_ids_and_nodes_drilldowns[opp_id] + diffs = [x for x in set(list(old_drilldowns + new_drilldowns)) if x not in new_drilldowns or x not in old_drilldowns] + if diffs: + impacted_opp_ids.append(opp_id) + impacted_drilldowns += diffs + hierarchy_impacted_opp_ids = [] + for opp_id in new_opp_ids_and_nodes_hierachy: + if opp_id in impacted_opp_ids: + continue + new_hierarchy = new_opp_ids_and_nodes_hierachy[opp_id] + old_hierarchy = None + if opp_id in old_opp_ids_and_nodes_hierarchy: + old_hierarchy = old_opp_ids_and_nodes_hierarchy[opp_id] + diffs = [x for x in set(list(old_hierarchy + new_hierarchy)) if x not in new_hierarchy or x not in old_hierarchy] + if diffs: + hierarchy_impacted_opp_ids.append(opp_id) + if hierarchy_impacted_opp_ids: + logger.warning("opp_ids impacted by hierarchy changes only %s" % hierarchy_impacted_opp_ids) + _remove_run_full_mode_flag() + return {'result': {'impacted_opp_ids': impacted_opp_ids, + 'impacted_drilldowns': list(set(impacted_drilldowns)), + 'hierarchy_impacted_opp_ids': hierarchy_impacted_opp_ids}, + 'success': True} + + def fetch_deal_results_from_gbm(self): + gbm_svc = sec_context.get_microservice_config('gbm_service') + if not gbm_svc: + logger.warning('no gbm service found') + return {} + fields = ['__segs'] + for owner_field, drilldown in self.config.owner_id_fields: + if drilldown == 'CRR': + continue # Note: we are not using load_changes task for CRR + fields.append(owner_field) + field_params = "&".join(["=".join(["fields", quote(field)]) for field in fields]) + url = '/gbm/deals_results?period={}&{}'.format(self.period, field_params) + gbm_shell = sec_context.gbm + return gbm_shell.api(url, None) + + def fetch_deals_from_deals_results(self, db=None): + deals_collection = db[DEALS_COLL] if db else sec_context.tenant_db[DEALS_COLL] + prev_periods = prev_periods_allowed_in_deals() + if self.period in prev_periods and self.config.config.get('update_new_collection'): + deals_collection = sec_context.tenant_db[NEW_DEALS_COLL] + project_fields = {'drilldown_list': 1, 'opp_id': 1, 'hierarchy_list': 1} + all_deals = list(deals_collection.find({}, project_fields)) + return all_deals + +class HierSyncLeadTask: + + def execute(self, period): + try: + sync_obj = Sync(period=period, service='leads') + sync_obj.process() + sync_obj.persist() + result = sync_obj.return_value + return {'success': True, 'result': result} + except Exception as e: + logger.exception(e) + return {'success': False, + 'error_msg': e} diff --git a/tasks/fields.py b/tasks/fields.py index 5ce98c0..00b9cb0 100644 --- a/tasks/fields.py +++ b/tasks/fields.py @@ -1,4 +1,4 @@ -from domainmodel import Model +from domainmodel.model import Model from utils.date_utils import EpochClass import logging from datetime import timedelta @@ -6,46 +6,6 @@ logger = logging.getLogger('gnana.%s' % __name__) -class StringMaps(Model): - tenant_aware = True - collection_name = 'stringmap' - version = 1 - kind = 'domianmodel.stringmap.StringMaps' - encrypted = False - - def __init__(self, attrs=None): - self.field_name = "" - self.maps = {} - self.rev_maps = {} - super(StringMaps, self).__init__(attrs) - - def encode(self, attrs): - attrs['field_name'] = self.field_name - attrs['strings'] = self.maps.items() - super(StringMaps, self).encode(attrs) - - def decode(self, attrs): - self.maps = dict(attrs['strings']) - self.field_name = attrs['field_name'] - self.rev_maps = [(v, k) for k, v in self.maps.items()] - return super(StringMaps, self).decode(attrs) - - @classmethod - def add_map(cls, field, value): - x = StringMaps.getByFieldValue('field_name', field) - - if not x: - x = StringMaps() - x.field_name = field - x.maps = {} - if value in x.maps: - return x.maps[value] - string_id = len(x.maps) - x.maps = dict(list(x.maps.items()) + [(value, string_id)]) - x.save() - return string_id - - def date_parser(value, out_type='epoch', fmt='%Y%m%d', add_seconds=None): ep = EpochClass.from_string(value, fmt, timezone='tenant') if add_seconds and isinstance(add_seconds, int): @@ -96,14 +56,6 @@ def parse_field(val, field_config): } -# need to update list manually if new parser included -def get_all_parser(): - all_parsers = [] - for parser,func in parse_fns.items(): - all_parsers.append(parser) - - return all_parsers - def _parse(val, fn, parser_fallback_lambda=None): if fn not in parse_fns: # Trying to see if the input parser is a lambda fn or not. @@ -119,13 +71,3 @@ def _parse(val, fn, parser_fallback_lambda=None): if parser_fallback_lambda: return eval(parser_fallback_lambda)(val) raise e - - -def parse_record_fields(ds, record): - """ - Converts the fields in the record into their respective data types. - """ - for key in record: - if key in ds.fields and "type" in ds.fields[key]: - record[key] = _parse(record.get(key), ds.fields[key]['type']) - return record diff --git a/tasks/hierarchy/hier_sync_tasks.py b/tasks/hierarchy/hier_sync_tasks.py index df0ba13..d2a4155 100644 --- a/tasks/hierarchy/hier_sync_tasks.py +++ b/tasks/hierarchy/hier_sync_tasks.py @@ -1,171 +1,23 @@ import logging +from aviso.settings import sec_context + from config import HierConfig -from config.hier_config import HIERARCHY_BUILDERS, write_hierarchy_to_gbm -from deal_service.tasks import DealsTask -from infra import DEALS_COLL, NEW_DEALS_COLL +from config import HIERARCHY_BUILDERS, write_hierarchy_to_gbm from infra.read import (fetch_hidden_nodes, fetch_node_to_parent_mapping_and_labels, get_period_as_of, get_period_begin_end) -from infra.write import _remove_run_full_mode_flag -from tasks import run_task, BaseTask +from tasks import BaseTask from tasks.hierarchy import (draw_tree, graft_pruned_tree, make_new_hierarchy, make_pruned_tree, make_valid_tree) -from tasks.sync_drilldown import SyncDrilldown -from utils.date_utils import prev_periods_allowed_in_deals from utils.misc_utils import is_lead_service, try_index from utils.mongo_writer import (create_many_nodes, create_node, hide_node, label_node, move_node, unhide_node) -from aviso.settings import sec_context -from aviso.framework.tasker import BaseV2Task -from urllib.parse import quote logger = logging.getLogger('aviso-core.%s' % __name__) -class DrilldownSyncTask(BaseV2Task): - - def create_dependencies(self): - if self.params.get('stand_alone'): - return {} - params = {'period': self.params.get('period'), 'temp_val': self.params.get('temp_val')} - return {'hier': HierSyncTask(params=params)} - - def execute(self, dep_results): - try: - result = run_task(SyncDrilldown, {'period': self.params.get('period')}) - return {'success': True, 'result': result} - except Exception as e: - logger.exception(e) - return {'success': False, - 'error_msg': e} - -class DrilldownSyncLeadTask(BaseV2Task): - - def create_dependencies(self): - #if self.params.get('stand_alone'): - # return {} - params = {'period': self.params.get('period'), 'service': 'leads', 'temp_val': self.params.get('temp_val')} - return {'hier': HierSyncLeadTask(params=params)} - - def execute(self, dep_results): - try: - result = run_task(SyncDrilldown, {'period': self.params.get('period'), 'service': 'leads'}) - return {'success': True, 'result': result} - except Exception as e: - logger.exception(e) - return {'success': False, - 'error_msg': e} - - -class CaptureDrilldownsDiffsTask(BaseV2Task, DealsTask): - - def execute_forcefully(self): - return True - - def create_dependencies(self): - if self.params.get('stand_alone'): - return {} - params = {'period': self.params.get('period'), 'temp_val': self.params.get('temp_val')} - return {'hier': HierSyncTask(params=params)} - - def execute(self, dep_results, *args, **kwargs): - self.period = self.params.get('period') - # fetch the result from gbm results - new_deal_results = self.fetch_deal_results_from_gbm() - new_opp_ids_and_nodes_drilldowns = {} - new_opp_ids_and_nodes_hierachy = {} - # prepare new drilldowns - for period, data in new_deal_results.iteritems(): - if not data: - continue - - as_of = data['timestamp'] - - for opp_id, deal in data['results'].iteritems(): - deal['opp_id'] = opp_id - hierarchy_list, drilldown_list = self.adorn_hierarchy(deal, as_of) - new_opp_ids_and_nodes_drilldowns[opp_id] = drilldown_list - new_opp_ids_and_nodes_hierachy[opp_id] = hierarchy_list - # fetch the current deals in deals collection - old_deal_results = self.fetch_deals_from_deals_results() - old_opp_ids_and_nodes_drilldowns = {} - old_opp_ids_and_nodes_hierarchy = {} - for res in old_deal_results: - old_opp_ids_and_nodes_drilldowns[res['opp_id']] = res.get('drilldown_list', []) - old_opp_ids_and_nodes_hierarchy[res['opp_id']] = res.get('hierarchy_list', []) - - # capture the impacted opp_ids - impacted_opp_ids = [] - impacted_drilldowns = [] - for opp_id in new_opp_ids_and_nodes_drilldowns: - new_drilldowns = new_opp_ids_and_nodes_drilldowns[opp_id] - old_drilldowns = None - if opp_id in old_opp_ids_and_nodes_drilldowns: - old_drilldowns = old_opp_ids_and_nodes_drilldowns[opp_id] - diffs = [x for x in set(list(old_drilldowns + new_drilldowns)) if x not in new_drilldowns or x not in old_drilldowns] - if diffs: - impacted_opp_ids.append(opp_id) - impacted_drilldowns += diffs - hierarchy_impacted_opp_ids = [] - for opp_id in new_opp_ids_and_nodes_hierachy: - if opp_id in impacted_opp_ids: - continue - new_hierarchy = new_opp_ids_and_nodes_hierachy[opp_id] - old_hierarchy = None - if opp_id in old_opp_ids_and_nodes_hierarchy: - old_hierarchy = old_opp_ids_and_nodes_hierarchy[opp_id] - diffs = [x for x in set(list(old_hierarchy + new_hierarchy)) if x not in new_hierarchy or x not in old_hierarchy] - if diffs: - hierarchy_impacted_opp_ids.append(opp_id) - if hierarchy_impacted_opp_ids: - logger.warning("opp_ids impacted by hierarchy changes only %s" % hierarchy_impacted_opp_ids) - _remove_run_full_mode_flag() - return {'result': {'impacted_opp_ids': impacted_opp_ids, - 'impacted_drilldowns': list(set(impacted_drilldowns)), - 'hierarchy_impacted_opp_ids': hierarchy_impacted_opp_ids}, - 'success': True} - - def fetch_deal_results_from_gbm(self): - gbm_svc = sec_context.get_microservice_config('gbm_service') - if not gbm_svc: - logger.warning('no gbm service found') - return {} - fields = ['__segs'] - for owner_field, drilldown in self.config.owner_id_fields: - if drilldown == 'CRR': - continue # Note: we are not using load_changes task for CRR - fields.append(owner_field) - field_params = "&".join(["=".join(["fields", quote(field)]) for field in fields]) - url = '/gbm/deals_results?period={}&{}'.format(self.period, field_params) - gbm_shell = sec_context.gbm - return gbm_shell.api(url, None) - - def fetch_deals_from_deals_results(self, db=None): - deals_collection = db[DEALS_COLL] if db else sec_context.tenant_db[DEALS_COLL] - prev_periods = prev_periods_allowed_in_deals() - if self.period in prev_periods and self.config.config.get('update_new_collection'): - deals_collection = sec_context.tenant_db[NEW_DEALS_COLL] - project_fields = {'drilldown_list': 1, 'opp_id': 1, 'hierarchy_list': 1} - all_deals = list(deals_collection.find({}, project_fields)) - return all_deals - -class HierSyncLeadTask(BaseV2Task): - - def create_dependencies(self): - return {} - - def execute(self, dep_results): - try: - result = run_task(Sync, {'period': self.params.get('period'), 'service': 'leads'}) - return {'success': True, 'result': result} - except Exception as e: - logger.exception(e) - return {'success': False, - 'error_msg': e} - - -class HierSyncTask((BaseV2Task)): +class HierSyncTask: def execute(self, period): try: diff --git a/tasks/hierarchy/hierarchy_utils.py b/tasks/hierarchy/hierarchy_utils.py index b40125b..69e7637 100644 --- a/tasks/hierarchy/hierarchy_utils.py +++ b/tasks/hierarchy/hierarchy_utils.py @@ -1,493 +1,12 @@ import logging -from collections import namedtuple -from itertools import chain, zip_longest, repeat from aviso.settings import sec_context -from utils.cache_utils import memcacheable -from utils.misc_utils import prune_pfx, try_float -from utils.time_series_utils import slice_timeseries logger = logging.getLogger("gnana." + __name__) -NodeName = namedtuple('NodeName', ['seg', 'segtype', 'full_name', 'display_name']) - - -def construct_child_nodes(parent, name, dd_flds, is_rep=False): - """ - gen exp of child nodes, will pad out repeated nodes to leaf level if is_rep is True - """ - try: - drilldowns = next(x for x in dd_flds if parent.startswith(x[0])) - except StopIteration: - drilldowns = dd_flds[0] - try: - - if parent == '.~::~summary': - back = [] - else: - back = parent.split('~::~')[1].split('~') - child_hier_levels = len(back) + 1 - if not is_rep: - child_front = chain(drilldowns) - child_back = chain(back, repeat(name)) - yield '~::~'.join(['~'.join(next(child_front) for x in range(child_hier_levels)), - '~'.join(next(child_back) for x in range(child_hier_levels))]) - else: - for level in range(child_hier_levels, len(drilldowns) + 1): - child_front = chain(drilldowns) - child_back = chain(back, repeat(name)) - yield '~::~'.join(['~'.join(next(child_front) for x in range(level)), - '~'.join(next(child_back) for x in range(level))]) - except IndexError: - logger.warning('scary node being added, parent: %s, name: %s', parent, name) - yield '~::~'.join([parent, name]) - - -def pad_to_rep_level(node, hier_flds): - """ - repeat a node to rep level of heirarchy - """ - back = node.split('~::~')[1].split('~') - name = back[-1] - rep_levels = len(hier_flds) - node_front = chain(hier_flds) - node_back = chain(back, repeat(name)) - return '~::~'.join(['~'.join(next(node_front) for x in range(rep_levels)), - '~'.join(next(node_back) for x in range(rep_levels)) - ]) - - -@memcacheable('hierarchy_fields') -def get_hierarchy_fields(): - # disgusting hack to match custom user ds levels - #TODO: Need discussion: Understanding how domainmodel is working - from domainmodel.datameta import Dataset, UIPIterator - ds = Dataset.getByNameAndStage('CustomUserDS') - if not ds: - logger.warning('no CustomUserDS found, using default fields') - return ['Level_0', 'Level_1', 'frozen_Owner'] - for idx, record in enumerate(UIPIterator(ds, {}, None)): - if idx >= 1: - break - rec = record.featMap - rec.pop('UIPCreatedDate') - return sorted(rec.keys()) - - -def get_ownerID_mapping(as_of=None): - owner_dict = {} - # TODO: Need discussion: Understanding how domainmodel is working - from domainmodel.datameta import Dataset, UIPIterator - ds = Dataset.getByNameAndStage('CustomUserDS') - if not ds: - return owner_dict - if not as_of: - as_of = float('inf') - uip_recs = list(UIPIterator(ds, {}, None)) - if not len(uip_recs): - raise Exception('Cannot build OwnerID mapping. No CustomUserDS records were found.') - for record in uip_recs: - rec = record.featMap - rec.pop('UIPCreatedDate', None) - node_string = '~'.join([slice_timeseries([t for t, _ in rec[key]], - [v for _, v in rec[key]], - as_of, - use_fv=True) for key in sorted(rec.keys())]) - owner_dict[record.ID] = node_string - owner_dict[node_string] = record.ID - - return owner_dict - - -def get_all_owner_ids(): - owner_ids = set() - # TODO: Need discussion: Understanding how domainmodel is working - from domainmodel.datameta import Dataset, UIPIterator - ds = Dataset.getByNameAndStage('CustomUserDS') - if not ds: - return owner_ids - for record in UIPIterator(ds, {}, None): - owner_ids.add(record.ID) - return owner_ids - - -def make_valid_crm_id(crm_id): - # TODO: make this not dumb - if len(crm_id) == 18 and crm_id[:3] == '005': - # This is a 18 character SFDC ID. We only want first 15 chars. - return crm_id[:15] - if (len(crm_id) < 15) or (crm_id[:3] != '005') or (crm_id in get_all_owner_ids()): - # Can't make a valid ID. - return None - return crm_id - - -def explode_node(node, user_id=None): - """ Explodes a self-describing node identifier into a dictionary with fields and vals.""" - ks, vs = [x.split('~') for x in node.split('~::~')] - - # dont include new mongo/aryaka style top hierarchy levels - vs = chain([v for i, v in enumerate(vs) if i or try_float(ks[i][-1], None) is not None], repeat(vs[-1])) - - try: - hier_levels = get_hierarchy_fields() - except: - hier_levels = ks - - return dict(zip(hier_levels, vs)) - - -def trim_node(node): - ks, vs = node.split('~::~') - back_parts = vs.split('~') - return '~::~'.join([ks, '']) - - -def node_values(node, hier_flds): - _, vs = node.split('~::~') - vs = vs.split('~') - return [x[0] for x in (zip_longest(vs, hier_flds, fillvalue=vs[-1]))] - - -# TODO: review and make sure this handles all required cases -def in_hierarchy(hierarchy, node, root_dim=None): - """ - checks if node is in hierarchy - hierarchy is first part of root dimension - in_hierarchy('as_of_OverlayOwner','as_of_OverlayOwner~frozen_Owner~::~Sandra Boyd~Austin Brannan') -> True - """ - if not hierarchy: - return True - elif node.startswith(hierarchy): - return True - elif node == '.~::~summary': - return root_dim is None or hierarchy == root_dim - elif node.startswith('.'): - return node.endswith(hierarchy) - return False - - -def hier_switch(node, replacement, begin, end, length): - """ - try to switch node to new hierarchy - """ - try: - front, back = node.split(begin) - back_parts = back.split(end) - back = end.join([replacement] + back_parts[length:]) - return begin.join([front, back]) - except Exception as e: - logger.warning(e) - return None - - -def check_parallel_nodes(nodes): - """ - check if nodes are same in parallel hierarchies - """ - if len(nodes) == 1: - return True - ancestors = set() - hier_flds = get_hierarchy_fields() - for node in nodes: - ks, vs = node.split('~::~') - ks, vs = ks.split('~'), vs.split('~') - ancestors.add('~'.join(vs[i] for i, k in enumerate(ks) if prune_pfx(k) in hier_flds)) - return len(ancestors) == 1 - - -def switch_hierarchy(origin_node, move_node): - move_ks, move_vs = [x.split('~') for x in move_node.split('~::~')] - origin_ks, origin_vs = [x.split('~') for x in origin_node.split('~::~')] - try: - hier_flds = [prune_pfx(x) for x in get_hierarchy_fields() + ['Owner']] - except: - hier_flds = max(move_ks, origin_ks) - - move_hier_parts = '~'.join(move_vs[i] for i, k in enumerate(move_ks) if prune_pfx(k) in hier_flds) - origin_hier_parts = '~'.join(origin_vs[i] for i, k in enumerate(origin_ks) if prune_pfx(k) not in hier_flds) - - return '~::~'.join(['~'.join(move_ks), '~'.join([origin_hier_parts, move_hier_parts])]) - - -def same_hierarchy(node, other_node): - hier_flds = [prune_pfx(x) for x in get_hierarchy_fields() + ['Owner']] - node_ks, node_vs = [x.split('~') for x in node.split('~::~')] - other_ks, other_vs = [x.split('~') for x in other_node.split('~::~')] - node_hier_parts = '~'.join(node_vs[i] for i, k in enumerate(node_ks) if prune_pfx(k) not in hier_flds) - other_hier_parts = '~'.join(other_vs[i] for i, k in enumerate(other_ks) if prune_pfx(k) not in hier_flds) - return node_hier_parts == other_hier_parts - - -def check_privilege(user, node): - if node is None: - return True - try: - user_nodes = get_user_permissions(user, 'results') - except: - user_nodes = ['*'] - node_pfx, node_sfx = node.split('~::~') - return any((user_node == '*' or visible(user_node, node_sfx, node_pfx)) for user_node in user_nodes) - - -def visible(from_node, node_sfx, node_pfx=''): - """Returns true if node is visible from from_node. - Eg, If I'm at fL_0~::~EMEA, then EMEA~Benelux is visible to me. - """ - try: - return (node_sfx is None or from_node == '*' or - from_node.endswith('summary') or - (node_sfx.startswith(from_node.split('~::~')[1]) and (not node_pfx or - node_pfx.startswith(from_node.split('~::~')[0])))) - except IndexError: - # there are several incompatible user configurations floating around - # preferred: user.roles is a dict with key 'results' which is another dict, the keys of which are nodes - # bad: user.roles is a list of lists, one of those lists starts with 'results' and the other items are nodes - # bad: user.roles is a dict with key 'results' which is another dict, the keys of which are node prefixes - # logger.warn('Cant evaluate visibility for node sfx: %s, for user node: %s, trying for node pfx: %s', node_sfx, from_node, node_pfx) - user = sec_context.get_effective_user() - # logger.warn('User details for unevaluable user" %s', user.roles) - return from_node in node_pfx - except AttributeError: - # if, eg. from_node is None - return False - - -def can_see(from_node, target_node): - """ Returns true if node is visible from from_node. - TODO: Combine with . - Eg, If I'm at fL_0~::~EMEA, then fL_0~fL_1~::~EMEA~Benelux is visible to me. - """ - return (target_node is None or - from_node.endswith('summary') or - target_node.split('~::~')[1].startswith(from_node.split('~::~')[1])) - - -def is_leaf(node, wrd=None): - """Approximates checkings whether a node is a leaf .""" - if not wrd: - wrd = get_wrd() - return node.split('~::~')[0] in wrd - - -def should_hide(node): - """ - returns true if node has __ in name and should be hidden - """ - _, v = node.split('~::~') - v_parts = v.split('~') - if '__' in v_parts[-1]: - return True - return False - - -def node_and_parent(k, v): - """Returns node and parent node""" - v_parts = v.split('~') - if v_parts[0] == 'summary': - # Fake parent of '.~::~summary' - parent_node = u'root~::~root' - elif len(v_parts) == 1: - parent_node = u'.~::~summary' - else: - parent_node = '~'.join(k.split('~')[:-1]) + '~::~' + '~'.join(v_parts[:-1]) - node = k + '~::~' + v - return node, parent_node - - -def get_parent(node): - """Given a node, return (my_node, parents_node)""" - k, v = node.split('~::~') - v_parts = v.split('~') - - if v_parts[0] == 'summary': - # Fake parent of '.~::~summary' - parent_node = u'root~::~root' - elif len(v_parts) == 1: - parent_node = u'.~::~summary' - else: - parent_node = '~'.join(k.split('~')[:-1]) + '~::~' + '~'.join(v_parts[:-1]) - - return parent_node - - -# TODO this is disgusting -# TODO ask team for help to make this more better :) -def get_toplevel_parent(node): - """Given a node, return (my_node, parents_node)""" - k, v = node.split('~::~') - v_parts = v.split('~') - - if v_parts[0] == 'summary': - # Fake parent of '.~::~summary' - parent_node = u'root~::~root' - elif len(v_parts) == 1: - parent_node = u'.~::~summary' - else: - rev = v_parts[:-1][::-1] - - if len(rev) != 1: - gen = (i for i, val in enumerate(rev) if val != rev[i + 1] and rev[i + 1]) - idx = next(gen) + 1 - parent_node = '~'.join(k.split('~')[:-idx]) + '~::~' + '~'.join(v_parts[:-idx]) - else: - parent_node = '~'.join(k.split('~')[:-1]) + '~::~' + '~'.join(v_parts[:-1]) - - return parent_node - - -def get_ancestors(node, root_dim=None): - """Returns ancestors of node""" - ancestors = ['.~::~summary'] - if node == '.~::~summary': - return ancestors - seg_and_seg_val = node.split('~::~') - try: - flds, vals = [x.split('~') for x in seg_and_seg_val] - except ValueError as e: - raise Exception('Invalid leaf: %s. Msg: %s' % (seg_and_seg_val, e)) - return ancestors + ['{}~::~{}'.format('~'.join(flds[0:i + 1]), '~'.join(vals[0:i + 1])) - for i, _fld in enumerate(flds)] - - -def get_depth(node): - """returns depth of node from top level""" - # TODO: figure out for repeating nodes - if node == '.~::~summary': - return 0 - return 1 + node.split('~::~')[-1].count("~") - - -def level_diff(node1, node2): - return get_depth(node1) - get_depth(node2) - - -def get_name(node, disp_map=None, global_mode=False): - segs, name = node.rsplit('~', 1) - if disp_map: - name = disp_map.get(name, name) - if not global_mode: - return name - elif node == '.~::~summary': - return 'Global' - return name - - -def make_display_name(node, display_mappings): - # TODO: retire get_name, replace with this - try: - seg, segtype = node.split('~::~') - mapped_segs = [display_mappings.get(node, node) for node in segtype.split('~')] - full_name = '~'.join(mapped_segs) - display_name = mapped_segs[-1] - except ValueError: - logger.warning('messed up node: %s', node) - seg, segtype, full_name, display_name = node, node, node, node - return NodeName(seg, segtype, full_name, display_name) - - -@memcacheable('weekly_report_dimensions') -def get_wrd(): - # TODO: Need discussion: Understanding how domainmodel is working - from domainmodel.datameta import Dataset - """ - get weekly report dimensions - """ - try: - return Dataset.getByNameAndStage(name='OppDS').params['general']['weekly_report_dimensions'] - except Exception as e: - logger.warning(e) - return ['frozen_Level_0~frozen_Level_1~frozen_Owner'] - - -def get_drilldowns(node, hier_flds): - try: - return next(x for x in hier_flds if node.startswith(x[0])) - except StopIteration: - return hier_flds[0] - - def get_user_permissions(user, app_section): - # there are several user configurations floating around roles = user.roles['user'] if isinstance(roles, dict): - # preferred: user.roles is a dict with key 'results' which is another dict - # the keys of which are nodes - # TODO: handle this bad case - # bad: user.roles is a dict with key 'results' which is another dict - # the keys of which are node prefixes return roles.get(app_section, {}).keys() elif isinstance(roles, list): - # bad: user.roles is a list of lists, one of those lists starts with 'results' - # and the other items are nodes return next(x for x in roles if x[0] == app_section)[1:] - - -def get_root_dim(user): - """ - get root dimension for a user, fall back to first weekly report dimension if wrongly configured - """ - user_nodes = get_user_permissions(user, 'results') - if '*' not in user_nodes: - top_node = max(user_nodes, key=lambda x: (x.count('~'), len(x))) - return top_node.split('~')[0] - wrds = get_wrd() - return wrds[0].split('~')[0] - - -def get_root_dims(): - return [wrd.split('~')[0] for wrd in get_wrd()] - - -def get_default_root_dim(): - wrds = get_wrd() - return wrds[0].split('~')[0] - - -def move_permissions(node_mapping, debug=False): - # TODO: Need discussion: Understanding how domainmodel is working - from domainmodel.app import User - for u in User.getAll(): - needs_update = False - try: - old_nodes = get_user_permissions(u, 'results') - except Exception as e: - logger.warning('bad user record: %s', u) - old_nodes = [] - for old_node in old_nodes: - if old_node in node_mapping: - new_node = node_mapping[old_node] - try: - # permissions is a list of: - # [app_section, node for read access, can write, can delegate] - perms = u.roles['user']['results'].pop(old_node) - perms[1] = new_node - u.roles['user']['results'][new_node] = perms - except KeyError: - perms_idx, nodes = next((i, x) for i, x in enumerate(u.roles['user']) if x[0] == 'results') - node_idx = next(i for i, x in enumerate(nodes) if x == old_node) - nodes[node_idx] = new_node - u.roles['user'][perms_idx] = nodes - needs_update = True - if needs_update: - if debug: - logger.info('user permissions: %s', u) - u.save() - - -def get_user_id(node, id_mappings, hier_flds=None): - if hier_flds: - drilldowns = get_drilldowns(node, hier_flds) - node = pad_to_rep_level(node, drilldowns) - node_end = node.split('~::~')[1] - user_id = id_mappings.get(node_end) - if user_id: - return user_id, node_end - # HACK: for mongo/aryaka the top level doesnt count.. - node_end = '~'.join(node_end.split('~')[1:]) - user_id = id_mappings.get(node_end, 'none') - return user_id, node_end - - -def hier_level(node): - return len(node.split('~::~')[0].split('~')) diff --git a/tasks/stage.py b/tasks/stage.py deleted file mode 100644 index 3f6497c..0000000 --- a/tasks/stage.py +++ /dev/null @@ -1,263 +0,0 @@ -import copy -import json -import logging -import os -import pprint -import sys -import tempfile -import time -import traceback - -import celery -from aviso.settings import (CNAME, CNAME_DISPLAY_NAME, EMAIL_SENDER, gnana_db, - gnana_storage, sec_context) - -from domainmodel import datameta -from tasks import dataload, gnana_task_support -from utils import GnanaError, diff_rec -from utils.config_utils import config_pattern_expansion, err_format -from utils.file_utils import gitbackup_dataset -from utils.mail_utils import send_mail2 - -logger = logging.getLogger('gnana.%s' % __name__) - -CONFIG_PATHS_FULL_PREPARE = ['params.general.indexed_fields', - 'params.general.dimensions', - 'maps'] - - -def capture_paths(diff, path): - - path_set = set() - - def capture_paths_internal(diff, path, path_set): - - try: - for i in diff: - if i == 'ValDiff': - capture_paths_internal(diff[i], path, path_set) - elif i == 'OnlyInRight' or i == 'OnlyInLeft' or isinstance(diff, tuple): - path_without_trailing_dot = path[:-1] - path_set.add(path_without_trailing_dot) - y = path_without_trailing_dot.split('.') - path = '.'.join(y[:-1]) - path = path + '.' if len(y) > 1 else path - if isinstance(diff, tuple): - break - elif isinstance(i, int): - path_without_trailing_dot = path[:-1] - path_set.add(path_without_trailing_dot) - else: - path += (i+'.') - path = capture_paths_internal(diff[i], path, path_set) - return path - except Exception as _: - exc_type, exc_value, exc_traceback = sys.exc_info() - err_msg = "\n".join(traceback.format_exception(exc_type, exc_value, exc_traceback)) - logger.error('Error while calculating the paths internally .\n%s', err_msg) - - capture_paths_internal(diff, path, path_set) - - return path_set - - -def validate_expression(ds, stage, save_on_error): - attrs = {} - ds.encode(attrs) - attr = copy.deepcopy(attrs) - res = config_pattern_expansion(attr) - eval_errors = res.get('eval_errs', None) - tdetails = sec_context.details - if eval_errors: - if save_on_error == 'True': - if tdetails.get_flag('save_on_error', ds.name+'~'+stage, {}): - tdetails.remove_flag('save_on_error', ds.name+'~'+stage) - tdetails.set_flag('save_on_error', ds.name, eval_errors.keys()) - else: - raise GnanaError(eval_errors) - else: - if tdetails.get_flag('save_on_error', ds.name+'~'+stage, {}): - tdetails.remove_flag('save_on_error', ds.name+'~'+stage) - if tdetails.get_flag('save_on_error', ds.name, {}): - tdetails.remove_flag('save_on_error', ds.name) - ds.decode(attrs) - ds.save() - return eval_errors - - -def backup_dataset(username, ds, inbox_name): - attrs = {} - stackspecific = ds.params['general'].get('stackspecific', False) - ds.encode(attrs) - for ft in attrs['filetypes']: - try: - del ft['last_approved_time'] - except KeyError: - pass - backup = json.dumps(ds.get_as_map()) - fp = tempfile.NamedTemporaryFile(prefix='dataset', delete=False) - fp.write(backup) - fp.close() - gnana_storage.add_adhoc_file(username, - "_".join([inbox_name, str(time.time())]), - fp.name, dataset=ds.name, stackspecific=stackspecific) - os.unlink(fp.name) - - -@celery.task -def purge_stage(user_and_tenant, **args): - return purge_stage_task(user_and_tenant, **args) - - -@gnana_task_support -def purge_stage_task(user_and_tenant, **args): - """ - Purge all the data (from file system, dataset, and results, if any) - for the specified data set and stage. - """ - - dataset = args.get('dataset') - inbox = args.get('stage') - retain_partiton = args.get('retain_partition', False) - ds = datameta.Dataset.getByName(dataset) - stackspecific = ds.params['general'].get('stackspecific', False) - - # 1. Remove the files in the stage - filelist = gnana_storage.filelist(sec_context.name, - inbox=inbox, - dataset=dataset, - stackspecific=stackspecific) - for filedef in filelist: - logger.debug("File def %s", filedef) - if filedef.filetype == 'partitions' and retain_partiton: - continue - try: - gnana_storage.deleteFile(sec_context.name, inbox, dataset, - filedef.filetype, filedef.name, - stackspecific=stackspecific) - except OSError: - continue - - # 2. Remove any stage entry in the dataset - datameta.Dataset.deleteStageData(dataset, inbox) - - # 3. Delete the InboxEntry (Staged Data) Collection - # Delete stage data in inbox collections - InboxClass = datameta.InboxEntryClass(dataset, inbox) - gnana_db.dropCollection(InboxClass.getCollectionName()) - - # 4. Delete the Staged Results collections - result_prefix = ".".join([user_and_tenant[1], dataset, "_results"]) - for collection_name in gnana_db.collection_names(prefix=result_prefix): - if collection_name.endswith(".%s" % inbox): - gnana_db.dropCollection(collection_name) - - # 5. Delete the staged rejections related collections - result_prefix = ".".join([user_and_tenant[1], dataset, "_rej"]) - for collection_name in gnana_db.collection_names(prefix=result_prefix): - if collection_name.endswith(".%s" % inbox): - gnana_db.dropCollection(collection_name) - - -def ds_stage_config(user_and_tenant, inbox, ds, uip_merge=False, **args): - git_errors = None - save_on_error = args.get('save_on_error', False) - dataset = args.get('dataset') - auto_purge = args.get('auto_purge') - stage = args.get('stage') - dstage = datameta.Dataset.getByNameAndStage(dataset, inbox, full_config=False) - mail_changes = diff_rec(ds.get_as_map(), dstage.get_as_map(), {'stages'}) - try: - paths_changed = list(capture_paths(mail_changes, '')) - logger.info('Paths changed %s' % paths_changed) - mark_full_prepare = False - for path in paths_changed: - for prepare_path in CONFIG_PATHS_FULL_PREPARE: - if path.startswith(prepare_path): - logger.info('As path %s changed. Marking for full_prepare ..' % path) - mark_full_prepare = True - break - if mark_full_prepare: - break - - if mark_full_prepare: - logger.info("Saving the full_prepare flag") - tdetails = sec_context.details - prep_status = tdetails.get_flag('prepare_status', ds.name, {}) - prep_status['full_prepare'] = mark_full_prepare - tdetails.set_flag('prepare_status', ds.name, prep_status) - except Exception as _: - exc_type, exc_value, exc_traceback = sys.exc_info() - err_msg = "\n".join(traceback.format_exception(exc_type, exc_value, exc_traceback)) - logger.error('Error while calculating the paths.\n%s', err_msg) - - dataset_changes = ds.stages.get(inbox, {}) - creator = dataset_changes.get('creator', args['approver']) - comment = dataset_changes.get('comment', "Modified " + dataset + " config") - gitcomment = (comment + - "-Creator:" + creator + - " -Approver:" + args['approver']) - file_name = dataset + ".json" - - # sending mail - if dataset_changes: - eval_errs = validate_expression(ds, stage, save_on_error) - status = gitbackup_dataset(file_name, dstage.get_as_map(), gitcomment) - git_errors = status[1] - cname = CNAME_DISPLAY_NAME if CNAME_DISPLAY_NAME else CNAME - if status[0]: - format_changes = pprint.pformat(mail_changes, indent=2) - tdetails = sec_context.details - send_mail2('dataset_changes.txt', - 'Aviso ', - tdetails.get_config('receivers', 'dataset_changes', ['gnackers@aviso.com']), - reply_to='Data Science Team ', - dataset=dataset, tenantname=sec_context.name, - comment=comment, changes=format_changes + err_format(eval_errs), - creator=creator, approver=args['approver'], cName=cname, creator_name=creator.rsplit('@')[0]) - logger.info('Sending dataset changes mail for dataset:' - ' %s, tenant: %s, user: %s' % - (dataset, sec_context.name, sec_context.user_name)) - - logger.info("Approving the stage %s", inbox) - - # 0. Save the encoded dataset into S3 storage before merging - backup_dataset(user_and_tenant[1], ds, inbox) - - # 1. Merge the dataset and save back - ds.merge_stage(inbox, delete_stage=True) - ds.save() - if uip_merge: - stackspecific = ds.params['general'].get('stackspecific', False) - pd = ds.get_partition_data(stage=inbox) - - # Add the dataset class for further processing - ds.DatasetClass = datameta.DatasetClass(ds) - # 2. Merge the inbox entries - # Calling this method will take care of distributing the work to - # multiple workers - dataload.merge_inbox_entries_to_uip(user_and_tenant, dataset, inbox, - auto_purge=False) - # 3. Move the files in S3 from the current stage into _data - for f in gnana_storage.filelist(user_and_tenant[1], inbox=inbox, dataset=dataset, stackspecific=stackspecific): - if f.filetype == 'partitions': - continue - logger.info("Moving file %s in area %s", f.fullpath, f.filetype) - gnana_storage.move_file(user_and_tenant[1], "_data", dataset, f.filetype, - f.name, - f.fullpath, stackspecific=stackspecific) - # 4. Auto purge - if auto_purge: - purge_stage(user_and_tenant, retain_partition=True, **args) - if pd: - logger.info("Saving the partition data back") - pd.save() - ds.stages['inbox'] = {} - ds.save() - - # 5. Update the data_update flag - logger.info("Saving the data_update flag") - tdetails = sec_context.details - tdetails.set_flag('data_update', dataset, time.time()) - - return git_errors diff --git a/tasks/sync_drilldown.py b/tasks/sync_drilldown.py index aad33fd..1788c77 100644 --- a/tasks/sync_drilldown.py +++ b/tasks/sync_drilldown.py @@ -4,10 +4,10 @@ from pymongo import UpdateOne from config import HierConfig -from config.fm_config import FMConfig -from config.hier_config import DRILLDOWN_BUILDERS -from config.periods_config import PeriodsConfig -from infra import DRILLDOWN_COLL +from config import FMConfig +from config import DRILLDOWN_BUILDERS +from config import PeriodsConfig +from infra.constants import DRILLDOWN_COLL from infra.read import (fetch_descendant_ids, fetch_hidden_nodes, fetch_node_to_parent_mapping_and_labels, fetch_top_level_nodes, get_as_of_dates, @@ -442,13 +442,13 @@ def persist(self): service=self.service) try: - if (len(self.deleted_nodes) or len(self.created_nodes)\ + if (len(self.deleted_nodes) or len(self.created_nodes) or len(self.resurrected_nodes) or len(self.moved_nodes)): fm_config = FMConfig() if fm_config.snapshot_feature_enabled: nodes_having_changes = self.deleted_nodes|set(self.created_nodes.keys())|self.resurrected_nodes|set(self.moved_nodes.keys()) - from fm_service.forecast_schedule import FMScheduleClass - fm_schedule_class = FMScheduleClass(self.period, list(nodes_having_changes)) + from fm_service.forecast_schedule import FMSchedule + fm_schedule_class = FMSchedule(self.period, list(nodes_having_changes)) fm_schedule_class.update_fm_schedule() except Exception as e: logger.exception(e) @@ -563,8 +563,8 @@ def persist_versioned(self): fm_config = FMConfig() nodes_having_changes = self.deleted_nodes_versioned[prd]|set(self.created_nodes_versioned[prd].keys())|self.resurrected_nodes_versioned[prd]|set(self.moved_nodes_versioned[prd].keys()) if fm_config.snapshot_feature_enabled: - from fm_service.forecast_schedule import FMScheduleClass - fm_schedule_class = FMScheduleClass(self.period, nodes_having_changes) + from fm_service.forecast_schedule import FMSchedule + fm_schedule_class = FMSchedule(self.period, nodes_having_changes) fm_schedule_class.update_fm_schedule() except Exception as e: logger.exception(e) diff --git a/tasks/targetspecTasks.py b/tasks/targetspecTasks.py deleted file mode 100644 index ae8c379..0000000 --- a/tasks/targetspecTasks.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging -import pprint - -from aviso.settings import sec_context, gnana_db, CNAME, CNAME_DISPLAY_NAME - -from domainmodel.datameta import TargetSpec -from tasks.stage import validate_expression -from utils import diff_rec -from utils.config_utils import err_format -from utils.file_utils import gitbackup_dataset -from utils.mail_utils import send_mail2 - -logger = logging.getLogger('gnana.%s' % __name__) - - -def verifyAndValidate(payload, target_name, save_on_error, old_target_spec=None, action=None): - target_spec = TargetSpec() - target_spec.name = target_name - target_spec.report_spec = payload.get('report_spec', {}) - target_spec.drilldowns = payload.get('drilldowns', []) - target_spec.module = payload.get('module', "") - target_spec.models = payload.get('models', {}) - target_spec.keys = payload.get('keys', {}) - target_spec.task_v2 = payload.get('task_v2', False) - eval_errors = validate_expression(target_spec, save_on_error, old_target_spec, action) - return eval_errors - - -def get_attr_or_item(c, i, default): - if isinstance(c, list): - try: - return c[i] - except IndexError: - return default - if isinstance(c, dict): - try: - return c[i] - except KeyError: - return default - return getattr(c, i, default) - - -def unset_attr_or_remove_item(c, i): - if isinstance(c, dict): - c.pop(i, None) - else: - delattr(c, i) - - -def set_attr_or_item(c, i, val): - if isinstance(c, list) or isinstance(c, dict): - c[i] = val - else: - setattr(c, i, val) - - -def target_spec_tasks(username, target_name, payload, save_on_error, action, old_target_spec=None, module_path=None, - commit_message=None): - eval_errors = {} - if action == 'create': - eval_errors = verifyAndValidate(payload, target_name, save_on_error, old_target_spec, action) - target_spec = TargetSpec.getByFieldValue('name', target_name) - if action == 'update' or action == 'delete_module': - module_segments = module_path.split('.') - container = target_spec - for x in module_segments[0:-1]: - next_container = get_attr_or_item(container, x, None) - if not next_container: - set_attr_or_item(container, x, {}) - container = get_attr_or_item(container, x, None) - else: - container = next_container - if action == 'update': - set_attr_or_item(container, module_segments[-1], payload) - else: - unset_attr_or_remove_item(container, module_segments[-1]) - eval_errors = validate_expression(target_spec, save_on_error) - elif action == 'purge_spec': - tdetails = sec_context.details - if tdetails.get_flag('save_on_error', target_name, {}): - tdetails.remove_flag('save_on_error', target_name) - tdetails.save() - target_spec.remove(target_spec.id) - gnana_db.dropCollectionsInNamespace("%s.combined_results.%s" % (sec_context.name, target_name)) - return {'success': True} - elif action == 'recreate': - eval_errors = verifyAndValidate(payload, target_name, save_on_error, old_target_spec, action) - target_spec = TargetSpec.getByFieldValue('name', target_name) - else: - # Create is already handled - pass - - attrs = {} - target_spec.encode(attrs) - # update into repository - target = target_name - new_target_spec = TargetSpec.getByFieldValue('name', target_name) - comment = "Modified " + target + " config" - gitcomment = (commit_message if commit_message else comment) - gitcomment += (gitcomment + ' -user: ' + username) - logger.info("Git comment : %s" % gitcomment) - file_name = "targetspec_" + target + ".json" - mail_changes = diff_rec(old_target_spec, new_target_spec.__dict__, {'id'}) - if mail_changes: - status = gitbackup_dataset(file_name, new_target_spec.__dict__, gitcomment) - if status[0]: - format_changes = pprint.pformat(mail_changes, indent=2) + err_format(eval_errors) - tdetails = sec_context.details - cname = CNAME_DISPLAY_NAME if CNAME_DISPLAY_NAME else CNAME - send_mail2('targetspec.txt', - 'Aviso ', - tdetails.get_config('receivers', 'dataset_changes', ['gnackers@aviso.com']), - reply_to='Data Science Team ', - target_name=target_name, tenantname=sec_context.name, - comment=comment, changes=format_changes, - user_name=sec_context.user_name, user=username, cname=cname) - return attrs diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..1398cf4 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,13 @@ +# MongoDB Test Setup + +## 🔧 Prerequisites + +- Docker & Docker Compose installed +- `pytest` and `pymongo` installed in your virtual environment + +## 🚀 Setup Instructions + +1. **Start MongoDB Container** + +```bash +docker-compose up -d diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..efc6334 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,32 @@ +import pytest +from pymongo import MongoClient +import os +from dotenv import load_dotenv + +load_dotenv() + +MONGO_URI = os.getenv("MONGO_URI") +TEST_DB_NAME = os.getenv("TEST_DB_NAME") + + +@pytest.fixture(scope="session") +def mongo_client(): + client = MongoClient(os.getenv("MONGO_URI")) + assert client.admin.command("ping")["ok"] != 0.0 + yield client + client.drop_database(os.getenv("TEST_DB_NAME")) + client.close() + + +@pytest.fixture(scope="function") +def test_db(mongo_client): + db = mongo_client[TEST_DB_NAME] + for col in db.list_collection_names(): + db[col].delete_many({}) + return db + + +def test_db_connection(): + client = MongoClient(MONGO_URI) + client.admin.command('ping') + print(f"\n{client.list_database_names()}\n") diff --git a/tests/test_basics.py b/tests/test_basics.py index 7b20b61..a01a026 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -1,3 +1,57 @@ +from config import write_hierarchy_to_gbm + def test_dummy(): assert 1+1 == 2 + +def test_insert_and_find_document(test_db): + test_collection = test_db["users"] + test_doc = {"name": "Faizan Mazhar", "email": "faizan@example.com"} + + # Insert document + insert_result = test_collection.insert_one(test_doc) + assert insert_result.acknowledged is True + + # Retrieve document + fetched_doc = test_collection.find_one({"_id": insert_result.inserted_id}) + assert fetched_doc is not None + assert fetched_doc["name"] == "Faizan Mazhar" + assert fetched_doc["email"] == "faizan@example.com" + + # Delete document + delete_result = test_collection.delete_one({"_id": insert_result.inserted_id}) + assert delete_result.deleted_count == 1 + + fetched_doc = test_collection.find_one({"_id": insert_result.inserted_id}) + assert fetched_doc is None + + + +from unittest.mock import MagicMock, patch + +@patch('utils.misc_utils.try_index') +@patch('infra.create_collection_checksum') +@patch('settings.sec_context') +def test_write_hierarchy_to_gbm_mismatch_triggers_sync(mock_sec_ctx, mock_checksum, mock_try_index): + # Setup + mock_config = MagicMock(debug=True) + + # Mock DB + mock_db = MagicMock() + mock_coll = MagicMock() + mock_coll.find.return_value = [{'a': 1}] + mock_db.__getitem__.return_value = mock_coll + + # Checksum mismatch + mock_sec_ctx.get_microservice_config.return_value = True + mock_sec_ctx.gbm.api.side_effect = ['remote_checksum', None] + mock_sec_ctx.tenant_db = mock_db + + mock_checksum.return_value = 'local_checksum' + mock_try_index.return_value = {'a': 1} + + result = write_hierarchy_to_gbm(config=mock_config) + + assert result is True + mock_sec_ctx.gbm.api.assert_any_call('/gbm/hier_metadata', None) + mock_sec_ctx.gbm.api.assert_any_call('/gbm/hier_sync', [{'a': 1}]) diff --git a/utils/cache_utils.py b/utils/cache_utils.py index 2db97a8..ca5bc99 100644 --- a/utils/cache_utils.py +++ b/utils/cache_utils.py @@ -268,203 +268,6 @@ def wrapped(*args, **kwargs): return decorator -def clears_memcache(cache_name, *cacheargs, **cache_kwargs): - """ - decorator to add to a function or class to clear cache - clears results in MEMCACHED - - MEMCACHED considerations: - - memcached is a {key: value} store, there is no nesting like {key: {inner_key: value}} - if you want to clear multiple keys at once you can use an arg_maker - arg_maker: - make more args out of the cache_args passed to clears_memcache - pass an arg_maker class which has a static 'make' method (ex MonthMaker) - - OR if you are tracking all the keys you are storing, with track_key in memcacheable - you can pass clear_all. this will wipe out everything in the cache for cache_name - - Arguments - --------- - cache_name: str, required - name of cache - - cacheargs: tuple, optional, (arg name, arg position) - name that maps to argument in decorated function, used to make key for storing in cache - and the position of that argument in function call - - arg_maker: class, optional - class providing a make method - - clear_all: bool, optional - if True, will clear all values inside cache for cache_name, default False - """ - - def decorator(func): - @wraps(func) - def wrapped(*args, **kwargs): - logger.info('clearing memcache for: %s', cache_name) - val = func(*args, **kwargs) - arg_maker = cache_kwargs.get('arg_maker') - arg_func = cache_kwargs.get('arg_func') - clear_all = cache_kwargs.get('clear_all') - if clear_all: - track_key = make_cache_key(cache_name, [], {}) - all_keys = get_from_cache(track_key, sec_context.details.name) - clear_keys = [key.split(',') for key in all_keys.split('||')] if all_keys else [] - else: - key_args = get_cache_args(cacheargs, *args, **kwargs) - if arg_func: - key_args = [arg_func(arg) if arg else arg for arg in key_args] - clear_keys = [key_args] if not arg_maker else [key_args + [arg] for arg in arg_maker.make(*key_args)] - mem_keys = [make_cache_key(cache_name, clear_key, {}) for clear_key in clear_keys] - remove_keys(mem_keys, sec_context.details.name) - if not clear_all: - for clear_key in clear_keys: - incr_cache_version('memcache', [cache_name] + clear_key) - else: - clear_cache_versions('memcache', cache_name) - return val - - return wrapped - - return decorator - - -def pycacheable(cache_name, *cacheargs, **cache_kwargs): - """ - decorator to add to a function or class to cache results - caches result of call or instantiation in PYCACHE - - PYCACHE considerations: - - PYCACHE is just a globally scoped python dictionary - since its a dictionary, you can nest keys {key: {inner_key: value}} - the keys for storing the value is constructed using the cache_name, and the function/class value of each cache_arg - example: - >>> @pycacheable('cachey', ('val1', 0), ('val2', 1)) - >>> def add_vals(val1, val2): - >>> return val1 + val2 - >>> add_vals(10, 5) - pycache will now look like {'cachey': {10: {5: 15}}} - - - PYCACHE lives on the backend server - since some of our environments have multiple boxes/thread, we have to do some work to keep them in sync - ex if the cache is invalidated on server A, it better get invalidated on server B too - pycache transparently handles this with cache versioning using the tenant flags as a store of the version - - - the hit rate for pycache will be lower than memcached - because of the multiple boxes/threads the hit rate between sessions will be around 25% - within sessions, if a session val is passed or memcached is on, the hit rate should be 100% - - - there is some overhead in checking the version number from the tenant flags - so it is not optimal to use this for a quick calculation that you make 100s of times - - Arguments - --------- - cache_name: str, required - name of cache - - cacheargs: tuple, optional, (arg name, arg position) - name that maps to argument in decorated function, used to make key for storing in cache - and the position of that argument in function call - - cache_kwargs: optional - add a session variable to ensure cache hits in same session - protects against not having memcached turn on in dev environments - """ - - def decorator(func): - @wraps(func) - def wrapped(*args, **kwargs): - if kwargs.get('skip_cache'): - return func(*args, **kwargs) - ucache_name = str(cache_name) - session = cache_kwargs.get('session', False) - cache_args = get_cache_args(cacheargs, *args, **kwargs) - cache_keys = [sec_context.name, ucache_name] + cache_args - latest_version = get_cache_version('pycache', [ucache_name] + cache_args) - cached_val, cached_version, cached_session = get_nested(_PYCACHE, cache_keys, (None, None, None)) - # regenerate val if: - # no value in cache - # no tenant flags found AND session doesnt match cached session - # version in memcached doesnt match version in cache - if cached_val is None or ( - latest_version is False and session != cached_session) or cached_version != latest_version: - if not latest_version: - latest_version = 1 - incr_cache_version('pycache', [ucache_name] + cache_args) - val = func(*args, **kwargs) - set_nested(_PYCACHE, cache_keys, (val, latest_version, session)) - return get_nested(_PYCACHE, cache_keys)[0] - - return wrapped - - return decorator - - -def clears_pycache(cache_name, *cacheargs, **cache_kwargs): - """ - decorator to remove items from cache - - PYCACHE considerations: - - PYCACHE is a dictionary, so instead of clearing value at a time, you can clear many levels at once - example: - >>> @pycacheable('cachey', ('Q', 0), ('P', 1)) - >>> def period_is_future(Q, P): - >>> return True - >>> period_is_future('2019Q1', '201802') - >>> period_is_future('2019Q1', '201803') - pycache will now look like {'cachey': {'2019Q1': {'201802': True, '201803': True}} - once it becomes 201803, 201802 is no longer true and so you could invalidate like - >>> @clears_pycache('cachey', ('Q', 0), ('P', 1)) - >>> def period_is_past(Q, P): - >>> return - >>> period_is_past('2019Q1', '201802') - pycache will now be {'cachey': {'2019Q1': {'201803': True}} - but if its now 2019Q2, you could also invalidate all of Q1 in one go - >>> @clears_pycache('cachey', ('Q', 0)) - >>> def quarter_is_past(Q): - >>> return - >>> quarter_is_past('2019Q1') - pycache will now be {'cachey': {'2019Q1': {}} - - - pycache versioning is saved in the tenant flags - when a value is first saved, its version is set to 0 - when the cache gets cleared, the version number gets incremented in the flag - when fetching from cache, we check that the cache version is not less than the latest version from the flag - - Arguments - --------- - cache_name: str, required - name of cache - - cacheargs: tuple, optional, (arg name, arg position) - name that maps to argument in decorated function, used to make key for storing in cache - and the position of that argument in function call - """ - - def decorator(func): - @wraps(func) - def wrapped(*args, **kwargs): - - ucache_name = str(cache_name) - ret_val = func(*args, **kwargs) - cache_args = get_cache_args(cacheargs, *args, **kwargs) - if cache_kwargs.get('clear_all'): - # TODO: THIS clear all - pass - arg_func = cache_kwargs.get('arg_func') - if arg_func: - cache_args = [arg_func(arg) if arg else arg for arg in cache_args] - cache_keys = [sec_context.name, ucache_name] + cache_args - pop_nested(_PYCACHE, cache_keys) - incr_cache_version('pycache', [ucache_name] + cache_args) - return ret_val - - return wrapped - - return decorator - - def get_cache_args(cacheargs, *args, **kwargs): return [str(kwargs.get(k, try_index(args, i))) for k, i in cacheargs] @@ -497,47 +300,3 @@ def incr_cache_version(cache_type, cache_keys): if isinstance(version, int): set_nested(version, path, val + 1) sec_context.details.set_flag(cache_type, 'versions', cache_flag) - - -def clear_cache_versions(cache_type, cache_name): - cache_flag = sec_context.details.get_flag(cache_type, 'versions', {}) - cache_flag.pop(cache_name, None) - sec_context.details.set_flag(cache_type, 'versions', cache_flag) - - -def list_all_memcached_keys(): - keys = defaultdict(list) - for cache_name, cache in cache_con.cache.__dict__.iteritems(): - if not cache: - keys[cache_name] = None - continue - for cache_server in cache.servers: - server, port = cache_server.address - keys[cache_name].extend(get_all_memcached_keys(server, port)) - - return keys - - -def get_all_memcached_keys(host='127.0.0.1', port=11211): - if host == 'localserver': - host = '127.0.0.1' - t = telnetlib.Telnet(host, port) - t.write('stats items STAT items:0:number 0 END\n') - items = t.read_until('END').split('\r\n') - keys = set() - for item in items: - parts = item.split(':') - if not len(parts) >= 3: - continue - slab = parts[1] - t.write( - 'stats cachedump {} 200000 ITEM views.decorators.cache.cache_header..cc7d9 [6 b; 1256056128 s] END\n'.format( - slab)) - cachelines = t.read_until('END').split('\r\n') - for line in cachelines: - parts = line.split(' ') - if not len(parts) >= 3: - continue - keys.add(parts[1]) - t.close() - return keys \ No newline at end of file diff --git a/utils/collab_connector_utils.py b/utils/collab_connector_utils.py deleted file mode 100644 index e2f16ee..0000000 --- a/utils/collab_connector_utils.py +++ /dev/null @@ -1,31 +0,0 @@ -import pymongo - -from domainmodel.app import User - - -def get_dict_of_all_fm_users(db_to_use=None, consider_admin=True): - db_to_use = db_to_use if db_to_use is not None else User.get_db() - all_users_list = list(db_to_use.findDocuments( - User.getCollectionName(), {"object.is_disabled": False}, sort=[('object.username', pymongo.ASCENDING)])) - all_users_dict = {} - for u in all_users_list: - data = u['object'] - roles = data['roles'].get("user", []) - if not consider_admin: - admin_user = data['roles'].get("administrator", None) - if admin_user: - continue - consider = True - for role in roles: - if role[0] == 'results' and role[1] == '*': - consider = False - break - if consider: - all_users_dict[data['username']] = { - "name": data["name"], - 'email': data['email'], - 'roles': data['roles'], - 'userId': data.get('user_id'), - 'user_role': data.get('user_role') - } - return all_users_dict diff --git a/utils/common.py b/utils/common.py index d568a9a..8d27a2d 100644 --- a/utils/common.py +++ b/utils/common.py @@ -1,10 +1,3 @@ -import json - -import netaddr -from aviso.framework.views import GnanaView -from django.http import HttpResponse - - class weekday: __slots__ = ["weekday", "n"] @@ -56,38 +49,3 @@ def __get__(self, instance, type=None): return self res = instance.__dict__[self.func.__name__] = self.func(instance) return res - -def ip_match(ip, valid_ip_list): - ip = ip.strip() - ip = netaddr.IPAddress(ip) - for n in valid_ip_list: - if '/' in n: - nw = netaddr.IPNetwork(n) - else: - nw = netaddr.IPNetwork(n+'/32') - if ip in nw: - return n - return None - -class MicroAppView(GnanaView): - """ - base class for all micro app views - """ - validators = [] - - @cached_property - def config(self): - raise NotImplementedError - - def dispatch(self, request, *args, **kwargs): - if request.method == 'OPTIONS': - # If the request method is OPTIONS, return an empty response - return HttpResponse() - self.debug = request.GET.get('debug') - if request.GET.get('help'): - return HttpResponse(json.dumps({'help': self.__doc__}), content_type='application/json') - for validator in self.validators: - response = validator(request, self.config) - if isinstance(response, HttpResponse): - return response - return super(MicroAppView, self).dispatch(request, *args, **kwargs) diff --git a/utils/config_utils.py b/utils/config_utils.py index 7e9aa35..1f1d9dc 100644 --- a/utils/config_utils.py +++ b/utils/config_utils.py @@ -2,8 +2,6 @@ from aviso.settings import sec_context import copy -import bson -from utils.constants import PRIVATE_MOXTRA_URL, PUBLIC_MOXTRA_URL logger = logging.getLogger('gnana.%s' % __name__) @@ -87,83 +85,3 @@ def checkpattern(attrs, i, path, i_type, in_and_ex): return None get_keys(attrs, attrs, 'attrs', '') return {'attrs': attrs, 'eval_errs': eval_errs} - - -def err_format(eval_errors): - if not eval_errors: - return '\n' - errstr = '\n \nThese are the fields bypassed by the evaluation due to save_on_error . \n ' - for key in eval_errors.keys(): - errstr = errstr+'destination = '+key+'\n' - errstr = errstr+'expression = '+str(eval_errors[key][0])+'\n' - errstr = errstr+'message_list = '+str(eval_errors[key][1])+'\n\n' - return errstr - - -def get_max_allowed_batch_size(list_of_records, model): - model_config = sec_context.details.get_config('compressed_model', model, dict()) - if not model_config: - b = bson.BSON() - size = len(b.encode(dict(list_of_records=list_of_records))) - num = len(list_of_records) - max_size = 16 * 1024 * 1024 - if size < max_size: - batch = num - else: - safefy_limit = 0.9 - single = (size * 1.0) / num - batch = int((max_size / single) * safefy_limit) - model_config = dict(max_allowed = batch) - try: - sec_context.details.set_config('compressed_model', model, model_config) - except Exception as e: - logger.warning("warn: failed to set model %s config %s", model, e) - try: - sec_context.details.set_config('compressed_model', "migrated", True) - except Exception as e: - logger.warning("warn: failed to set Migrated config %s", e) - return model_config.get('max_allowed', 10000) - - -def get_moxtra_url(guest=None, domain=None): - is_moxtra_public = False - try: - is_moxtra_public = sec_context.details.get_config('collaboration', - 'is_public', - False) - except Exception as mex: - if guest and domain and ".aviso.com" in domain: - try: - tenant_splt = domain.split(".aviso.com")[0].split(".") - if len(tenant_splt) == 2: - tenant_name = tenant_splt[0].replace("-", "_") - from domainmodel.tenant import Tenant - tenants_list = Tenant.getDistinctValues("name") - for t_name in tenants_list: - if t_name.startswith(tenant_name): - sec_context.set_context("local_cache", t_name, t_name) - is_moxtra_public = sec_context.details.get_config('collaboration', 'is_public', False) - break - except: - pass - config_private_url = None - config_public_url = None - if not guest: - try: - from config.fm_config import FMConfig - fm_config = FMConfig() - config_private_url = fm_config.get_moxtra_private_url - config_public_url = fm_config.get_moxtra_public_url - except: - pass - if is_moxtra_public: - if config_public_url: - moxtra_url = config_public_url - else: - moxtra_url = PUBLIC_MOXTRA_URL - else: - if config_private_url: - moxtra_url = config_private_url - else: - moxtra_url = PRIVATE_MOXTRA_URL - return moxtra_url diff --git a/utils/constants.py b/utils/constants.py deleted file mode 100644 index 7211765..0000000 --- a/utils/constants.py +++ /dev/null @@ -1,351 +0,0 @@ -from aviso.settings import CNAME - - -ALGOLIA_INDEX_NAME_FOR_DEAL_RECORDS = 'deal_records' -APPLICATION_JSON = 'application/json' -MEETID_NOT_ALLOWED_CHARS = r'[^:_a-zA-Z0-9]' -MEET_FILE_NAME_NOT_ALLOWED_CHARS = r'[^:.-_a-zA-Z0-9@!#$%&*()<>?/|}{~]' -EXTERNAL = 'external' -FILE_PATH_FORMAT = '{tenant}/{meeting_type}' -MEETINGS_BUCKET_NAME = ('collaboration-meetings-prod' - if CNAME == 'app' else - 'collaboration-meetings-qa') -PRIVATE_MOXTRA_URL = "https://avisomeet.aviso.com/web/websdk/dist/mepsdk.js?ver=7.16.6" -PUBLIC_MOXTRA_URL = "https://aviso-dev.moxtra.com/web/websdk/dist/mepsdk.js?ver=7.16.6" - -VERSION_CONFIGURABLE_PRIVATE_MOXTRA_URL = "https://avisomeet.aviso.com/web/websdk/dist/mepsdk.js?ver={sdk_version}" -VERSION_CONFIGURABLE_PUBLIC_MOXTRA_URL = "https://aviso-dev.moxtra.com/web/websdk/dist/mepsdk.js?ver={sdk_version}" - -VALID = '_' -doc360URL = 'https://identity.document360.io/jwt/generateCode' -doc360AuthURL = 'https://dochelp.aviso.com/jwt/authorize?code=' -doc360username = '46e44d8b-8e4d-4d53-8bf6-74728c66e765' -doc360secrect = 'oEYfwwvRppeIE7tuyoCAolGxp61h1h6U0JML4HkrC80' - -FERNET_SECRET_KEY = "FeYaD7iCSS7z2PC8lvECgiyvjFS40HHtk4ebH6JxBHU=" - -FILTER_TABLE_SEARCH_SCHEMA = [ - { - "index": "Calls", - "filter_fields": - { - 'order': ['participants', - 'words', - 'trackers', - 'date', - 'call_duration', - 'deal_name', - 'account_name', - 'web_conference', - 'buyer_interest_score', - 'talk_ratio', - 'interactivity_ratio', - 'longest_monologue'], - "schema": [ - { - "domain": [ - - ], - "key": "participants.name", - "label": "Call Participants", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "words", - "label": "Themes", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - ], - "key": "trackers", - "label": "Keywords", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "date", - "label": "Date", - "type": "date", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "call_duration", - "label": "Call Duration", - "type": "slider", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "deal_name", - "label": "Deal Name", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "account_name", - "label": "Account Name", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "web_conference", - "label": "Web Conference", - "type": "object", - "sub_filters": None, - "isActive" : False, - "isDefault" : False - }, - { - "domain": [ - - ], - "key": "buyer_interest_score", - "label": "Buyer iInterest Score", - "type": "slider", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "participants.talk_ratio", - "label": "Talk ratio", - "type": "slider", - "sub_filters": None, - "isActive" : False, - "isDefault" : False - }, - { - "domain": [ - - ], - "key": "participants.interactivity_ratio", - "label": "Interactivity ratio", - "type": "slider", - "sub_filters": None, - "isActive" : False, - "isDefault" : False - }, - { - "domain": [ - - ], - "key": "participants.longest_monologue", - "label": "Longest Monologue", - "type": "slider", - "sub_filters": None, - "isActive" : False, - "isDefault" : False - } - ] - } - }, - { - "index": "Deals", - "filter_fields": - { - 'order': ['close_date', #date - 'account', #object - 'amount', #slider - 'owner_name', - 'stage', - 'engagement_grade', - 'forecast_category', - 'region', - 'vertical', - 'win_score'], #slider - 'schema': [ - { - "domain": [ - - ], - "key": "close_date", - "label": "Close Date", - "type": "date", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "account", - "label": "Account", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "amount", - "label": "Amount", - "type": "slider", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "owner_name", - "label": "Owner Name", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "stage", - "label": "Stage", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "engagement_grade", - "label": "Engagement Grade", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "forecast_category", - "label": "Forecast Category", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "region", - "label": "Region", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : False - }, - { - "domain": [ - - ], - "key": "vertical", - "label": "Vertical", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : False - }, - { - "domain": [ - - ], - "key": "win_score", - "label": "Win Score", - "type": "slider", - "sub_filters": None, - "isActive" : True, - "isDefault" : False - } - ] - } - }, - { - "index": "Accounts", - "filter_fields": - { - 'order': ['owner_name', - 'vertical', - 'engagement_grade'], - 'schema': [ - { - "domain": [ - - ], - "key": "owner_name", - "label": "Account Owner", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "vertical", - "label": "Vertical", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - }, - { - "domain": [ - - ], - "key": "engagement_grade", - "label": "Engangement grade", - "type": "object", - "sub_filters": None, - "isActive" : True, - "isDefault" : True - } - ] - } - } -] diff --git a/utils/crypto_utils.py b/utils/crypto_utils.py index f88fd83..8cc5389 100644 --- a/utils/crypto_utils.py +++ b/utils/crypto_utils.py @@ -1,6 +1,5 @@ from Crypto.Cipher import AES import base64 -import hashlib from aviso.settings import sec_context from bson import BSON @@ -8,78 +7,10 @@ def encrypt(key, message): cipher = AES.new(key, mode=AES.MODE_CFB, IV="0123456789012345") return base64.b64encode(cipher.encrypt(message)) -def encrypt_stream(key, message): - cipher = AES.new(key, mode=AES.MODE_CFB, IV="0123456789012345") - for x in message: - yield base64.b64encode(cipher.encrypt(x)) - def decrypt(key, message): cipher = AES.new(key, mode=AES.MODE_CFB, IV="0123456789012345") return cipher.decrypt(base64.b64decode(message)) -def decrypt_stream(key, message): - cipher = AES.new(key, mode=AES.MODE_CFB, IV="0123456789012345") - for x in message: - yield cipher.decrypt(base64.b64decode(x)) - - -class Wallet: - def __init__(self, wallet): - - # Find the master key - try: - f = open('/etc/master_key','r') - self.master_key = f.readline() - f.close() - except: - self.master_key = 'h6nTho9Mi4i7e77BX7miJ8DFrqiOxfzf' - - self.wallet = wallet - - # Load keys - keys = {} - try: - f = open(wallet,'r') - for kline in f: - (n,k)=kline.split('=',1) - if k: - keys[n]=k - f.close() - except IOError as e: - if not e.errno == 2: - pass - self.keys = keys - - def add(self, key, name): - self.keys[name]= encrypt(self.master_key, key) - - def __getitem__(self, name): - return decrypt(self.master_key, self.keys[name]) - - def save(self): - # Save all the keys - f = open(self.wallet,'w') - for n,k in self.keys.items(): - f.write("=".join([n,k])) - f.write('\n') - f.close() - -def getfilemd5(file_path, block_size=2**24): - md5 = hashlib.md5() - with open (file_path,'r') as f: - while True: - data = f.read(block_size) - if not data: - break - md5.update(data) - return md5.hexdigest() - -def setpath(dic, atr, val): - if len(atr) == 1: - dic[atr[0]] = val - else: - setpath(dic.setdefault(atr[0], {}), atr[1:], val) - def extract_index(index_list, attributes, query_fields=[]): new_obj = {} @@ -129,12 +60,3 @@ def encrypt_record(index, record, query_fields=[], postgres=False, cls=None): result['object'] = extract_index(index, record['object'], query_fields) result['_check'] = new_check return result - - -def encrypt_dict_leaves(d): - if isinstance(d, dict): - return dict((k, encrypt_dict_leaves(v)) for k,v in d.iteritems()) - elif isinstance(d, (list,set,tuple)): - return [encrypt_dict_leaves(v1) for v1 in d] - else: - return sec_context.encrypt(d) diff --git a/utils/date_utils.py b/utils/date_utils.py index c487344..af75afa 100644 --- a/utils/date_utils.py +++ b/utils/date_utils.py @@ -7,7 +7,6 @@ import pytz from aviso.settings import sec_context, CNAME -from domainmodel.csv_data import CSVDataClass from .cache_utils import memcached, memcacheable from .relativedelta import relativedelta @@ -585,20 +584,6 @@ def next_period_by_epoch(epoch_time, period_type='Q', skip=1, count=1): def prev_period_by_epoch(epoch_time, period_type='Q', skip=1, count=1): return period_details(epoch2datetime(epoch_time), period_type, delta=-skip, count=count) - -def is_same_day(first, second): - """ - Accepts two dates in epoch and tells whether they fall on the same day. - """ - if first is None or second is None: - return False - first_dt = first.as_datetime() - second_dt = second.as_datetime() - if (first_dt.year, first_dt.month, first_dt.day) == (second_dt.year, second_dt.month, second_dt.day): - return True - else: - return False - def get_eom(ep): """ get end of month @@ -623,24 +608,6 @@ def get_eod(ep): eod = tzinfo.localize(eod, is_dst=tzinfo.dst(eod)) return epoch(eod) - -def get_eoq_for_period(period, period_info, return_type='str'): - """ - get end of period based on period and period_info - """ - if len(period) == 4: - if return_type == 'datetime': - return epoch(current_period(period_info[0].begin, 'Y').end).as_datetime() - return str(epoch(current_period(period_info[0].begin,'Y').end)) - if 'Q' not in period: - return None - for p in period_info: - if p.mnemonic == period: - if return_type == 'datetime': - return epoch(p.end).as_datetime() - return str(epoch(p.end)) - return None - def prev_period(a_datetime=None, period_type='Q', skip=1, count=1): return period_details(a_datetime, period_type, delta=-skip, count=count) @@ -654,26 +621,6 @@ def prev_periods_allowed_in_deals(): prev_periods.append(prev_period(period_type='Q', skip=j + (4 * i)).mnemonic) return prev_periods -def get_nested_with_placeholder(input_dict, nesting, placeholder_dict, default=None): - """ - get value (or dict) from nested dict by specifying the levels to go through - """ - if not nesting: - return input_dict - try: - - for level in nesting[:-1]: - input_dict = input_dict.get(level % placeholder_dict, {}) - - return input_dict.get(nesting[-1], default) - except AttributeError: - # Not a completely nested dictionary - return default - -@memcached -def get_all_periods__m(period_type, filt_out_qs=None): - return get_all_periods(period_type, filt_out_qs) - def period_rng_from_mnem(mnemonic): # using tuples instead of namedtuples in here because this is performance critical tenant_name = sec_context.name @@ -716,39 +663,6 @@ def period_rng_from_mnem(mnemonic): else: raise Exception("No match for mnemonic: %s", mnemonic) -def get_bow2(ep): - """ - Compute the weekly boundary (get_bow) timestamp for STLY/STLQ calculations. - - For Sunday, Monday, Tuesday, return the previous Saturday's EOD. - For Wednesday, Thursday, Friday, Saturday, return the upcoming Saturday's EOD. - - Parameters: - ep: An epoch instance that provides an `as_datetime()` method. - - Returns: - An epoch instance representing the boundary of the week, set to Saturday 23:59:59.999. - """ - # Convert epoch to datetime - epdt = ep.as_datetime() - # Python's weekday: Monday=0, Tuesday=1, ... Saturday=5, Sunday=6 - w = epdt.weekday() - - if w in [6, 0, 1]: - # For Sunday (6), Monday (0), Tuesday (1), return previous Saturday's date. - # Calculate days to subtract: (w - 5) mod 7 gives: for Sunday: 1, Monday: 2, Tuesday: 3. - delta = (w - 5) % 7 - boundary_dt = epdt - timedelta(days=delta) - else: - # For Wednesday (2), Thursday (3), Friday (4), Saturday (5), return upcoming Saturday. - # Calculate days to add: (5 - w) mod 7 gives: for Wednesday: 3, Thursday: 2, Friday: 1, Saturday: 0. - delta = (5 - w) % 7 - boundary_dt = epdt + timedelta(days=delta) - - # Set the time to 23:59:59.999 (using 999000 microseconds) - boundary_dt = boundary_dt.replace(hour=23, minute=59, second=59, microsecond=999000) - return epoch(boundary_dt) - def get_week_ago(ep): """ get date a week ago from epoch, returns an epoch @@ -778,23 +692,6 @@ def get_boq(ep): period_info = period_details(ep.as_datetime()) return epoch(period_info.begin) - -def lazy_get_boq(ep): - """ - get inexact beginning of quarter by going back 90 days, returns an epoch - """ - day = ep.as_datetime() - return epoch(day.replace(minute=0, second=0) - timedelta(days=90)) - - -def get_eoq(ep): - """ - get end of period from epoch, returns an epoch - """ - period_info = period_details(ep.as_datetime()) - return epoch(period_info.end) - - def get_future_bom(ep, delta=None, months=0): """ get beginning of a future month that months away from epoch, returns an epoch @@ -903,7 +800,7 @@ def get_weekly_periods_info_in_quarter(q_mnemonic, today_epoch): dict: A dictionary mapping weekly period mnemonics to their details. """ - from config.periods_config import PeriodsConfig + from config import PeriodsConfig from infra.read import render_period period_config = PeriodsConfig() @@ -934,6 +831,7 @@ def get_now(verbose=False,): if not verbose and not is_demo(): return epoch() tc = sec_context.details.get_config(category='forecast', config_name='tenant') + from domainmodel.csv_data import CSVDataClass BookingsCacheClass = CSVDataClass(BOOKINGS_CACHE_TYPE, BOOKINGS_CACHE_SUFFIX) @@ -1029,7 +927,7 @@ def rng_from_prd_str(std_prd_str, fmt='epoch', period_type='Q', user_level=False return the boundaries of that period as a tuple of (begin, end)""" if user_level: - from config.periods_config import PeriodsConfig + from config import PeriodsConfig periodconfig = PeriodsConfig() from infra.read import get_now as infra_get_now today = infra_get_now(periodconfig) @@ -1123,7 +1021,7 @@ def rng_from_prd_str(std_prd_str, fmt='epoch', period_type='Q', user_level=False if day_of_week in cur_week_days: target_week_status.append('c') - from config.periods_config import PeriodsConfig + from config import PeriodsConfig from infra.read import render_period periodconfig = PeriodsConfig() current_p = current_period().mnemonic diff --git a/utils/file_utils.py b/utils/file_utils.py index 292382e..9a7b840 100644 --- a/utils/file_utils.py +++ b/utils/file_utils.py @@ -14,92 +14,6 @@ logger = logging.getLogger('gnana.%s' % __name__) -def retry(retries=3): - def retry_decorator(f): - def wrapper(*args, **kwargs): - for i in range(retries): - try: - ret = f(*args, **kwargs) - break - except Exception as e: - logger.exception(f"__func__ {f.__name__} failed! :-( ") - wait_time = (i + 1) * 30 - logger.info("Waiting for %d seconds before retrying %s" % (wait_time, f.__name__)) - time.sleep(wait_time) - if i == 2: - raise e - pass - return ret - return wrapper - return retry_decorator - - -@retry(3) -def process_shell_command(path, process_command_list, command): - try: - process_cmd = Popen(process_command_list, cwd=path, stdout=PIPE) - exit_code = os.waitpid(process_cmd.pid, 0) - output = process_cmd.communicate()[0] - if exit_code[1] != 0: - logger.info("command:%s :error:%s", command, str(output)) - raise GnanaError("command:%s :error:%s" % (command, str(output))) - return - except: - raise - - -def pull_config_repo(): - try: - repopath = os.environ.get('CONFIG_REPO', '/opt/gnana/config') - logger.info("Pulling the repo") - process_shell_command(repopath, ["git", "pull"], "pull") - logger.info("repo pulled") - except: - logger.warning("Problem in pulling config repository ") - raise - -def gitbackup_dataset(file_name, content, comment): - status = True - git_errors = None - - if ISPROD or BACKUP_CONFIG: - try: - repopath = os.environ.get('CONFIG_REPO', '/opt/gnana/config') - pull_config_repo() - - tenantpath = os.path.join(repopath, sec_context.name) - logger.info(f"tenantpath: {tenantpath}") - - filepath = os.path.join(sec_context.name, file_name) - logger.info(f"filepath: {filepath}") - - if not os.path.exists(tenantpath): - os.mkdir(tenantpath) - logger.info("Created folder with tenant name") - - full_file_path = os.path.join(repopath, filepath) - with open(full_file_path, "w") as f: - f.write(pprint.pformat(content, indent=2)) - logger.info("Writing to file completed") - - logger.info("Adding the file") - process_shell_command(repopath, ["git", "add", filepath], "add") - logger.info("Added file to the repo") - - process_shell_command(repopath, ["git", "commit", "-m", comment], "commit") - logger.info("Repo committed") - - logger.info("Pushing the repo") - process_shell_command(repopath, ["git", "push"], "push") - logger.info("Config repo updated") - except Exception as e: - logger.warning("Problem in updating config repository ") - git_errors = f"Problem in updating config repository e" - logger.info(git_errors) - status = False - - return status, git_errors - def filemd5(filename, block_size=2**20): f = open(filename) md5 = hashlib.md5() diff --git a/utils/mail_utils.py b/utils/mail_utils.py index b184136..ab95a60 100644 --- a/utils/mail_utils.py +++ b/utils/mail_utils.py @@ -1,15 +1,11 @@ import logging import os -import pprint - import jinja2 from aviso import settings from aviso.settings import sec_context, CNAME_DISPLAY_NAME, CNAME from django.core.mail import EmailMultiAlternatives from django.core.mail.message import EmailMessage -from utils import diff_rec -from utils.file_utils import gitbackup_dataset logger = logging.getLogger('gnana.%s' % __name__) @@ -133,24 +129,3 @@ def send_mail2(fname, sender, tolist, is_html=False, **kwargs): except Exception as e: logger.exception('kwargs %s - %s - subject %s' % (kwargs, e, subject_line)) raise e - - -def backup_and_mail_changes(old_tdetails, new_tdetails, tenant_name, username, comment, category): - - gitcomment = (comment + " -user:" + username) - file_name = "tenantconfig_" + tenant_name + ".json" - content = new_tdetails - mail_changes = diff_rec(old_tdetails, new_tdetails) - if mail_changes: - gitbackup_dataset(file_name, content, gitcomment) - format_changes = pprint.pformat(mail_changes, indent=2) - cname = CNAME_DISPLAY_NAME if CNAME_DISPLAY_NAME else CNAME - send_mail2('tenant_config.txt', - 'Aviso ', - new_tdetails.get('receivers', {}).get('dataset_changes', ['gnackers@aviso.com']), - reply_to='Data Science Team ', - tenantname=sec_context.name, - comment=comment, changes=format_changes, category=category, - modifier=username, user_name=sec_context.user_name, - cName=cname) - return mail_changes diff --git a/utils/misc_utils.py b/utils/misc_utils.py index 1b2df86..b2520b4 100644 --- a/utils/misc_utils.py +++ b/utils/misc_utils.py @@ -1,9 +1,12 @@ +import logging +import time from datetime import datetime from itertools import chain, combinations from aviso.settings import sec_context from operator import lt, gt, le, ge, itemgetter +logger = logging.getLogger('aviso-core.%s' % __name__) range_lambdas = {} CURR_TABLE = {'USD':'$', 'CAD': '$', 'GBP':u'\xa3','JPY':u'\xa5', @@ -409,3 +412,248 @@ def prune_pfx(fld): return fld[7:] else: return fld + +def update_rtfm_flag(td, last_exec_time=None, fastlane_sync_until=None, chipotle_last_exec_time=None, eoq_time=None, period=None, status='active'): + t_fl = td.get_flag('molecule_status', 'rtfm') + if last_exec_time: + from utils.date_utils import epoch + t_fl['last_execution_time'] = last_exec_time + t_fl['ui_display_time'] = epoch().as_epoch()#Just to know when the chipotle was completed + logger.info("UI time updated as %s" %t_fl['ui_display_time']) + if fastlane_sync_until: + t_fl['fastlane_sync_until'] = fastlane_sync_until + if chipotle_last_exec_time: + t_fl['chipotle_last_execution_time'] = chipotle_last_exec_time + if eoq_time and period: + t_fl[period+'_eoq_time'] = eoq_time + if not t_fl[period+'_eoq_time']: + t_fl[period+'_eoq_time'] = {} + t_fl[period+'_eoq_time'] = eoq_time + t_fl['status'] = status + logger.info("Updating the rtfm flag : %s " % t_fl) + td.set_flag('molecule_status', 'rtfm', t_fl) + +def update_stats(params, status='started', event_type='all_caches', etl_time=None): + from aviso.framework import tracer + from aviso.settings import CNAME + from gnana.events import EventBus + EVENT_BUS = EventBus() + if params.get('etl_time'): + if 'start_time' not in params: + params['start_time'] = int(time.time() * 1000) + payload_data = {'service': CNAME, + 'stack': CNAME, + 'parent_task': 'caches', + 'tenant': sec_context.name, + 'run_type': params.get('run_type', 'daily'), + 'etl_time': etl_time if etl_time else params.get('etl_time'), + 'main_id': params.get('etl_time'), + 'event_type': event_type, + 'times': {'start_time' : params['start_time']}, + 'time_taken': 0, + 'status': status, + 'trace_id': tracer.trace} + if status != 'started': + payload_data['time_taken'] = (time.time() * 1000 - params['start_time']) / 1000 + if event_type == 'all_caches' and status in ['finished', 'skipped', 'veto']: + payload_data['end_time'] = time.time() * 1000 + tdetails = sec_context.details + if status in ['finished', 'veto']: + mol_stat = tdetails.get_flag('molecule_status', 'rtfm') + payload_data['UI_time'] = mol_stat.get('last_execution_time') + if status == 'failed': + payload_data['end_time'] = time.time() * 1000 + logger.info("payload data %s ", payload_data) + EVENT_BUS.publish("$Stats", **payload_data) + else: + logger.info("stats were not published") + +def _get_daily_trace_date(): + try: + return sec_context.details.get_flag('molecule', 'daily_last_execution_date', 0) + except: + return None + +def _set_daily_trace_date(value): + try: + return sec_context.details.set_flag('molecule', 'daily_last_execution_date', value) + except: + return None + + +def _get_snapshot_trace_datetime(): + try: + return sec_context.details.get_flag('molecule', 'snapshot_trace_execution_date', 0) + except: + return None + +def _set_snapshot_trace_datetime(value): + try: + return sec_context.details.set_flag('molecule', 'snapshot_trace_execution_date', value) + except: + return None + +def check_trace(params, trace_name='chipotle_trace', report='all_caches'): + retry = 0 + from aviso.framework import tracer + t = sec_context.details + caches_trace = None + etl_time = params.get('etl_time') + run_type = params.get('run_type', 'chipotle') + is_eoq_run = params.get("is_eoq_run", False) + if is_eoq_run: + return True, tracer.trace + wait_time = t.get_flag('chipotle', 'wait_time_min', 10) + if run_type == 'daily': + wait_time = 60 + while retry <= wait_time: + caches_trace = t.get_flag('molecule', trace_name, 'finished') + if caches_trace in ['finished', tracer.trace]: + t.set_flag('molecule', trace_name, tracer.trace) + break + elif trace_name == 'daily_trace': + from utils.date_utils import epoch + as_of = epoch().as_datetime() + as_of_date = as_of.strftime("%Y-%m-%d") + daily_run_date = _get_daily_trace_date() + if not daily_run_date or (daily_run_date and daily_run_date < as_of_date): + _set_daily_trace_date(as_of_date) + t.set_flag('molecule', trace_name, tracer.trace) + break + elif trace_name == 'snapshot_trace': + from utils.date_utils import epoch + as_of = epoch().as_datetime() + as_of_date = as_of.strftime("%Y-%m-%d") + snapshot_trace_run_datetime = _get_snapshot_trace_datetime() + if snapshot_trace_run_datetime: + snapshot_run_date = epoch(_get_snapshot_trace_datetime()).as_datetime().strftime("%Y-%m-%d") + if not snapshot_run_date or (snapshot_run_date and snapshot_run_date < as_of_date): + _set_snapshot_trace_datetime(epoch().as_epoch()) + t.set_flag('molecule', trace_name, tracer.trace) + break + else: + _set_snapshot_trace_datetime(epoch().as_epoch()) + t.set_flag('molecule', trace_name, tracer.trace) + break + + logger.info("%s are running by other tasks %s waiting for 1 min", report, caches_trace) + time.sleep(60) + retry += 1 + else: + update_stats(params, 'skipped', report, etl_time) + logger.error("waited for %s min, %s will taken care by next event, %s is %s", wait_time, report, trace_name, caches_trace) + return False, caches_trace + return True, tracer.trace + +def update_trace(trace_name='chipotle_trace', trace_id=None, force_update=False): + from aviso.framework import tracer + t = sec_context.details + if force_update: + t.set_flag('molecule', trace_name, 'finished') + return + if not trace_id: + trace_id = tracer.trace + if t.get_flag('molecule', trace_name, 'finished') == trace_id: + t.set_flag('molecule', trace_name, 'finished') + if trace_name == 'snapshot_trace': + from utils.date_utils import epoch + _set_snapshot_trace_datetime(epoch().as_epoch()) + +def describe_trend(val): + if val in [0, None]: + return 'no_arrow', 'no difference to', 'ok' + if val < 0: + return 'down_arrow', 'below', 'bad' + elif val > 0: + return 'up_arrow', 'above', 'good' + return 'no_arrow', 'no difference to', 'ok' + +def mongify_field(field, no_replace_quote=False): + if isinstance(field, str): + return field.replace('.', '|') if no_replace_quote else field.replace('.', '|').replace("'", "#") + if isinstance(field, int): + return field + if type(field) is list: + return [mongify_field(x, no_replace_quote) for x in field] + if type(field) is tuple: + return tuple(mongify_field(x, no_replace_quote) for x in field) + if type(field) is set: + return {mongify_field(x, no_replace_quote) for x in field} + +def unmongify_field(field): + # return field.replace("#", "'").replace('|', '.') + if isinstance(field, str): + return field.replace("#", "'").replace('|', '.') + if isinstance(field, int): + return field + if type(field) is list: + return [unmongify_field(x) for x in field] + if type(field) is tuple: + return tuple(unmongify_field(x) for x in field) + if type(field) is set: + return {unmongify_field(x) for x in field} + +def mongify_dict(dct): + copy = {} + for k, v in dct.iteritems(): + if type(v) is dict: + copy.update({mongify_field(k): mongify_dict(v)}) + else: + copy.update({mongify_field(k): mongify_field(v)}) + return copy + +def unmongify_dict(d): + copy = {} + for k, v in d.iteritems(): + if type(v) is dict: + copy.update({unmongify_field(k): unmongify_dict(v)}) + else: + copy.update({unmongify_field(k): unmongify_field(v)}) + return copy + +def create_nested(input_dict, attr_list): + """ + create a value from nested input dict by specifying the levels to go through and write it to output dict + """ + if not attr_list: + return None + try: + + + val = get_nested(input_dict, attr_list) + + dict_by_level = val + for key in reversed(attr_list): + dict_by_level = {key: dict_by_level} + + return dict_by_level + except AttributeError: + # Not a completely nested dictionary + return None + +def adorn_dlf_fcst(deal, dlf_fcst_coll_schema): + """ + Create deal level dlf_fcst collection required information + """ + try: + dlf_fcst_record = {'criteria': {'period': deal['period'], + 'opp_id': deal['opp_id']}, + 'set': {}} + for attr in dlf_fcst_coll_schema: + attr_as_list = attr.split('.') + # TODO: The limitation with the below code is if there is are 2 nestings i.e, dlf.in_fcst and dlf.fcst_flag, + # the dlf.in_fcst will be removed and dlf.fcst_flag will be added + if len(attr_as_list) > 0: + dlf_fcst_record['set']= dict(dlf_fcst_record['set'], **create_nested( deal, attr_as_list)) + else: + dlf_fcst_record['set'][attr_as_list[0]] = deal[attr_as_list[0]] + + return dlf_fcst_record + except: + return None + +def get_last_execution_time(): + try: + return sec_context.details.get_flag('molecule_status', 'rtfm', {}).get('chipotle_last_execution_time', 0) + except: + return 0 diff --git a/utils/mongo_reader.py b/utils/mongo_reader.py index 662d71b..29585b3 100644 --- a/utils/mongo_reader.py +++ b/utils/mongo_reader.py @@ -1,50 +1,9 @@ -import copy -import csv import logging -import threading -from collections import OrderedDict -from datetime import datetime, timedelta -from io import BytesIO -from itertools import product -from multiprocessing.pool import ThreadPool import boto3 -import pytz from aviso.settings import sec_context -from date_utils import prev_periods_allowed_in_deals -from config.fm_config import DEFAULT_ROLE, FMConfig -from config.periods_config import PeriodsConfig -from infra import (ADMIN_MAPPING, CRM_SCHEDULE, EDW_DATA, EDW_PROCESS_UPDATE, - EXPORT_ALL, FM_COLL, FM_LATEST_COLL, FM_LATEST_DATA_COLL, - FORECAST_SCHEDULE_COLL, FORECAST_UNLOCK_REQUESTS, - GBM_CRR_COLL, MOBILE_SNAPSHOT_COLL, - MOBILE_SNAPSHOT_ROLE_COLL, NEXTQ_COLL, SNAPSHOT_COLL, - SNAPSHOT_HIST_COLL, SNAPSHOT_HIST_ROLE_COLL, - SNAPSHOT_ROLE_COLL, USER_LEVEL_SCHEDULE, WATERFALL_COLL, - WATERFALL_HISTORY_COLL, WEEKLY_EDW_DATA, - WEEKLY_EDW_PROCESS_START_TIME, WEEKLY_EDW_PROCESS_STATUS, - WEEKLY_FORECAST_EXPORT_ALL, WEEKLY_FORECAST_EXPORT_COLL, - WEEKLY_FORECAST_FM_COLL) -from infra.read import (fetch_boundry, fetch_children, fetch_crr_deal_totals, - fetch_deal_totals, fetch_descendants, - fetch_eligible_nodes_for_segment, fetch_labels, - fetch_many_dr_from_deals, fetch_prnt_DR_deal_totals, - find_all_subtree_height, - find_map_in_nodes_and_lth_grand_children, - get_available_quarters_and_months, get_now, - get_period_and_close_periods, - get_period_and_component_periods, get_period_as_of, - get_period_begin_end, get_periods_editable, - get_quarter_period, get_time_context, - get_waterfall_weekly_totals, render_period) -from infra.read import time_context as time_context_tuple -from infra.rules import node_aligns_to_segment -from utils.date_utils import (EpochClass, datetime2epoch, epoch, - epoch2datetime, get_all_periods__m, get_eod, - get_eom, get_eoq_for_period, is_same_day, - monthly_periods, weekly_periods, xl2datetime_ttz) -from utils.misc_utils import get_nested, try_float +from infra.constants import FORECAST_SCHEDULE_COLL logger = logging.getLogger('gnana.%s' % __name__) @@ -55,7184 +14,33 @@ MAX_THREADS = 200 SIX_HOURS = 6 * 60 * 60 * 1000 - -def read_from_collection(year, timestamp=None, field_type=None, call_from=None, config=None, - year_accepted_from=None): - if timestamp is not None: - return FM_COLL - fm_config = config if config else FMConfig() - read_from_latest_collection = fm_config.read_from_latest_collection - if year_accepted_from is None: - try: - t = sec_context.details - year_accepted_from = int(t.get_flag('fm_latest_migration', 'year', 0)) - except: - year_accepted_from = 0 - coll = FM_COLL - if year_accepted_from != 0 and read_from_latest_collection and int(year) >= year_accepted_from: - if field_type is not None and field_type in ['DR', 'PrntDR']: - coll = FM_LATEST_COLL - else: - coll = FM_LATEST_DATA_COLL - return coll - - -# not in use please verify -def bulk_fetch_fm_recs(time_context, - descendants_list, - fields, - segments, - config, - eligible_nodes_for_segs, - timestamp=None, - recency_window=None, - db=None, - ): - """ - fetch fm records from db for multiple nodes/fields - optimized for fetching for many nodes and fields at once - it may turn out this is faster in all cases than regular fetch_fm_recs - in which case we should just delete that function and always use this one - you are 100% guaranteed to get records for all the params you request - - Arguments: - time_context {time_context} -- fm period, components of fm period - deal period, close periods of deal period - deal expiration timestamp, - relative periods of component periods - ('2020Q2', ['2020Q2'], - '2020Q2', ['201908', '201909', '201910'], - 1556074024910, - ['h', 'c', 'f']) - descendants_list {list} -- list of tuples of - (node, [children], [grandchildren]) - fetches data for each node - using children/grandkids to compute sums - [('A', ['B, 'C'], ['D', 'E'])] - fields {list} -- list of field names - ['commit', 'best_case'] - segments {list} -- list of segment names - ['all_deals', 'new', 'upsell'] - config {FMConfig} -- instance of FMConfig - - Keyword Arguments: - timestamp {int} -- epoch timestamp to get data as of (default: {None}) - if None, gets most recent record - 1556074024910 - recency_window {int} -- window to reject data from before timestamp - recency_window (default: {None}) - if None, will accept any record, regardless of how stale - ONE_DAY - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - dict -- {(period, node, field, segment, timestamp): {'period': period, - 'val': val, - 'by': by, - 'how': how, - 'found': found, - 'timestamp': timestamp, - 'node': node, - 'field': field}} - """ - threads, cache = [], {} - db = db if db else sec_context.tenant_db - - if timestamp: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, list) else time_context.fm_period - dlf_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - - if config.debug: - for descendants, field, segment in product(descendants_list, fields, segments): - if segment == 'all_deals' or descendants[0] in eligible_nodes_for_segs.get(segment, []): - fetch_fm_rec(time_context, - descendants, - field, - segment, - config, - (timestamp, dlf_ts, deal_ts, timestamp - recency_window if recency_window else None), - db, - cache) - else: - # instead of spinning up a million threads and waiting a bunch, make a pool of 200 and cycle through em - # theres some threshold of how many is too many threads where we wind up waiting too much - # from my expirementing locally it looks like we only hit it on this bulk fetch for exports and not on the snapshots - # but if that changes, may want to take this approach in fetch_fm_recs and fetch_fm_recs_history - # NOTE: if this ever gets upgraded to python3, we should really be using concurrent.futures ThreadPoolExecutor, not this - count = 0 - actual_count = 0 - pool = ThreadPool(MAX_THREADS) - fm_rec_params = [] - for descendants, field, segment in product(descendants_list, fields, segments): - count += 1 - if segment == 'all_deals' or descendants[0] in eligible_nodes_for_segs.get(segment, []): - actual_count += 1 - fm_rec_params.append((time_context, - descendants, - field, - segment, - config, - (timestamp, dlf_ts, deal_ts, timestamp - recency_window if recency_window else None), - db, - cache)) - pool.map(_fetch_fm_rec, tuple(fm_rec_params)) - pool.close() - pool.join() - logger.info('actual comibnations: %s, threads executed count : %s', count, actual_count) - - return cache - - -def bulk_fetch_recs_by_timestamp(time_context, - descendants, - fields, - segments, - config, - eligible_nodes_for_segs, - timestamps, - db=None, - round_val=False, - node=None, - is_pivot_special=False, - call_from=None, - time_context_list = [], - recalc_UE_fields = [], - recalc_DR_fields = [] - ): - fm_recs = {} - for timestamp in timestamps: - fm_recs.update(bulk_fetch_recs_for_data_export(time_context, - descendants, - fields, - segments, - config, - eligible_nodes_for_segs, - timestamp, - db, - round_val=round_val, - is_pivot_special=is_pivot_special, - call_from=call_from, - time_context_list = time_context_list, - recalc_UE_fields = recalc_UE_fields, - recalc_DR_fields = recalc_DR_fields - )) - unique_fields = set(fields) - for timestamp, field, segment, descendant in product(timestamps, unique_fields, segments, descendants): - field_type = config.fields[field]['type'] - node, children, _ = descendant - if field_type == 'NC': - mgr_field, rep_field = config.fields[field]['source'] - val = {} - if children: - val = fm_recs.get((time_context.fm_period, node, mgr_field, segment, timestamp)) - else: - val = fm_recs.get((time_context.fm_period, node, rep_field, segment, timestamp)) - if val: - fm_recs[(time_context.fm_period, node, field, segment, timestamp)] = val - - - return fm_recs - -def bulk_fetch_recs_by_timestamp_cq(time_context, - descendants, - fields, - segments, - config, - eligible_nodes_for_segs, - timestamps, - db=None, - round_val=False, - node=None - ): - fm_recs = {} - #for timestamp in timestamps: - fm_recs.update(bulk_fetch_recs_for_data_export_new(time_context, - descendants, - fields, - segments, - config, - eligible_nodes_for_segs, - timestamps, - db, - round_val=round_val, - node=node)) - unique_fields = set(fields) - for timestamp, field, segment, descendant in product(timestamps, unique_fields, segments, descendants): - field_type = config.fields[field]['type'] - node, children, _ = descendant - if field_type == 'NC': - mgr_field, rep_field = config.fields[field]['source'] - val = {} - if children: - val = fm_recs.get((time_context.fm_period, node, mgr_field, segment, timestamp)) - else: - val = fm_recs.get((time_context.fm_period, node, rep_field, segment, timestamp)) - if val: - fm_recs[(time_context.fm_period, node, field, segment, timestamp)] = val - - - return fm_recs - -def get_fields_by_type(fields, config, time_context, special_cs_fields=[]): - fields_by_type = {} - for field in set(fields): - if 'hist_field' in config.fields[field] and 'h' in time_context.relative_periods: - # historic period, switch to using true up field if it exists - field_type = 'PC' - if config.quarter_editable: - if not all(elem == 'h' for elem in time_context.relative_periods): - field_type = config.fields[field]['type'] - else: - field_type = config.fields[field]['type'] - if field_type == 'NC': - mgr_field, rep_field = config.fields[field]['source'] - - mgr_field_type = config.fields[mgr_field]['type'] - rep_field_type = config.fields[rep_field]['type'] - - if mgr_field_type not in fields_by_type: - fields_by_type[mgr_field_type] = [] - fields_by_type[mgr_field_type].append(mgr_field) - if mgr_field_type == 'CS': - source_fields = config.fields[mgr_field]['source'] - eager_fields_by_type, special_cs_fields = get_fields_by_type(source_fields, config, time_context, special_cs_fields) - for type in eager_fields_by_type: - if type not in fields_by_type: - fields_by_type[type] = [] - fields_by_type[type].extend(eager_fields_by_type[type]) - - if rep_field_type not in fields_by_type: - fields_by_type[rep_field_type] = [] - fields_by_type[rep_field_type].append(rep_field) - if rep_field_type == 'CS': - source_fields = config.fields[rep_field]['source'] - eager_fields_by_type, special_cs_fields = get_fields_by_type(source_fields, config, time_context, special_cs_fields) - for type in eager_fields_by_type: - if type not in fields_by_type: - fields_by_type[type] = [] - fields_by_type[type].extend(eager_fields_by_type[type]) - - elif field_type == 'CS': - source_fields = config.fields[field]['source'] - field_type_ = config.fields[source_fields[0]]['type'] - special_case = False - if 'hist_field' in config.fields[source_fields[0]] and 'h' in time_context.relative_periods: - # historic period, switch to using true up field if it exists - field_type_ = 'PC' - if config.quarter_editable: - if not all(elem == 'h' for elem in time_context.relative_periods): - field_type_ = config.fields[source_fields[0]]['type'] - if field_type_ == 'NC': - mgr_field, rep_field = config.fields[source_fields[0]]['source'] - if config.fields[mgr_field]['type'] == 'CS' or config.fields[rep_field]['type'] == 'CS': - special_cs_fields.append(field) - special_case = True - elif field_type_ == 'PC': - special_cs_fields.append(field) - special_case = True - - if not special_case: - if field_type not in fields_by_type: - fields_by_type[field_type] = [] - fields_by_type[field_type].append(field) - eager_fields_by_type, special_cs_fields = get_fields_by_type(source_fields, config, time_context, special_cs_fields) - for type in eager_fields_by_type: - if type not in fields_by_type: - fields_by_type[type] = [] - fields_by_type[type].extend(eager_fields_by_type[type]) - else: - if field_type not in fields_by_type: - fields_by_type[field_type] = [] - fields_by_type[field_type].append(field) - return fields_by_type, special_cs_fields - - -def bulk_fetch_recs_for_data_export_new(time_context, - descendants, - fields, - segments, - config, - eligible_nodes_for_segs, - timestamps=None, - db=None, - round_val=True, - node=None - ): - cache = {} - - db = db if db else sec_context.tenant_db - - '''if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, list) else time_context.fm_period - dlf_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0))''' - - special_cs_fields = [] - fields_by_type, special_cs_fields = get_fields_by_type(fields, config, time_context, special_cs_fields) - - eager_retrieval_fields = [] - if 'FN' in fields_by_type: - for fn_field in fields_by_type['FN']: - eager_retrieval_fields.extend(config.fields[fn_field]['source']) - - eager_fields_by_type, special_cs_fields = get_fields_by_type(eager_retrieval_fields, config, time_context, special_cs_fields) - - for type in eager_fields_by_type: - if type not in fields_by_type: - fields_by_type[type] = [] - fields_by_type[type].extend(eager_fields_by_type[type]) - - for type, fields in fields_by_type.items(): - fields_by_type[type] = list(set(fields)) - - # AV-14059 Commenting below as part of the log fix - # logger.info("fields for bulk data fetch are %s" % fields_by_type) - - if len(segments) == 1 and 'all_deals' in segments and config.has_segments: - segments = config.segments - - segments = [segment for segment in segments] - - if any([segment in config.FN_segments for segment in segments]): - segments += add_FN_dependent_segments(segments, config, []) - segments = list(set(segments)) - - '''timestamp_info = (timestamp, dlf_ts, deal_ts,None)''' - - threads = 0 - pool = ThreadPool(MAX_THREADS) - - if 'UE' in fields_by_type: - UE_params = [] - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - UE_params.append((time_context, - descendants, - fields_by_type['UE'], - segments, - config, - timestamp_info, - db, - cache, - True, - round_val)) - threads += 1 - pool.map(_bulk_fetch_user_entered_recs_pool, tuple(UE_params)) - - if 'DR' in fields_by_type: - DR_params = [] - config.debug = True - try: - dr_from_fm_coll_for_snapshot = sec_context.details.get_flag('deal_rollup', 'snapshot_dr_from_fm_coll', - False) - except: - dr_from_fm_coll_for_snapshot = False - if config.deal_config.segment_field or not config.has_segments or dr_from_fm_coll_for_snapshot: - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - DR_params.append((config, - fields_by_type['DR'], - time_context, - descendants, - segments, - timestamp_info, - db, - cache, - round_val, - timestamp, - eligible_nodes_for_segs, - True, - True - )) - threads += 1 - pool.map(handle_dr_fields_pool, tuple(DR_params)) - elif not config.deal_config.segment_field and config.has_segments: - config.debug = True - for field, segment, descendant, timestamp in product(fields_by_type['DR'], segments, descendants, timestamps): - if segment == 'all_deals' or descendants[0] in eligible_nodes_for_segs.get(segment, []): - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - DR_params.append((time_context, - descendant, - field, - segment, - config, - (timestamp, dlf_ts, deal_ts, None), - db, - cache)) - threads += 1 - pool.map(_fetch_fm_rec, tuple(DR_params)) - - if 'print_DR' in fields_by_type: - print_DR_params = [] - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - print_DR_params.append((config, - fields_by_type['prnt_DR'], - time_context, - descendants, - segments, - timestamp_info, - db, - cache, - timestamp, - eligible_nodes_for_segs, - round_val, - True)) - threads += 1 - pool.map(_bulk_fetch_prnt_dr_recs_pool, tuple(print_DR_params)) - - if 'AI' in fields_by_type: - AI_params = [] - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - AI_params.append((time_context, - descendants, - fields_by_type['AI'], - segments, - config, - timestamp_info, - db, - cache, - False, - round_val)) - threads += 1 - pool.map(_bulk_fetch_ai_recs_pool, tuple(AI_params)) - - descendants_by_segments = {} - rollup_segments = config.rollup_segments - for segment in segments: - descendants_list = [] - if segment == 'all_deals': - descendants_list = descendants - else: - for node in descendants: - if node[0] in eligible_nodes_for_segs.get(segment, []): - descendants_list.append(node) - descendants_by_segments[segment] = descendants_list - if 'PC' in fields_by_type: - PC_params = [] - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - PC_params.append((time_context, - descendants_list, - fields_by_type['PC'], - segment, - config, - timestamp_info, - db, - cache, - round_val)) - threads += 1 - pool.map(_bulk_period_conditional_rec_pool, tuple(PC_params)) - - for segment in segments: - descendants_list = descendants_by_segments[segment] - if 'FN' in fields_by_type: - FN_params = [] - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - FN_params.append((time_context, - descendants_list, - fields_by_type['FN'], - segment, - config, - timestamp_info, - db, - cache)) - threads += 1 - pool.map(_bulk_fetch_formula_recs_pool, tuple(FN_params)) - - if 'CS' in fields_by_type: - CS_derived_fields, _ = get_fields_by_type(fields_by_type['CS'], config, time_context) - CS_derived_UE_fields = CS_derived_fields.get('UE', []) - CS_derived_DR_fields = CS_derived_fields.get('DR', []) - CS_derived_FN_fields = CS_derived_fields.get('FN', []) - descendants_list = descendants - child_descendants = [] - for descendant in descendants_list: - node, children, grandchildren = descendant - for child in children: - grandkids = {grandkid: parent for grandkid, parent in grandchildren.iteritems() - if parent == child} - child_descendant = (child, grandkids, {}) - child_descendants.append(child_descendant) - for grandkid in grandkids: - grandkid_descendant = (grandkid, {}, {}) - child_descendants.append(grandkid_descendant) - cs_cache = {} - CS_UE_params = [] - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - CS_UE_params.append((time_context, - child_descendants, - CS_derived_UE_fields, - segments, - config, - timestamp_info, - db, - cs_cache, - True, - round_val)) - threads += 1 - pool.map(_bulk_fetch_user_entered_recs_pool, tuple(CS_UE_params)) - if CS_derived_DR_fields: - CS_DR_params = [] - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - config.debug = True - CS_DR_params.append((config, - CS_derived_DR_fields, - time_context, - child_descendants, - segments, - timestamp_info, - db, - cs_cache, - round_val, - timestamp, - eligible_nodes_for_segs, - True, - True - )) - threads += 1 - pool.map(handle_dr_fields_pool, tuple(CS_DR_params)) - if CS_derived_FN_fields: - CS_FN_params = [] - for segment in segments: - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - CS_FN_params.append((time_context, - child_descendants, - CS_derived_FN_fields, - segment, - config, - timestamp_info, - db, - cs_cache)) - threads += 1 - pool.map(_bulk_fetch_formula_recs_pool, tuple(CS_FN_params)) - - for segment in segments: - descendants_list = descendants_by_segments[segment] - if special_cs_fields: - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - _bulk_fetch_child_sum_rec_special(time_context, - descendants_list, - special_cs_fields, - segment, - config, - timestamp_info, - db, - cache, - round_val=round_val) - for segment in segments: - descendants_list = descendants_by_segments[segment] - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, - list) else time_context.fm_period - dlf_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float( - sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - cache.update(_bulk_fetch_child_sum_rec(time_context, - child_descendants, - fields_by_type['CS'], - segment, - config, - timestamp_info, - db, - cs_cache, - round_val=round_val)) - - cache.update(_bulk_fetch_child_sum_rec(time_context, - descendants_list, - fields_by_type['CS'], - segment, - config, - timestamp_info, - db, - cs_cache, - round_val=round_val)) - if len(config.segments) > 1: - by_val = "system" - latest_timestamp = 0 - period, comp_periods = time_context.fm_period, time_context.submission_component_periods - cs_field_with_fn_source=[] - for field in fields_by_type['CS']: - source_field = config.fields[field]['source'][0] - if config.fields[source_field]['type'] == 'FN': - cs_field_with_fn_source.append(field) - cs_field_without_fn_source = [x for x in fields_by_type['CS'] if x not in cs_field_with_fn_source] - for descendants, field in product(descendants_list, cs_field_without_fn_source): - node, children, grandchildren = descendants - val = 0 - for segment in segments: - if segment != "all_deals": - ch_key = (period, node, field, segment, timestamp) - if segment in rollup_segments: - val += try_float(get_nested(cache, [ch_key, 'val'], 0)) - seg_timestamp = get_nested(cache, [ch_key, 'timestamp'], 0) - if seg_timestamp > latest_timestamp: - latest_timestamp = seg_timestamp - by_val = get_nested(cache, [ch_key, 'by'], "system") - res = {'period': period, - 'segment': config.primary_segment, - 'val': val, - 'by': by_val, - 'how': 'sum_of_children', - 'found': True if val else False, - 'timestamp': latest_timestamp, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, config.primary_segment, timestamp)] = res - pool.close() - pool.join() - - for timestamp in timestamps: - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, list) else time_context.fm_period - dlf_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - - timestamp_info = (timestamp, dlf_ts, deal_ts,None) - all_fields = set([]) - for _, type_fields in fields_by_type.items(): - all_fields.update(type_fields) - populate_FN_segments(time_context, - all_fields, - config.FN_segments, - descendants_by_segments, - cache, - timestamp, - [], - config, - fields_by_type, - timestamp_info) - - return cache - -def bulk_fetch_recs_for_data_export_only_UE(time_context, - descendants, - fields, - segments, - config, - eligible_nodes_for_segs, - timestamp=None, - db=None, - round_val=True, - node=None, - includefuturetimestamp=False, - is_pivot_special=False, - call_from=None, - time_context_list = [], - exclude_empty = None, - updated_since = None, - skip=None, - limit=None): - cache = {} - - db = db if db else sec_context.tenant_db - - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, list) else time_context.fm_period - dlf_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - - special_cs_fields = [] - fields_by_type, special_cs_fields = get_fields_by_type(fields, config, time_context, special_cs_fields) - - is_special_pivot_segmented = is_pivot_special - if node: - is_special_pivot_segmented = config.is_special_pivot_segmented(node) - if len(segments) == 1 and 'all_deals' in segments and config.has_segments and is_special_pivot_segmented: - segments = config.segments - - segments = [segment for segment in segments] - - if any([segment in config.FN_segments for segment in segments]): - segments += add_FN_dependent_segments(segments, config, []) - segments = list(set(segments)) - - timestamp_info = (timestamp, dlf_ts, deal_ts,None) - - - if 'UE' in fields_by_type: - records = _bulk_fetch_user_entered_recs_v2(time_context, - descendants, - list(set(fields_by_type['UE'])), - segments, - config, - timestamp_info, - db, - cache, - get_all_segments=True, - round_val=round_val, - includefuturetimestamp=includefuturetimestamp, - exclude_empty=exclude_empty, - updated_since=updated_since, - skip=skip, - limit=limit) - - return records - -def bulk_fetch_recs_for_data_export(time_context, - descendants, - fields, - segments, - config, - eligible_nodes_for_segs, - timestamp=None, - db=None, - round_val=True, - node=None, - includefuturetimestamp=False, - is_pivot_special=False, - call_from=None, - time_context_list = [], - recalc_UE_fields = [], - recalc_DR_fields = [] - ): - cache = {} - - db = db if db else sec_context.tenant_db - - if timestamp and round_val: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, list) else time_context.fm_period - dlf_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - - special_cs_fields = [] - fields_by_type, special_cs_fields = get_fields_by_type(fields, config, time_context, special_cs_fields) - - for type_key in ('DR', 'UE'): - if type_key in fields_by_type: - - if type_key == 'UE': - fields_by_type[type_key].extend(recalc_UE_fields) - elif type_key == 'DR': - fields_by_type[type_key].extend(recalc_DR_fields) - - fields_by_type[type_key] = list(set(fields_by_type[type_key])) - - - bucket_forecast_field = config.config.get('bucket_forecast_field') - bucket_fields = config.config.get('bucket_fields', []) - - eager_retrieval_fields = [] - if 'FN' in fields_by_type: - for fn_field in fields_by_type['FN']: - eager_retrieval_fields.extend(config.fields[fn_field]['source']) - - eager_fields_by_type, special_cs_fields = get_fields_by_type(eager_retrieval_fields, config, time_context, special_cs_fields) - - FN_field_prioritize = [] - for type in eager_fields_by_type: - if type == 'FN': - FN_field_prioritize.extend(eager_fields_by_type[type]) - continue - if type not in fields_by_type: - fields_by_type[type] = [] - fields_by_type[type].extend(eager_fields_by_type[type]) - - for type, fields in fields_by_type.items(): - fields_by_type[type] = list(set(fields)) - - for fn_field in FN_field_prioritize: - if fn_field in fields_by_type['FN']: - fields_by_type['FN'].remove(fn_field) - - # AV-14059 Commenting below as part of the log fix - logger.info("fields for bulk data fetch are %s special_cs_fields %s" % (fields_by_type, - special_cs_fields)) - is_special_pivot_segmented = is_pivot_special - if node: - is_special_pivot_segmented = config.is_special_pivot_segmented(node) - if len(segments) == 1 and 'all_deals' in segments and config.has_segments and is_special_pivot_segmented: - segments = config.segments - - segments = [segment for segment in segments] - - if any([segment in config.FN_segments for segment in segments]): - segments += add_FN_dependent_segments(segments, config, []) - segments = list(set(segments)) - - timestamp_info = (timestamp, dlf_ts, deal_ts,None) - FN_derived_UE_fields = [] - FN_derived_DR_fields = [] - if FN_field_prioritize: - FN_derived_fields, _ = get_fields_by_type(FN_field_prioritize, config, time_context) - FN_derived_UE_fields = FN_derived_fields.get('UE', []) - FN_derived_DR_fields = FN_derived_fields.get('DR', []) - - if 'UE' in fields_by_type: - if not config.month_to_qtr_rollup: - _bulk_fetch_user_entered_recs(time_context, - descendants, - list(set(fields_by_type['UE'] + FN_derived_UE_fields)), - segments, - config, - timestamp_info, - db, - cache, - get_all_segments=True, - round_val=round_val, - includefuturetimestamp=includefuturetimestamp) - else: - for field_ in config.weekly_data: - if field_ in fields_by_type['UE']: - fields_by_type['UE'].remove(field_) - if field_ in FN_derived_UE_fields: - FN_derived_UE_fields.remove(field_) - if field_ in config.quarterly_high_low_fields: - FN_derived_UE_fields.remove(field_) - - if len(list(set(fields_by_type['UE'] + FN_derived_UE_fields))) >= 1: - _bulk_fetch_user_entered_recs(time_context, - descendants, - list(set(fields_by_type['UE'] + FN_derived_UE_fields)), - segments, - config, - timestamp_info, - db, - cache, - get_all_segments=True, - round_val=round_val, - includefuturetimestamp=includefuturetimestamp) - - if config.weekly_data + config.quarterly_high_low_fields: - _bulk_fetch_user_entered_recs(time_context, - descendants, - config.weekly_data + config.quarterly_high_low_fields, - segments, - config, - timestamp_info, - db, - cache, - get_all_segments=True, - round_val=round_val, - includefuturetimestamp=includefuturetimestamp) - - - if 'DR' in fields_by_type: - handle_dr_fields(config, - list(set(fields_by_type['DR'] + FN_derived_DR_fields)), - time_context, - descendants, - segments, - timestamp_info, - db, - cache, - round_val, - timestamp, - eligible_nodes_for_segs, - call_from=call_from, - time_context_list = time_context_list - ) - - if 'prnt_DR' in fields_by_type: - _bulk_fetch_prnt_dr_recs(config, - fields_by_type['prnt_DR'], - time_context, - descendants, - segments, - timestamp_info, - db, - cache, - timestamp, - eligible_nodes_for_segs, - round_val=round_val, - get_all_segments=True, - node=node) - - if 'AI' in fields_by_type: - _bulk_fetch_ai_recs(time_context, - descendants, - fields_by_type['AI'], - segments, - config, - timestamp_info, - db, - cache, - get_all_segments=True, - round_val=round_val, - bucket_fields=bucket_fields, - bucket_forecast_field=bucket_forecast_field) - - descendants_by_segments = {} - rollup_segments = config.rollup_segments - for segment in segments: - descendants_list = [] - if segment == 'all_deals': - descendants_list = descendants - else: - for node in descendants: - if node[0] in eligible_nodes_for_segs.get(segment, []): - descendants_list.append(node) - descendants_by_segments[segment] = descendants_list - if 'PC' in fields_by_type: - _bulk_period_conditional_rec(time_context, - descendants_list, - fields_by_type['PC'], - segment, - config, - timestamp_info, - db, - cache, - round_val=round_val) - - for segment in segments: - descendants_list = descendants_by_segments[segment] - if FN_field_prioritize: - _bulk_fetch_formula_recs(time_context, - descendants_list, - FN_field_prioritize, - segment, - config, - timestamp_info, - db, - cache) - for segment in segments: - descendants_list = descendants_by_segments[segment] - if 'FN' in fields_by_type: - _bulk_fetch_formula_recs(time_context, - descendants_list, - fields_by_type['FN'], - segment, - config, - timestamp_info, - db, - cache) - - if 'CS' in fields_by_type: - CS_derived_fields, _ = get_fields_by_type(fields_by_type['CS'], config, time_context) - CS_derived_UE_fields = CS_derived_fields.get('UE', []) - CS_derived_DR_fields = CS_derived_fields.get('DR', []) - CS_derived_FN_fields = list(set(CS_derived_fields.get('FN', []))) - eager_CS_FN_fields_by_type, _ = get_fields_by_type(CS_derived_FN_fields, config, - time_context, []) - - # # cs derived fn fields which are dependent on other FN fields should be prioritized - CS_FN_field_prioritize = [] - for type in eager_CS_FN_fields_by_type: - if type == 'FN': - CS_FN_field_prioritize.extend(eager_CS_FN_fields_by_type[type]) - CS_FN_derived_UE_fields = [] - CS_FN_derived_DR_fields = [] - if CS_FN_field_prioritize: - CS_FN_derived_fields, _ = get_fields_by_type(CS_FN_field_prioritize, config, time_context) - CS_FN_derived_UE_fields = CS_FN_derived_fields.get('UE', []) - CS_FN_derived_DR_fields = CS_FN_derived_fields.get('DR', []) - for fn_field_ in list(set(CS_FN_field_prioritize)): - if fn_field_ in CS_derived_FN_fields: - CS_derived_FN_fields.remove(fn_field_) - - descendants_list = descendants - child_descendants = [] - segments_enabled_list = None - for descendant in descendants_list: - node, children, grandchildren = descendant - if node and segments_enabled_list is None: - segments_enabled_list = config.get_segments(epoch().as_epoch(), node) - for child in children: - grandkids = {grandkid: parent for grandkid, parent in grandchildren.iteritems() - if parent == child} - child_descendant = (child, grandkids, {}) - child_descendants.append(child_descendant) - for grandkid in grandkids: - grandkid_descendant = (grandkid, {}, {}) - child_descendants.append(grandkid_descendant) - cs_cache = {} - if not config.month_to_qtr_rollup: - _bulk_fetch_user_entered_recs(time_context, - child_descendants, - list(set(CS_derived_UE_fields + CS_FN_derived_UE_fields)), - segments, - config, - timestamp_info, - db, - cs_cache, - get_all_segments=True, - round_val=round_val) - else: - for field_ in config.weekly_data: - if field_ in CS_derived_UE_fields: - CS_derived_UE_fields.remove(field_) - if field_ in CS_FN_derived_UE_fields: - CS_FN_derived_UE_fields.remove(field_) - if field_ in config.quarterly_high_low_fields: - CS_derived_UE_fields.remove(field_) - if len(list(set(CS_derived_UE_fields + CS_FN_derived_UE_fields))) >= 1: - _bulk_fetch_user_entered_recs(time_context, - child_descendants, - list(set(CS_derived_UE_fields + CS_FN_derived_UE_fields)), - segments, - config, - timestamp_info, - db, - cs_cache, - get_all_segments=True, - round_val=round_val) - if config.weekly_data + config.quarterly_high_low_fields: - _bulk_fetch_user_entered_recs(time_context, - child_descendants, - config.weekly_data + config.quarterly_high_low_fields, - segments, - config, - timestamp_info, - db, - cs_cache, - get_all_segments=True, - round_val=round_val) - - if CS_derived_DR_fields: - handle_dr_fields(config, - list(set(CS_derived_DR_fields + CS_FN_derived_DR_fields)), - time_context, - child_descendants, - segments, - timestamp_info, - db, - cs_cache, - round_val, - timestamp, - eligible_nodes_for_segs - ) - if CS_derived_FN_fields: - if CS_FN_field_prioritize: - for segment in segments: - if segment in segments_enabled_list: - _bulk_fetch_formula_recs(time_context, - child_descendants, - CS_FN_field_prioritize, - segment, - config, - timestamp_info, - db, - cs_cache, - main_cache=cache) - for segment in segments: - if segment in segments_enabled_list: - _bulk_fetch_formula_recs(time_context, - child_descendants, - CS_derived_FN_fields, - segment, - config, - timestamp_info, - db, - cs_cache, - main_cache=cache) - - for segment in segments: - descendants_list_local = descendants_by_segments[segment] - if special_cs_fields: - _bulk_fetch_child_sum_rec_special(time_context, - descendants_list_local, - special_cs_fields, - segment, - config, - timestamp_info, - db, - cache, - round_val=round_val) - for segment in segments: - descendants_list_local = descendants_by_segments[segment] - cache.update(_bulk_fetch_child_sum_rec(time_context, - child_descendants, - fields_by_type['CS'], - segment, - config, - timestamp_info, - db, - cs_cache, - round_val=round_val, - main_cache=cache)) - - cache.update(_bulk_fetch_child_sum_rec(time_context, - descendants_list_local, - fields_by_type['CS'], - segment, - config, - timestamp_info, - db, - cs_cache, - round_val=round_val, - main_cache=cache)) - if len(config.segments) > 1: - by_val = "system" - latest_timestamp = 0 - period, comp_periods = time_context.fm_period, time_context.submission_component_periods - cs_field_with_fn_source=[] - for field in fields_by_type['CS']: - source_field = config.fields[field]['source'][0] - if config.fields[source_field]['type'] == 'FN': - cs_field_with_fn_source.append(field) - cs_field_without_fn_source = [x for x in fields_by_type['CS'] if x not in cs_field_with_fn_source] - for descendants, field in product(descendants_list, cs_field_without_fn_source): - node, children, grandchildren = descendants - val = 0 - for segment in segments: - if segment != "all_deals": - ch_key = (period, node, field, segment, timestamp) - if segment in rollup_segments: - val += try_float(get_nested(cache, [ch_key, 'val'], 0)) - seg_timestamp = get_nested(cache, [ch_key, 'timestamp'], 0) - if seg_timestamp > latest_timestamp: - latest_timestamp = seg_timestamp - by_val = get_nested(cache, [ch_key, 'by'], "system") - if not config.is_special_pivot_segmented(node): - ch_key = (period, node, field, 'all_deals', timestamp) - val = try_float(get_nested(cache, [ch_key, 'val'], 0)) - if config.forecast_service_editable_fields: - source_fields = get_all_source_fields(config, field) - if any(item in config.forecast_service_editable_fields for item in source_fields): - ch_key = (period, node, field, 'all_deals', timestamp) - val = try_float(get_nested(cache, [ch_key, 'val'], 0)) - res = {'period': period, - 'segment': config.primary_segment, - 'val': val, - 'by': by_val, - 'how': 'sum_of_children', - 'found': True if val else False, - 'timestamp': latest_timestamp, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, config.primary_segment, timestamp)] = res - - all_fields = set([]) - - for fn_field in FN_field_prioritize: - if 'FN' in fields_by_type and fn_field not in fields_by_type['FN']: - if isinstance(fields_by_type['FN'], set): - fields_by_type['FN'].add(fn_field) - elif isinstance(fields_by_type['FN'], list): - fields_by_type['FN'].append(fn_field) - - for _, type_fields in fields_by_type.items(): - all_fields.update(type_fields) - - - time_context_list = time_context_list if time_context_list else [time_context] - for _time_context in time_context_list: - _period = _time_context.deal_period if not isinstance(_time_context.deal_period, list) else _time_context.fm_period - _dlf_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(_period), 0)) - _deal_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(_period), 0)) - _timestamp_info = (timestamp, _dlf_ts, _deal_ts, None) - populate_FN_segments(_time_context, - all_fields, - config.FN_segments, - descendants_by_segments, - cache, - timestamp, - [], - config, - fields_by_type, - _timestamp_info) - - return cache - -def populate_FN_segments(time_context, - all_fields, - FN_segments, - descendants_by_segments, - cache, - timestamp, - segs_populated, - config, - fields_by_type, - timestamp_info): - missing_fields = set() - for key in cache.keys(): - if isinstance(key, tuple) and len(key) == 5: - field = key[2] - if field not in all_fields: - missing_fields.add(field) - - if missing_fields: - if isinstance(all_fields, set): - all_fields.update(missing_fields) - elif isinstance(all_fields, list): - all_fields.extend(missing_fields) - - for field_key in set(all_fields): - if config.fields.get(field_key, {}).get('ignore_func_segment', False): - all_fields.remove(field_key) - - for segment, segment_func in FN_segments.items(): - if segment in segs_populated: - continue - descendants_list = descendants_by_segments.get(segment, []) - period = time_context.fm_period - for descendants, field in product(descendants_list, all_fields): - node, _, _ = descendants - source = [] - for source_seg in segment_func.get('source'): - if source_seg in FN_segments: - populate_FN_segments(time_context, - all_fields, - {source_seg: FN_segments[source_seg]}, - descendants_by_segments, - cache, - timestamp, - segs_populated, - config, - fields_by_type, - timestamp_info) - source.append(cache.get((period, node, field, source_seg, timestamp), {}).get('val', 0)) - try: - val = eval(segment_func.get('func')) - except ZeroDivisionError: - val = 0 - except Exception: - val = 0 - res = {'period': period, - 'segment': segment, - 'val': val, - 'by': "system", - 'how': 'derived_segment', - 'found': True if val else False, - 'timestamp': timestamp, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - - ## Need to re-calculate some FN fields as values may change after FN segment operations based on config - if 'FN' in fields_by_type: - recalc_fields = [] - for field in fields_by_type['FN']: - is_recalc = config.fields[field].get('recalc', False) - if is_recalc: - recalc_fields.append(field) - _bulk_fetch_formula_recs(time_context, - descendants_list, - recalc_fields, - segment, - config, - timestamp_info, - sec_context.tenant_db, - cache) - segs_populated.append(segment) - -def add_FN_dependent_segments(segments, config, segs_populated): - try: - FN_segments = config.FN_segments - for segment in segments: - if segment in segs_populated: - continue - if segment in FN_segments: - for source_seg in FN_segments[segment].get('source'): - if source_seg not in segments: - segs_populated.append(source_seg) - if source_seg in FN_segments: - segs_populated += add_FN_dependent_segments(segs_populated, config, []) - return list(set(segs_populated)) - except Exception as e: - logger.error("Error while fetching source segments, error is %s", str(e)) - return [] - - - -def handle_dr_fields(config, - dr_fields, - time_context, - descendants, - segments, - timestamp_info, - db, - cache, - round_val, - timestamp, - eligible_nodes_for_segs, - dr_from_fm_coll_for_snapshot=None, - found=None, - call_from=None, - dlf_ts=None, - deal_ts=None, - trend=False, - time_context_list = [] - ): - # if segment_field is defined or tenant does not have segments use bulk fetch for dr records - # for tenant like honeywell we have segment field named pulsesegment so we can group by segment. - try: - dr_from_fm_coll_for_snapshot = dr_from_fm_coll_for_snapshot if dr_from_fm_coll_for_snapshot is not None else sec_context.details.get_flag('deal_rollup', 'snapshot_dr_from_fm_coll', False) - except: - dr_from_fm_coll_for_snapshot = False - try: - found = found if found is not None else False - except: - found = False - - if config.deal_config.segment_field or not config.has_segments or dr_from_fm_coll_for_snapshot: - dlf_fields = [] - fm_fields_ = [] - instant_dlf_update_fields = [] - fm_dlf_instant_update_fields = config.config.get('fm_dlf_instant_update_fields', []) - for field in dr_fields: - if field in fm_dlf_instant_update_fields: - instant_dlf_update_fields.append(field) - elif config.deal_rollup_fields[field].get('dlf') or any(filt.get('op') == 'dlf' if isinstance(filt, dict) else \ - False for filt in - config.deal_rollup_fields[field]['filter']): - dlf_fields.append(field) - else: - fm_fields_.append(field) - - if dlf_fields: - _bulk_fetch_deal_rollup_recs(time_context, - descendants, - dlf_fields, - segments, - config, - timestamp_info, - db, - cache, - round_val=round_val, - get_all_segments=True, - is_dr_deal_fields=False, - dr_from_fm_coll_for_snapshot=dr_from_fm_coll_for_snapshot, - found=found, - eligible_nodes_for_segs=eligible_nodes_for_segs, - call_from=call_from, - trend=trend - ) - - if fm_fields_: - _bulk_fetch_deal_rollup_recs(time_context, - descendants, - fm_fields_, - segments, - config, - timestamp_info, - db, - cache, - round_val=round_val, - get_all_segments=True, - is_dr_deal_fields=True, - dr_from_fm_coll_for_snapshot=dr_from_fm_coll_for_snapshot, - found=found, - eligible_nodes_for_segs=eligible_nodes_for_segs, - call_from=call_from, - trend=trend - ) - - if instant_dlf_update_fields: - fetch_fm_recs_history(time_context, - descendants, - instant_dlf_update_fields, - segments, - config, - [timestamp], - db=db, - cache=cache, - eligible_nodes_for_segs=eligible_nodes_for_segs, - call_from=call_from, - dlf_ts=dlf_ts, - deal_ts=deal_ts, - time_context_list = time_context_list - ) - # if segment_field is not defined for tenants having segments get dr records one by one for each segment. - # for tenant like github we don't have a unique way to identify segment in deals info so, - # retrieving data one by one for each segment. - elif not config.deal_config.segment_field and config.has_segments: - fetch_fm_recs_history(time_context, - descendants, - dr_fields, - segments, - config, - [timestamp], - db=db, - cache=cache, - eligible_nodes_for_segs=eligible_nodes_for_segs, - call_from=call_from, - time_context_list = time_context_list - ) - - -def fetch_fm_recs(time_context, - descendants_list, - fields, - segments, - config, - timestamp=None, - recency_window=None, - db=None, - eligible_nodes_for_segs={} - ): - """ - fetch fm records from db for multiple nodes/fields - you are 100% guaranteed to get records for all the params you request - - Arguments: - time_context {time_context} -- fm period, components of fm period - deal period, close periods of deal period - deal expiration timestamp, - relative periods of component periods - ('2020Q2', ['2020Q2'], - '2020Q2', ['201908', '201909', '201910'], - 1556074024910, - ['h', 'c', 'f']) - descendants_list {list} -- list of tuples of - (node, [children], [grandchildren]) - fetches data for each node - using children/grandkids to compute sums - [('A', ['B, 'C'], ['D', 'E'])] - fields {list} -- list of field names - ['commit', 'best_case'] - segments {list} -- list of segment names - ['all_deals', 'new', 'upsell'] - config {FMConfig} -- instance of FMConfig - - Keyword Arguments: - timestamp {int} -- epoch timestamp to get data as of (default: {None}) - if None, gets most recent record - 1556074024910 - recency_window {int} -- window to reject data from before timestamp - recency_window (default: {None}) - if None, will accept any record, regardless of how stale - ONE_DAY - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - dict -- {(period, node, field, segment, timestamp): {'period': period, - 'val': val, - 'by': by, - 'how': how, - 'found': found, - 'timestamp': timestamp, - 'node': node, - 'field': field}} - """ - threads, cache = [], {} - db = db if db else sec_context.tenant_db - - if timestamp: - dlf_ts, deal_ts = 0, 0 - else: - period = time_context.deal_period if not isinstance(time_context.deal_period, list) else time_context.fm_period - dlf_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - - if any([segment in config.FN_segments for segment in segments]): - segments += add_FN_dependent_segments(segments, config, []) - segments = list(set(segments)) - - for descendants, field, segment in product(descendants_list, fields, segments): - fetch_fm_rec(time_context, - descendants, - field, - segment, - config, - (timestamp, dlf_ts, deal_ts, timestamp - recency_window if recency_window else None), - db, - cache) - - if config.FN_segments: - descendants_by_segments = {} - for segment in segments: - descendants_list = [] - for node in descendants: - if node in eligible_nodes_for_segs.get(segment, []) or segment == 'all_deals': - descendants_list.append((node, {}, {})) - descendants_by_segments[segment] = descendants_list - all_fields = copy.deepcopy(fields) - timestamp_info = (timestamp, dlf_ts, deal_ts, None) - for field_key in set(all_fields): - if config.fields.get(field_key, {}).get('ignore_func_segment', False): - all_fields.remove(field_key) - fields_by_type, _ = get_fields_by_type(fields, config, time_context) - populate_FN_segments(time_context, - all_fields, - config.FN_segments, - descendants_by_segments, - cache, - timestamp, - [], - config, - fields_by_type, - timestamp_info) - return cache - - -def fetch_fm_recs_history(time_context, - descendants_list, - fields, - segments, - config, - timestamps, - recency_window=None, - db=None, - cache={}, - ignore_recency_for=[], - eligible_nodes_for_segs={}, - call_from=None, - dlf_ts=None, - deal_ts=None, - time_context_list = [] - ): - """ - fetch history of fm records for multiple dbs/fields - you are 100% guaranteed to get records for all the params you request - - Arguments: - time_context {time_context} -- fm period, components of fm period - deal period, close periods of deal period - deal expiration timestamp, - relative periods of component periods - ('2020Q2', ['2020Q2'], - '2020Q2', ['201908', '201909', '201910'], - 1556074024910, - ['h', 'c', 'f']) - descendants_list {list} -- list of tuples of - (node, [children], [grandchildren]) - fetches data for each node - using children/grandkids to compute sums - [('A', ['B, 'C'], ['D', 'E'])] - fields {list} -- list of field names - ['commit', 'best_case'] - segments {list} -- list of segment names - ['all_deals', 'new', 'upsell'] - config {FMConfig} -- instance of FMConfig - timestamps {list} -- list of epoch timestamps to get data as of - [1556074024910, 1556075853732, ...] - - Keyword Arguments: - recency_window {int} -- window to reject data from before timestamp - recency_window (default: {None}) - if None, will accept any record, regardless of how stale - ONE_DAY - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - dict -- {(period, node, field, segment, timestamp): {'period': period, - 'val': val, - 'by': by, - 'how': how, - 'found': found, - 'timestamp': timestamp, - 'node': node, - 'field': field}} - """ - # WARN: performance for this might suck, is threading the right call? - original_recency_window = recency_window - if cache is None: - cache = {} - db = db if db else sec_context.tenant_db - period = time_context.deal_period if not isinstance(time_context.deal_period, list) else time_context.fm_period - dlf_ts = dlf_ts if dlf_ts is not None else try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = deal_ts if deal_ts is not None else try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - try: - t = sec_context.details - year_accepted_from = int(t.get_flag('fm_latest_migration', 'year', 0)) - except: - year_accepted_from = 0 - - if not eligible_nodes_for_segs: - for segment in config.segments: - if segment == config.primary_segment: - continue - eligible_nodes_for_segs[segment] = [dd['node'] for dd in fetch_eligible_nodes_for_segment(time_context.now_timestamp, segment)] - - if config.debug: - for timestamp, field, segment, descendants in product(timestamps, fields, segments, descendants_list): - if field in ignore_recency_for: - recency_window = None - else: - recency_window = original_recency_window - if segment == 'all_deals' or descendants[0] in eligible_nodes_for_segs.get(segment, []): - fetch_fm_rec(time_context, - descendants, - field, - segment, - config, - (timestamp, dlf_ts, deal_ts, timestamp - recency_window if recency_window and timestamp else None), - db=db, - cache=cache, - call_from=call_from) - else: - count = 0 - pool = ThreadPool(MAX_THREADS) - fm_rec_params = [] - threads = 0 - user = sec_context.login_user_name - time_context_list = time_context_list if time_context_list else [time_context] - - for timestamp, field, segment, descendants, _time_context in product(timestamps, fields, segments, descendants_list, time_context_list): - # TODO: bug here ... with deal and dlf expirations not reflecting correct value to dashboard - if field in ignore_recency_for: - recency_window = None - else: - recency_window = original_recency_window - count = count + 1 - _period = _time_context.deal_period if not isinstance(_time_context.deal_period, list) else _time_context.fm_period - _dlf_ts = dlf_ts if dlf_ts is not None else try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(_period), 0)) - _deal_ts = deal_ts if deal_ts is not None else try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(_period), 0)) - - if segment == 'all_deals' or descendants[0] in eligible_nodes_for_segs.get(segment,[]): - fm_rec_params.append((_time_context, - descendants, - field, - segment, - config, - (timestamp, _dlf_ts, _deal_ts, timestamp - recency_window if recency_window and timestamp else None), - db, - cache, - None, - call_from, - year_accepted_from, - user)) - threads += 1 - - pool.map(_fetch_fm_rec, tuple(fm_rec_params)) - pool.close() - pool.join() - - # AV-14059 Commenting below as part of the log fix - # logger.info('actual comibnations: %s, threads executed count : %s', count, threads) - - return cache - - -def bulk_fetch_fm_recs_history(time_context, - descendants_list, - fields, - segments, - config, - timestamps, - recency_window=None, - db=None, - cache={}, - ignore_recency_for=[], - eligible_nodes_for_segs={}, - fields_by_type={}, - get_all_segments=True - ): - - try: - # WARN: performance for this might suck, is threading the right call? - original_recency_window = recency_window - if cache is None: - cache = {} - fm_recs = {} - db = db if db else sec_context.tenant_db - period = time_context.deal_period if not isinstance(time_context.deal_period, list) else time_context.fm_period - dlf_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_dlf_expiration_timestamp'.format(period), 0)) - deal_ts = try_float(sec_context.details.get_flag('deal_svc', '{}_deal_expiration_timestamp'.format(period), 0)) - if not eligible_nodes_for_segs: - for segment in config.segments: - if segment == config.primary_segment: - continue - eligible_nodes_for_segs[segment] = [dd['node'] for dd in - fetch_eligible_nodes_for_segment(time_context.now_timestamp, segment)] - if any([segment in config.FN_segments for segment in segments]): - segments += add_FN_dependent_segments(segments, config, []) - segments = list(set(segments)) - - threads = 0 - count = 0 - pool = ThreadPool(MAX_THREADS) - try: - dr_from_fm_coll_for_snapshot = sec_context.details.get_flag('deal_rollup', 'snapshot_dr_from_fm_coll', False) - except: - dr_from_fm_coll_for_snapshot = False - for field_type, fields in fields_by_type.items(): - final_timestamps = {} - if field_type == 'UE' or field_type == 'AI' or field_type == 'DR' or field_type == 'prnt_DR': - for timestamp, field in product(timestamps, fields): - if field in ignore_recency_for: - recency_window = None - else: - recency_window = original_recency_window - final_timestamp = ( - timestamp, dlf_ts, deal_ts, timestamp - recency_window if recency_window and timestamp else None) - - if final_timestamp not in final_timestamps: - final_timestamps[final_timestamp] = [field] - if field not in final_timestamps[final_timestamp]: - final_timestamps[final_timestamp].append(field) - - if field_type == 'UE': - fm_rec_params = [] - all_segments = config.segments - all_segments = [segment for segment in all_segments] - rollup_segments = [segment for segment in config.rollup_segments] - if 'all_deals' not in rollup_segments: - rollup_segments.append('all_deals') - node_children_check = {node: any(node_children) for node, node_children, _ in descendants_list} - nodes = node_children_check.keys() - nodes_and_eligible_segs_len = {} - if config.partially_segmented: - for node in nodes: - nodes_and_eligible_segs_len[node] = len(config.get_segments(epoch().as_epoch(), node)) - if 'newrelic' in sec_context.login_tenant_name: - #temporarily added for newewlic - try: - for final_timestamp, fields in final_timestamps.iteritems(): - fm_recs = _bulk_fetch_user_entered_recs(time_context, - descendants_list, - fields, - all_segments, - config, - final_timestamp, - db, - cache, - True, - True, - True, - nodes_and_eligible_segs_len) - except Exception as e: - logger.error(e) - raise e - else: - for final_timestamp, fields in final_timestamps.iteritems(): - count = count + 1 - # TODO: bug here ... with deal and dlf expirations not reflecting correct value to dashboard - fm_rec_params.append((time_context, - descendants_list, - fields, - rollup_segments if (len(segments) == 1 \ - and segments[0] == 'all_deals') else segments, - config, - final_timestamp, - db, - cache, - True, - True, - True, - nodes_and_eligible_segs_len)) - threads += 1 - pool.map(_bulk_fetch_user_entered_recs_pool, tuple(fm_rec_params)) - elif field_type == 'AI': - fm_rec_params = [] - bucket_fields = FMConfig().config.get('bucket_fields', []) - if any([field in bucket_fields for field in fields]): - bucket_forecast_field = FMConfig().config.get('bucket_forecast_field') if FMConfig().config.get( - 'bucket_forecast_field') else [] - else: - bucket_fields = [] - bucket_forecast_field = [] - - - for final_timestamp, fields in final_timestamps.iteritems(): - count = count + 1 - # TODO: bug here ... with deal and dlf expirations not reflecting correct value to dashboard - fm_rec_params.append((time_context, - descendants_list, - fields, - segments, - config, - final_timestamp, - db, - cache, - get_all_segments, - True, - bucket_fields, - bucket_forecast_field)) - threads += 1 - - pool.map(_bulk_fetch_ai_recs_pool, tuple(fm_rec_params)) - elif field_type == 'DR': - try: - dr_from_fm_coll_for_snapshot = sec_context.details.get_flag('deal_rollup', 'snapshot_dr_from_fm_coll', False) - except: - dr_from_fm_coll_for_snapshot = False - fm_rec_params = [] - fm_params = [] - config.debug = True - if config.deal_config.segment_field or not config.has_segments or dr_from_fm_coll_for_snapshot: - for final_timestamp, fields in final_timestamps.iteritems(): - count = count + 1 - found = True # passed directly as the last arg - fm_rec_params.append((config, - fields, - time_context, - descendants_list, - segments, - final_timestamp, - db, - cache, - True, - final_timestamp[0], - eligible_nodes_for_segs, - True, - True, - None, - dlf_ts, - deal_ts, - True)) - threads += 1 - pool.map(handle_dr_fields_pool, tuple(fm_rec_params)) - elif not config.deal_config.segment_field and config.has_segments: - config.debug = True - for timestamp, field, segment, descendants in product(timestamps, fields, segments, descendants_list): - # TODO: bug here ... with deal and dlf expirations not reflecting correct value to dashboard - if field in ignore_recency_for: - recency_window = None - else: - recency_window = original_recency_window - count = count + 1 - if segment == 'all_deals' or descendants[0] in eligible_nodes_for_segs.get(segment, []): - fm_params.append((time_context, - descendants, - field, - segment, - config, - (timestamp, dlf_ts, deal_ts, - timestamp - recency_window if recency_window and timestamp else None), - db, - cache)) - threads += 1 - pool.map(_fetch_fm_rec, tuple(fm_params)) - elif field_type == 'prnt_DR': - fm_rec_params = [] - prev_periods = prev_periods_allowed_in_deals() - for final_timestamp, fields in final_timestamps.iteritems(): - count = count + 1 - fm_rec_params.append((config, - fields, - time_context, - descendants_list, - rollup_segments if (len(segments) == 1 \ - and segments[0] == 'all_deals') else segments, - final_timestamp, - db, - cache, - final_timestamp[0], - eligible_nodes_for_segs, - True, - True, - None, - prev_periods)) - threads += 1 - pool.map(_bulk_fetch_prnt_dr_recs_pool, tuple(fm_rec_params)) - else: - fm_rec_params = [] - for timestamp, field, segment, descendants in product(timestamps, fields, segments, descendants_list): - # TODO: bug here ... with deal and dlf expirations not reflecting correct value to dashboard - if field in ignore_recency_for: - recency_window = None - else: - recency_window = original_recency_window - count = count + 1 - if segment == 'all_deals' or descendants[0] in eligible_nodes_for_segs.get(segment, []): - fm_rec_params.append((time_context, - descendants, - field, - segment, - config, - (timestamp, dlf_ts, deal_ts, - timestamp - recency_window if recency_window and timestamp else None), - db, - cache)) - threads += 1 - pool.map(_fetch_fm_rec, tuple(fm_rec_params)) - pool.close() - pool.join() - - all_fields = set([]) - for _, type_fields in fields_by_type.items(): - all_fields.update(type_fields) - descendants_by_segments = {} - for segment in segments: - descendants_by_segments[segment] = descendants_list - - for timestamp in timestamps: - populate_FN_segments(time_context, - all_fields, - config.FN_segments, - descendants_by_segments, - cache, - timestamp, - [], - config, - fields_by_type, - timestamp_info=(timestamp, dlf_ts, deal_ts, None) - ) - - logger.info('actual comibnations: %s, threads executed count : %s', count, threads) - return cache - except Exception as e: - logger.exception(e) - raise e - - -def fetch_fm_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info=None, - db=None, - cache=None, - prnt_node=None, - call_from=None, - year_accepted_from=None, - user=None - ): - """ - fetch a single fm record for a node/field - - Arguments: - time_context {time_context} -- fm period, components of fm period - deal period, close periods of deal period - deal expiration timestamp, - relative periods of component periods - ('2020Q2', ['2020Q2'], - '2020Q2', ['201908', '201909', '201910'], - 1556074024910, - ['h', 'c', 'f']) - descendants {tuple} -- (node, [children], [grandchildren]) - fetches data for node - using children/grandkids to compute sums - ('A', ['B, 'C'], ['D', 'E']) - field {str} -- field name - 'commit' - segment {str} -- segment name - 'all deals' - config {FMConfig} -- instance of FMConfig - - Keyword Arguments: - timestamp_info {tuple} -- (ts to get data as of, - ts dlf DR recs expired as of, - ts deal DR recs expired as of, - ts to reject data from before (None to accept any level of staleness)) - if None passed, gets most recent record - (default: {None}) - (1556074024910, 1556074024910, 1556074024910) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - (pass in to avoid overhead when fetching many times) - cache {dict} -- dict to hold records fetched by (default: {None}) - (used to memoize fetching many recs) - prnt_node {str} -- parent of the particular node. Value will same as node when it's already the - parent. prnt_node is required for prnt_DR fields - - Returns: - dict -- a single fm record - """ - node, _, _ = descendants - period = time_context.fm_period - if not timestamp_info: - timestamp_info = (None, 0, 0, None) - if not db: - db = sec_context.tenant_db - timestamp, _, _, _ = timestamp_info - - # TODO: dont love this implementation... - # should PC be defined in config, instead of down here - # that would be cleaner, but make the code slower probably - if ('hist_field' in config.fields[field] - and 'h' in time_context.relative_periods and 'overwrite_field' not in config.fields[field]): - # historic period, switch to using true up field if it exists - fetch_type = 'PC' - if config.quarter_editable: - #If quarter editable, use PC type only when the quarter is a previous quarter - if not all(elem == 'h' for elem in time_context.relative_periods): - fetch_type = config.fields[field]['type'] - else: - fetch_type = config.fields[field]['type'] - - - if cache: - try: - return cache[(period, node, field, segment, timestamp)] - except KeyError: - pass - - fetch_map = { - 'UE': _fetch_user_entered_rec, - 'US': _fetch_child_sum_rec, # TODO: deprecate this option - 'DR': _fetch_deal_rollup_rec, - 'FN': _fetch_formula_rec, - 'AI': _fetch_ai_rec, - 'NC': _fetch_node_conditional_rec, - 'CS': _fetch_child_sum_rec, - 'PC': _fetch_period_conditional_rec, # TODO: i dont like the name for this - 'prnt_DR': fetch_deal_rollup_for_prnt_dr_rec #Not in use here because of if condition below, added for readability - } - if fetch_type == 'prnt_DR': - res = fetch_deal_rollup_for_prnt_dr_rec(time_context, descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - prnt_node) - else: - if fetch_type == 'DR': - res = fetch_map[fetch_type](time_context, descendants, field, segment, config, timestamp_info, db, cache, - year_accepted_from=year_accepted_from, user=user) - else: - res = fetch_map[fetch_type](time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=year_accepted_from) - - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - - return res - - -def fetch_fm_records_from_time_range(period, - time_range, - nodes=None, - fields=None, - db=None): - """ - fetch fm records that occured within time range, and have had no updates since - - Arguments: - period {str} -- ???? uh - time_range {tuple} -- begin timestamp, end timestamp (1573440113408, 1573540213408) - - Keyword Arguments: - nodes {list} -- list of nodes to limit query to (default: {None}) - fields {list} -- list of fm fields to limit query to (default: {None}) - db {pymongo.database.Database} -- instance of tenant_db - (default: {None}) - if None, will create one - - Returns: - list -- [fm records] - """ - # TODO: bunch of logic - # like can only be component period, and can only be fields that are actually saved in db, not computed - fm_data_collection = db[FM_COLL] if db else sec_context.tenant_db[FM_COLL] - - match = {'period': period} - if fields: - match['field'] = {'$in': fields} - if nodes: - match['node'] = {'$in': nodes} - - sort = {'timestamp': 1} - - group = {'_id': {'period': '$period', - 'node': '$node', - 'field': '$field'}, - 'timestamp': {'$last': '$timestamp'}, - 'doc': {'$last': '$$ROOT'}} - - rematch = {'timestamp': {'$gte': time_range[0], '$lte': time_range[1]}} - - project = {'doc': 1, '_id': 0} - - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$match': rematch}, - {'$project': project}] - - return fm_data_collection.aggregate(pipeline, allowDiskUse=True) - - -def fetch_latest_dr_count(periods, db=None): - if not isinstance(periods, list): - periods = [periods] - if not db: - db = sec_context.tenant_db - fm_latest_data_collection = db[FM_LATEST_COLL] - return fm_latest_data_collection.count_documents({'period': {'$in': periods}}) - -def fetch_latest_deal_rollups(period, - nodes, - fields, - segments, - config, - db=None, - ): - """ - fetch latest deal rollup values from fm_latest_data_collection - - Arguments: - period {str} -- component period mnemonic (month if monthly fm, quarter if quarterly) - '2020Q2' - nodes {list} -- list of nodes to fetch values for - ['0050000FLN2C9I2',] - fields {str} -- list of field names, must be deal rollup fields - ['commit', 'best_case'] - config {FMConfig} -- instance of FMConfig - - Keyword Arguments: - db {pymongo.database.Database} -- instance of tenant_db - (default: {None}) - if None, will create one - Returns: - dict -- {(segment, field, node): val} - """ - if not db: - db = sec_context.tenant_db - fm_latest_data_collection = db[FM_LATEST_COLL] - - match = {'period': period, - 'segment': {'$in': list(segments)}, - 'node': {'$in': list(nodes)}, - 'field': {'$in': list(fields)}} - project = {'period': 1, 'segment': 1, 'node': 1, 'field': 1, 'val': 1} - pipeline = [{'$match': match}, - {'$project': project}] - - if config.debug: - logger.info("pipeline for latest deal rollups is %s " % pipeline) - - return {(rec['segment'], rec['field'], rec['node']): rec['val'] - for rec in fm_latest_data_collection.aggregate(pipeline, allowDiskUse=True)} - - -def fetch_throwback_totals(node, segment, save_history, db=None): - """ - get the historical throwback data computed in insight task - - Arguments: - week_num {int} -- number of week - - Returns: - dict -- proportion of each week wins. Values sum up to 1 - """ - fi_collection = db['forecast_explanation_insights'] if db else sec_context.tenant_db['forecast_explanation_insights'] - if save_history: - timestamp = list(fi_collection.aggregate([{ "$sort" : { "timestamp" : -1 } }, { "$project": {"timeMax": {"$max": "$timestamp"}} }]))[0]["timeMax"] - else: - timestamp = 0 - criteria = {'$and':[{'timestamp': timestamp, - 'node': node, - 'segment': segment}]} - return list(fi_collection.find(criteria)) - - -def _fetch_user_entered_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=None - ): - """ - fetch record that came from user entering data in aviso - """ - # NOTE: much of this logic is duplicated in _bulk_fetch_user_entered_recs - # if you are changing logic here, be sure to also change in the bulk version of the function - node, children, _ = descendants - segments = config.segment_map[segment] - # below code is commented as this is creating CS-21073 issue. 'segment_rep' looks legacy code which is not present anymore - # if issue arises, try removing the code, but handle CS-21073 as well. - if 'consider_only_segment_value' in config.fields[field]: #or ((not children and segment == config.primary_segment) and 'segment_rep' not in config.fields[field]): - segments = [segment] - if config.primary_segment == segment and config.config_based_segments_rollup: - segments = config.rollup_segments - period, comp_periods = time_context.fm_period, time_context.submission_component_periods - timestamp, _, _, cutoff_timestamp = timestamp_info - how = 'sum_of_users' if comp_periods[0] != period or len(segments) != 1 else 'fm_upload' - found = True - FM_COLL = read_from_collection(period[:4], timestamp=timestamp, call_from='_fetch_user_entered_rec', config=config, - year_accepted_from=year_accepted_from) - fm_data_collection = db[FM_COLL] - - if field in config.quarterly_high_low_fields and "Q" in period: - comp_periods = [period] - - match = {'period': {'$in': comp_periods}, - 'segment': {'$in': segments}, - 'node': node, - 'field': field} - if timestamp is not None: - match['timestamp'] = {'$lte': timestamp} - if len(comp_periods) == 1 and cutoff_timestamp is not None: - # BUG: can only support cutoff_timestamp for non aggregated :( - match['timestamp']['$gt'] = cutoff_timestamp - sort = {'timestamp': 1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'segment': '$segment', - 'field': '$field'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$last': '$val'}} - regroup = {'_id': None, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$sum': '$val'}} - if (segment == config.primary_segment and len(config.segment_map[config.primary_segment]) > 1) or (len(comp_periods)>1): - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$sort': sort}, - {'$group': regroup}] - else: - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$group': regroup}] - - if FM_COLL == FM_LATEST_DATA_COLL: - regroup = {'_id': None, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$sum': '$val'}} - pipeline = [{'$match': match}, - {'$group': regroup}] - - - if config.debug: - logger.info('fetch_user_entered_rec pipeline: %s reading from %s', (pipeline, FM_COLL)) - - try: - aggs = list(fm_data_collection.aggregate(pipeline, allowDiskUse=True)) - val, by, ts = aggs[0]['val'], aggs[0]['by'], aggs[0]['timestamp'] - except IndexError: - val, by, ts, found = 0, 'system', 0, False - - overwritten_val = overwrite_fields(config, field, comp_periods, period, node, timecontext=time_context) - if overwritten_val is not None: - val = overwritten_val - - return {'period': period, - 'segment': segment, - 'val': val, - 'by': by, - 'how': how, - 'found': found, - 'timestamp': ts, - 'node': node, - 'field': field} - - -def _fetch_deal_rollup_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - call_from=None, - year_accepted_from=None, - user=None, - period_and_close_periods=None - ): - """ - fetch record that is sum of deal amount field for deals matching a filter - """ - # NOTE: much of this logic is duplicated in _bulk_fetch_deal_rollup_recs - # if you are changing logic here, be sure to also change in the bulk version of the function - node, _, _ = descendants - segments = [segment] - current_segment_dtls = config.segments[segment] - period, comp_periods = time_context.fm_period, time_context.component_periods - timestamp, dlf_expiration, deal_expiration, _ = timestamp_info - FM_COLL = read_from_collection(period[:4], timestamp=timestamp, field_type='DR', call_from='_fetch_deal_rollup_rec', - config=config, year_accepted_from=year_accepted_from) - fm_data_collection = db[FM_COLL] - expiration_timestamp = dlf_expiration if config.fields[field].get('dlf') else deal_expiration - if timestamp and timestamp < expiration_timestamp: - query_expiration_timestamp = 0 - else: - query_expiration_timestamp = expiration_timestamp - found = True - # BUG: when rollup updates some but not all of the component periods totals for a node/field combo - # the updated records will pass the timestamp criteria, but the others will not - # so we only sum for the ones that got updated, missing the older values - # this crept up when we stopped saving duping records in Decemberish - val = None - config_based_segments_rollup = config.config_based_segments_rollup - use_all_deals_for_past_data_rollup = config.use_all_deals_for_past_data_rollup - is_pivot_special = node and node.split('#')[0] in config.deal_config.not_deals_tenant.get('special_pivot', []) - - segments_sum_field = [] - if segment == 'all_deals': - segments = [] - # Checking if segments are present or not - try: - as_of = timestamp if timestamp else get_period_as_of(period) - if timestamp and use_all_deals_for_past_data_rollup: - segments = [segment] - else: - for x in config.rollup_segments: - if node_aligns_to_segment(as_of, node, x, config) and x not in segments: - segments.append(x) - except: - segments = config.rollup_segments - - # for checking if segments have same sum_field or different for rollup. - for seg in segments: - sum_field = config.segment_amount_fields.get(seg, config.fields[field]['sum_field']) - if sum_field not in segments_sum_field: - segments_sum_field.append(sum_field) - - is_dlf_field = config.deal_rollup_fields[field].get('dlf', False) - field_filters = config.deal_rollup_fields[field]['filter'] - if not is_dlf_field: - for filt in field_filters: - is_dlf_field = is_dlf_field or (filt.get('op') == 'dlf' if isinstance(filt, dict) else False) - - if is_dlf_field and call_from == 'UploadStatus' and field in config.fm_status_fields: - is_dlf_field = False - # If sum_field_override is been used in the segment config then we are overrding the field and fetching it directly from deals - ts = None - if 'sum_field_override' in current_segment_dtls and field in current_segment_dtls['sum_field_override'] \ - and not (timestamp and not is_same_day(EpochClass(), epoch(timestamp))): - val = None - elif segment == 'all_deals' and len(segments_sum_field) > 1 and \ - not config_based_segments_rollup and \ - not config.config.get("segments", {}).get("rollup_segments", {}) and \ - (not timestamp or (timestamp and is_same_day(EpochClass(), epoch(timestamp)))): - val = None - elif (timestamp and not is_same_day(EpochClass(), epoch(timestamp))) or (len(period) == 4 and \ - config.read_from_fm_collection_for_year): - if timestamp and not is_same_day(EpochClass(), epoch(timestamp)) \ - and len(segments) > 1 and segment == 'all_deals': - segments = [seg for seg in segments if seg != 'all_deals'] if not is_pivot_special else segments - if config.fields[field].get('is_cumulative', False) and len(comp_periods) > 1: - comp_periods = [max(comp_periods)] - match = {'period': {'$in': comp_periods}, - 'node': node, - 'segment': {'$in': segments}, - 'field': field} - if timestamp is not None: - # for view past forecast, when query timestamp is end of quarter and we check for end quarter \ - # we don't need to pass greater than condition - if 'Q' not in period: - end_timestamp = get_eom(epoch(timestamp)) - else: - period_info = time_context.period_info - end_timestamp = get_eoq_for_period(period, period_info) - if query_expiration_timestamp and str(epoch(int(query_expiration_timestamp)))[:19] == str(end_timestamp)[:19]: - match['timestamp'] = {} - if 'timestamp' not in match: - match['timestamp'] = {} - match['timestamp'].update({'$lte': timestamp}) - sort = {'timestamp': 1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'segment': '$segment', - 'field': '$field'}, - 'val': {'$last': '$val'}, - 'timestamp': {'$last': '$timestamp'}} - regroup = {'_id': None, - 'val': {'$sum': '$val'}, - 'timestamp': {'$last': '$timestamp'}} - if call_from == "UploadStatus" and is_pivot_special: - additional_fields = {'by': {'$last': '$by'}, 'how': {'$last': '$how'}} - group.update(additional_fields) - regroup.update(additional_fields) - - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$group': regroup}] - if FM_COLL == FM_LATEST_COLL: - regroup = {'_id': None, - 'val': {'$sum': '$val'}, - 'timestamp': {'$last': '$timestamp'}} - pipeline = [{'$match': match}, - {'$group': regroup}] - - if config.debug: - logger.info('fetch_deal_rollup_rec pipeline: %s from %s' % (pipeline, FM_COLL)) - - try: - aggs = list(fm_data_collection.aggregate(pipeline, allowDiskUse=True)) - val = aggs[0]['val'] - ts = aggs[0]['timestamp'] - if call_from == "UploadStatus" and is_pivot_special: - by = aggs[0]['by'] - how = aggs[0]['how'] - except IndexError: - # make value 0 to avoid reading from deals collection in case of past date. - if timestamp and not is_same_day(EpochClass(), epoch(timestamp)): - val = 0 - else: - val = None - - if val is None: - sum_field = config.segment_amount_fields.get(segment, config.fields[field]['sum_field']) - seg_filter = config.segment_filters.get(segment) - name, crit, ops = config.fields[field]['crit_and_ops'] - is_cumulative_field = config.fields[field].get('is_cumulative', False) - close_periods = time_context.close_periods if not is_cumulative_field else time_context.cumulative_close_periods - crit = {'$and': [crit, seg_filter]} - if segment and segment != 'all_deals' and sum_field != config.fields[field]['sum_field']: - ops = [(sum_field, sum_field, '$sum')] - # Overriding the segment field with the overrided field in config - try: - if len(period) == 4: - period_and_close_periods = time_context.period_and_close_periods_for_year or \ - get_period_and_close_periods(period, False, deal_svc=True) - else: - period_and_close_periods = [(time_context.deal_period, time_context.close_periods)] - except Exception as e: - logger.exception(e) - if 'sum_field_override' in current_segment_dtls and field in current_segment_dtls['sum_field_override']: - if segment != "all_deals": - ovveride_field = current_segment_dtls['sum_field_override'][field] - name, crit, ops = config.fields[ovveride_field]['crit_and_ops'] - sum_field = config.fields[ovveride_field]['sum_field'] - crit = {'$and': [crit, seg_filter]} - else: - total_value = 0 - # If sum_field_override in all_deals segment, then we are overriding the field with sum of all the segment values for that field. - for seg,segment_dtls in config.segments.items(): - if seg != "all_deals": - seg_filter = config.segment_filters.get(seg) - ovveride_field = segment_dtls['sum_field_override'][field] if 'sum_field_override' in segment_dtls else field - name, crit, ops = config.fields[ovveride_field]['crit_and_ops'] - sum_field = config.fields[ovveride_field]['sum_field'] if 'sum_field_override' in segment_dtls else segment_dtls['sum_field'] - crit = {'$and': [crit, seg_filter]} - resp = fetch_deal_totals(period_and_close_periods, - node, - ops, - config.deal_config, - filter_criteria=crit, - timestamp=time_context.deal_timestamp, - db=db, - cache=cache, - prev_periods=time_context.prev_periods, - user=user, - sfdc_view=True if len(period) == 4 else False, - actual_view=True if len(period)== 4 else False - ) - val = resp.get(sum_field, 0) - if (ts and ts < resp.get("timestamp", 0)) or not ts: - ts = resp.get("timestamp") - total_value += len(val) if isinstance(val,list) else val - res = {'period': period, - 'segment': segment, - 'val': total_value, - 'by': 'system', - 'how': 'deal_rollup', - 'found': found, - 'timestamp': ts, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - return res - if config.debug: - logger.info('fetch_deal_rollup_rec field: %s, deal svc crit: %s, ops: %s, ts: %s, exp_ts: %s, sum field %s', - field, crit, ops, timestamp, expiration_timestamp, sum_field) - # looking for a historic record that doesnt exist, maybe still a bug though - # TODO: this needs a unit test - - try: - if segment != "all_deals" or\ - (segment == "all_deals" and (not segments or \ - (len(segments) == 1 and segments[0] == 'all_deals') or\ - (len(segments_sum_field) > 1 and \ - not config_based_segments_rollup and \ - not config.config.get("segments", {}).get("rollup_segments", {})))): - pivot_node = node.split("#")[0] - if pivot_node == 'CRR': - resp = fetch_crr_deal_totals(period_and_close_periods, - node, - ops, - config.deal_config, - filter_criteria=crit, - timestamp=time_context.deal_timestamp, - db=db, - cache=cache - ) - else: - resp = fetch_deal_totals(period_and_close_periods, - node, - ops, - config.deal_config, - filter_criteria=crit, - timestamp=time_context.deal_timestamp, - db=db, - cache=cache, - prev_periods=time_context.prev_periods, - user=user, - sfdc_view=True if len(period) == 4 else False, - actual_view=True if len(period) == 4 else False - ) - val = resp.get(sum_field, 0) - ts = resp.get("timestamp") - else: - total_value = 0 - for seg, segment_dtls in config.segments.items(): - if seg != "all_deals" and seg in segments: - seg_filter = config.segment_filters.get(seg) - overide_field = segment_dtls['sum_field_override'][ - field] if 'sum_field_override' in segment_dtls and \ - field in segment_dtls['sum_field_override'] else field - name, crit, ops = config.fields[overide_field]['crit_and_ops'] - sum_field_ = config.segment_amount_fields.get(seg, config.fields[overide_field]['sum_field']) - sum_field = config.fields[overide_field][ - 'sum_field'] if 'sum_field_override' in segment_dtls and \ - overide_field in segment_dtls[ - 'sum_field_override'] else sum_field_ - if sum_field != config.fields[overide_field]['sum_field']: - ops = [(sum_field, sum_field, '$sum')] - crit = {'$and': [crit, seg_filter]} - resp = fetch_deal_totals(period_and_close_periods, - node, - ops, - config.deal_config, - filter_criteria=crit, - timestamp=time_context.deal_timestamp, - db=db, - cache=cache, - prev_periods=time_context.prev_periods, - user=user, - sfdc_view=True if len(period) == 4 else False, - actual_view=True if len(period) == 4 else False - ) - val = resp.get(sum_field, 0) - if (ts and ts < resp.get("timestamp", 0)) or not ts: - ts = resp.get("timestamp") - total_value += len(val) if isinstance(val, list) else val - res = {'period': period, - 'segment': segment, - 'val': total_value, - 'by': 'system', - 'how': 'deal_rollup', - 'found': found, - 'timestamp': ts, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - return res - except: - val, found = 0, False - - overwritten_val = overwrite_fields(config, field, comp_periods, period, node, timecontext=time_context) - if overwritten_val is not None: - val = overwritten_val - - if call_from == "UploadStatus" and is_pivot_special: - res = {'period': period, - 'segment': segment, - 'val': len(val) if isinstance(val,list) else val, - 'by': by, - 'how': how, - 'found': found, - 'timestamp': ts, - 'node': node, - 'field': field} - else: - res = {'period': period, - 'segment': segment, - 'val': len(val) if isinstance(val,list) else val, - 'by': 'system', - 'how': 'deal_rollup', - 'found': found, - 'timestamp': ts, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - return res - - -def overwrite_fields(_config, _field, _comp_periods, _period, _nodes, timecontext=None): - # -- AV-7725 -- - overwrite_field = None - pipeline = None - period_in_past = False - try: - if 'Q' in _period and all(elem == 'h' for elem in timecontext.relative_periods): - period_in_past = True - elif 'h' in timecontext.relative_periods: - period_in_past = True - if not all(elem == 'h' for elem in timecontext.relative_periods): - period_in_past = False - if _config.fields[_field].get('overwrite_field') and period_in_past: - # HINT: overwrite_field = 'ACT_CRR.value' - overwrite_field = str(_config.fields[_field].get('overwrite_field')) - _col = sec_context.tenant_db[GBM_CRR_COLL] - if not isinstance(_nodes, list): - _nodes = [_nodes] - match = {'monthly_period': {'$in': _comp_periods}, - '__segs': {'$in': _nodes}} - projection = {'_id': 0, overwrite_field: 1} - group = {'_id': 0, _field: {"$sum": "$" + overwrite_field}} - pipeline = [{'$match': match}, - {'$project': projection}, - {'$group': group}] - aggs = list(_col.aggregate(pipeline, allowDiskUse=True)) - val = aggs[0][_field] if aggs else 0 - # AV-14059 Commenting below as part of the log fix - # logger.info( - # "overwriting {} with {} for period {} pipeline {}".format(_field, - # overwrite_field, - # _period, - # pipeline)) - return val - except Exception as _err: - logger.exception( - "Failed overwriting {} with {} for period {} {} {}".format(_field, overwrite_field, _period, pipeline, - _err)) - # -x- AV-7725 -x- - -def _bulk_fetch_formula_recs(time_context, - descendants_list, - fields, - segment, - config, - timestamp_info, - db, - cache, - main_cache=None): - """ - fetch many user entered records for multiple nodes and fields - returns nothing, just puts records in cache - """ - # NOTE: much of this logic is duplicated in _fetch_formula_rec - # if you are changing logic here, be sure to also change in the non bulk version of the function - year_accepted_from = 0 - try: - t = sec_context.details - year_accepted_from = int(t.get_flag('fm_latest_migration', 'year', 0)) - except: - year_accepted_from = 0 - for descendants, field in product(descendants_list, fields): - _fetch_formula_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=year_accepted_from, - main_cache=main_cache) - return cache - -def _fetch_formula_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=None, - main_cache=None - ): - """ - fetch record that is result of a specified function applied to other fm fields - """ - node, _, _ = descendants - period, comp_periods = time_context.fm_period, time_context.component_periods - source_fields = config.fields[field]['source'] - timestamp, _, _, _ = timestamp_info - - threads = [] - if cache is None: - cache = {} - - for source_field in source_fields: - ch_key = (period, node, source_field, segment, timestamp) - if ch_key not in cache and ((main_cache and ch_key not in main_cache) or main_cache is None or not main_cache): - t = threading.Thread(target=fetch_fm_rec, - args=(time_context, descendants, source_field, segment, - config, timestamp_info, db, cache, None, - None, - year_accepted_from)) - threads.append(t) - t.start() - for t in threads: - t.join() - - # source is actually used as a local variable by the eval - source = [] - ts = None - for source_field in source_fields: - val = cache.get((period, node, source_field, segment, timestamp), {'val': 0}).get('val') - ts_ = cache.get((period, node, source_field, segment, timestamp), {'timestamp': 0}).get('timestamp', 0) - field_type = config.fields[source_field]['type'] - by = 'system' - if field_type == 'UE': - by = cache.get((period, node, source_field, segment, timestamp), {'by': 'system'}).get('by', 'system') - by_ = 'system' - if val == 0 and main_cache: - val = main_cache.get((period, node, source_field, segment, timestamp), {'val': 0}).get('val') - ts_ = main_cache.get((period, node, source_field, segment, timestamp), {'timestamp': 0}).get('timestamp', 0) - if field_type == 'UE': - by_ = main_cache.get((period, node, source_field, segment, timestamp), {'by': 'system'}).get('by', 'system') - source.append(val) - if (ts and ts < ts_) or not ts: - ts = ts_ - by = by_ - if config.debug: - logger.info('fetch_formula_rec field: %s source : %s node %s', field, source, node) - try: - val = eval(config.fields[field]['func']) - except: - val = 0 - - if 'or' in config.fields[field]['func']: - source = [] - for source_field in source_fields: - field_type = config.fields[source_field]['type'] - by = cache.get((period, node, source_field, segment, timestamp), {'by': 'system'}).get('by', 'system') - source.append(by) - try: - by = eval(config.fields[field]['func']) - except: - by = 'system' - - res = {'period': period, - 'segment': segment, - 'val': val, - 'by': by, - 'how': 'formula', - 'found': True, - 'timestamp': ts, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - return res - -def _fetch_ai_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=None - ): - """ - fetch record that is generated from aviso predictions - """ - # NOTE: much of this logic is duplicated in _bulk_fetch_ai_recs - # if you are changing logic here, be sure to also change in the bulk version of the function - # TODO: think, make sure this is safe - node, _, _ = descendants - period, comp_periods = time_context.fm_period, time_context.component_periods - timestamp, _, _, cutoff_timestamp = timestamp_info - FM_COLL = read_from_collection(period[:4], timestamp=timestamp, call_from='_fetch_ai_rec', config=config, - year_accepted_from=year_accepted_from) - fm_data_collection = db[FM_COLL] - found = True - - # NOTE: taking a step backwards here until we have augustus monthly - # ai numbers are just taken straight from gbm, and are no longer the sum of component periods - # this might cause some inconsistencies for tenants with monthly predictions, so we should be cautious here - # Match query - match = {'period': {'$in': [period]}, - 'node': node, - 'field': field} - - # Check if the segment has a 'segment_func' in config - segment_config = config.segments.get(segment, None) - - if segment_config and 'segment_func' in segment_config: - # Segment has a segment_func, so we use its 'source' list for aggregation - segments = segment_config['segment_func']['source'] - match['segment'] = {"$in": segments} # Include all relevant sub-segments - else: - # No segment_func, process as a single segment - match['segment'] = segment - - if timestamp is not None: - match['timestamp'] = {'$lte': timestamp} - if cutoff_timestamp is not None: - # BUG: this doesnt work with comp periods - match['timestamp']['$gt'] = cutoff_timestamp - sort = {'timestamp': -1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'field': '$field', - 'segment': '$segment'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$first': '$val'}} - regroup = {'_id': None, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$sum': '$val'}} - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$group': regroup}] - if FM_COLL == FM_LATEST_DATA_COLL: - regroup = {'_id': None, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$sum': '$val'}} - pipeline = [{'$match': match}, - {'$group': regroup}] - - if config.debug: - logger.info('fetch_ai_rec pipeline: %s', pipeline) - - try: - aggs = list(fm_data_collection.aggregate(pipeline, allowDiskUse=True)) - val, by, ts = aggs[0]['val'], aggs[0]['by'], aggs[0]['timestamp'] - except IndexError: - val, by, ts, found = 0, 'system', timestamp, False - - return {'period': period, - 'segment': segment, - 'val': val, - 'by': by, - 'how': 'ai', - 'found': found, - 'timestamp': ts, - 'node': node, - 'field': field} - - -def _fetch_node_conditional_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=None - ): - """ - fetch record that is either a mgr field or a rep field depending on node - """ - node, children, _ = descendants - timestamp, _, _, _ = timestamp_info - - mgr_field, rep_field = config.fields[field]['source'] - - if not children: # TODO: think and make sure this is guaranteed - return fetch_fm_rec(time_context, - descendants, - rep_field, - segment, - config, - timestamp_info, - db=db, - cache=cache, - year_accepted_from=year_accepted_from - ) - - return fetch_fm_rec(time_context, - descendants, - mgr_field, - segment, - config, - timestamp_info, - db=db, - cache=cache, - year_accepted_from=year_accepted_from - ) - -def _bulk_fetch_child_sum_rec(time_context, - descendants_list, - fields, - segment, - config, - timestamp_info, - db, - cache, - round_val=True, - main_cache=None): - """ - fetch many child sum records for multiple nodes and fields - returns nothing, just puts records in cache - """ - # NOTE: much of this logic is duplicated in _fetch_formula_rec - # if you are changing logic here, be sure to also change in the non bulk version of the function - if main_cache is None: - main_cache = {} - - def get_actioned_user_and_timestamp(ch_key, latest_timestamp, by_val): - """ - :param ch_key: - :param latest_timestamp: - :param by_val: - :return: - - This functionality will be useful to retrive latest timestamp of FM Status Record and the user who did the activity. - """ - cs_timestamp = get_nested(cache, [ch_key, 'timestamp'], 0) or get_nested(main_cache, [ch_key, 'timestamp'], 0) - if cs_timestamp > latest_timestamp: - latest_timestamp = cs_timestamp - cs_actioned_user = get_nested(cache, [ch_key, 'by'], "system") or get_nested(main_cache, [ch_key, 'by'], "system") - if cs_actioned_user != "system": - by_val = cs_actioned_user - - return latest_timestamp, by_val - - period, comp_periods = time_context.fm_period, time_context.submission_component_periods - timestamp, _, _, _ = timestamp_info - for descendants, field in product(descendants_list, fields): - node, children, grandchildren = descendants - source_fields = config.fields[field]['source'] - val = 0 - latest_timestamp = 0 - by_val = 'system' - if not children: - for source_field in source_fields: - field_type = config.fields[source_field]['type'] - if field_type == 'NC': - mgr_field, rep_field = config.fields[source_field]['source'] - ch_key = (period, node, rep_field, segment, timestamp) - val = try_float(get_nested(cache, [ch_key, 'val'], 0)) or try_float(get_nested(main_cache, [ch_key, 'val'], 0)) - latest_timestamp, by_val = get_actioned_user_and_timestamp(ch_key, latest_timestamp, by_val) - else: - ch_key = (period, node, source_field, segment, timestamp) - val = try_float(get_nested(cache, [ch_key, 'val'], 0)) or try_float(get_nested(main_cache, [ch_key, 'val'], 0)) - latest_timestamp, by_val = get_actioned_user_and_timestamp(ch_key, latest_timestamp, by_val) - else: - for source_field, child in product(source_fields, children): - field_type = config.fields[source_field]['type'] - if field_type == 'NC': - grandkids = [grandkid for grandkid, parent in grandchildren.iteritems() if parent == child] - mgr_field, rep_field = config.fields[source_field]['source'] - if grandkids: - ch_key = (period, child, mgr_field, segment, timestamp) - val += try_float(get_nested(cache, [ch_key, 'val'], 0)) or try_float(get_nested(main_cache, [ch_key, 'val'], 0)) - latest_timestamp, by_val = get_actioned_user_and_timestamp(ch_key, latest_timestamp, by_val) - else: - ch_key = (period, child, rep_field, segment, timestamp) - val += try_float(get_nested(cache, [ch_key, 'val'], 0)) or try_float(get_nested(main_cache, [ch_key, 'val'], 0)) - latest_timestamp, by_val = get_actioned_user_and_timestamp(ch_key, latest_timestamp, by_val) - else: - ch_key = (period, child, source_field, segment, timestamp) - val += try_float(get_nested(cache, [(period, child, source_field, segment, timestamp), 'val'], 0)) or \ - try_float(get_nested(main_cache, [(period, child, source_field, segment, timestamp), 'val'], 0)) - latest_timestamp, by_val = get_actioned_user_and_timestamp(ch_key, latest_timestamp, by_val) - if round_val: - val = round(val) - overwritten_val = overwrite_fields(config, field, comp_periods, period, node, timecontext=time_context) - if overwritten_val is not None: - val = overwritten_val - res = {'period': period, - 'segment': segment, - 'val': val, - 'by': by_val, - 'how': 'sum_of_children', - 'found': True if val else False, - 'timestamp': timestamp if timestamp else latest_timestamp, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - return cache - -def _bulk_fetch_child_sum_rec_special(time_context, - descendants_list, - fields, - segment, - config, - timestamp_info, - db, - cache, - round_val=True): - """ - fetch many child sum records for multiple nodes and fields - returns nothing, just puts records in cache - """ - # NOTE: much of this logic is duplicated in _fetch_formula_rec - # if you are changing logic here, be sure to also change in the non bulk version of the function - period, comp_periods = time_context.fm_period, time_context.submission_component_periods - timestamp, _, _, _ = timestamp_info - try: - t = sec_context.details - year_accepted_from = int(t.get_flag('fm_latest_migration', 'year', 0)) - except: - year_accepted_from = 0 - nodes_all = [] - for descendants, field in product(descendants_list, fields): - node, children, grandchildren = descendants - nodes_all += children if not config.fields[field].get('grandkids') else grandchildren - nodes_and_eligible_segs_len = {} - if nodes_all and config.partially_segmented: - for node in nodes_all: - nodes_and_eligible_segs_len[node] = len(config.get_segments(epoch().as_epoch(), node)) - - for descendants, field in product(descendants_list, fields): - node, _, _ = descendants - res = _fetch_child_sum_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=year_accepted_from, - nodes_and_eligible_segs_len=nodes_and_eligible_segs_len) - if round_val: - res['val'] = round(res['val']) - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - return cache - - -def _fetch_child_sum_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=None, - nodes_and_eligible_segs_len=None - ): - """ - fetch record that is the sum of childrens fm fields - """ - node_parent, children, grandchildren = descendants - segments = config.segment_map[segment] - period, comp_periods = time_context.fm_period, time_context.submission_component_periods - source_fields = config.fields[field]['source'] - timestamp, _, _, cutoff_timestamp = timestamp_info - found = True - - node_children_check = {node: any(node_children) for node, node_children, _ in [descendants]} - - for child in children: - grandkids = {grandkid: parent for grandkid, parent in grandchildren.iteritems() - if parent == child} - node_children_check[child] = any(grandkids) - - field_type = config.fields[source_fields[0]]['type'] - if 'hist_field' in config.fields[source_fields[0]] and 'h' in time_context.relative_periods and not config.show_raw_data_in_trend: - # historic period, switch to using true up field if it exists - field_type = 'PC' - if config.quarter_editable: - if not all(elem == 'h' for elem in time_context.relative_periods): - field_type = config.fields[source_fields[0]]['type'] - - - if year_accepted_from is None: - try: - t = sec_context.details - year_accepted_from = int(t.get_flag('fm_latest_migration', 'year', 0)) - except: - year_accepted_from = 0 - - if field_type != 'PC' and all(source_field in config.user_entered_fields for source_field in source_fields): - if segment == 'all_deals': - segments = [segment for segment in config.segments] - else: - segments = config.segment_map[segment] - nodes = children if not config.fields[field].get('grandkids') else grandchildren - val = sum([try_float(get_nested(cache, [(period, child, source_field, segment, timestamp), 'val'], 0)) for - source_field, child in product(source_fields, nodes)]) - if val: - return {'period': period, - 'segment': segment, - 'val': val, - 'by': 'system', - 'how': 'sum_of_children', - 'found': found, - 'timestamp': timestamp, - 'node': node_parent, - 'field': field} - - FM_COLL = read_from_collection(period[:4], timestamp=timestamp, call_from='_fetch_child_sum_rec', config=config, - year_accepted_from=year_accepted_from) - fm_data_collection = db[FM_COLL] - by_val = 'system' - match = {'period': {'$in': comp_periods}, - 'segment': {'$in': segments}, - 'node': {'$in': nodes.keys() if nodes else [node_parent]}, - 'field': {'$in': config.fields[field]['source']}} - if timestamp is not None: - match['timestamp'] = {'$lte': timestamp} - sort = {'timestamp': 1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'segment': '$segment', - 'field': '$field'}, - 'val': {'$last': '$val'}, - 'by': {'$last': '$by'}, - 'timestamp': {'$last': '$timestamp'}} - regroup = {'_id': {'node': '$_id.node', - 'field': '$_id.field'}, - 'val': {'$sum': '$val'}, - 'by': {'$last': '$by'}, - 'timestamp': { - '$last': '$timestamp' - }, - 'comments':{'$last': '$comments'} - } - - group['_id']['segment'] = '$segment' - regroup['_id']['segment'] = '$_id.segment' - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$sort': sort}, - {'$group': regroup}] - if FM_COLL == FM_LATEST_DATA_COLL: - regroup = {'_id': {'node': '$node', - 'field': '$field'}, - 'val': {'$sum': '$val'}, - 'by': {'$last': '$by'}, - 'timestamp': { - '$last': '$timestamp' - }} - regroup['_id']['segment'] = '$segment' - pipeline = [{'$match': match}, - {'$group': regroup}] - if config.debug: - logger.info('fetch child sum pipeline: %s from %s' % (pipeline, FM_COLL)) - - recs = fm_data_collection.aggregate(pipeline, allowDiskUse=True) - - db_recs = {(rec['_id']['node'], rec['_id']['field'], rec['_id']['segment']): rec - for rec in recs} - rollup_segments = config.rollup_segments - if nodes_and_eligible_segs_len is None: - nodes_and_eligible_segs_len = {} - if config.partially_segmented: - for node_ in nodes: - nodes_and_eligible_segs_len[node_] = len(config.get_segments(epoch().as_epoch(), node_)) - - res_val = 0 - res_timestamp = 0 - for node_ in set(nodes): - aggregated_val = 0 - len_of_segments = len(segments) - # if the node has only one segment, aggregated_val condition should be passed. THis is in case the tenant has segments, - # but the current node doesn't have segments - if len_of_segments == 1 and 'all_deals' in segments: - # value in all_deals is summation of segments. Hence all segments need to be looped through - # for non-segmented tenants ['all_deals'] will be retained by default - segments = [segment_ for segment_ in config.segments] - len_of_segments = len(segments) - len_of_eligible_segments = nodes_and_eligible_segs_len.get(node_) - segment_values_dict = {} - for segment_ in segments: - val = get_nested(db_recs, [(node_, config.fields[field]['source'][0], segment_), 'val'], 0) - overwritten_val = overwrite_fields(config, config.fields[field]['source'][0], comp_periods, period, node_, timecontext=time_context) - if overwritten_val is not None: - val = overwritten_val - if ((not node_children_check[node_]) or (len_of_segments == 1) or (len_of_eligible_segments == 1) or (len_of_segments > 1 and segment_ != "all_deals")) and segment_ in rollup_segments: - aggregated_val += val - new_timestamp = get_nested(db_recs, [(node_, config.fields[field]['source'][0], segment), 'timestamp'], 0) - if res_timestamp is None or res_timestamp < new_timestamp: - res_timestamp = get_nested(db_recs, [(node_, config.fields[field]['source'][0], segment), 'timestamp'], 0) - if not ('all_deals' in segments and config.is_special_pivot_segmented(node_)): - res_val += val - segment_values_dict[segment_] = val - - if 'all_deals' in segments and config.is_special_pivot_segmented(node_): - prev_val = segment_values_dict.get("all_deals", 0) - val = 0.0 if (config.segment_percentage_rollup and config.fields[config.fields[field]['source'][0]].get('format') == 'percentage') else aggregated_val - - if (prev_val is not None and config.fields[field]['source'][0] in config.forecast_service_editable_fields): - val = prev_val - res_val += val - return {'period': period, - 'segment': segment, - 'val': res_val, - 'by': 'system', - 'how': 'sum_of_children', - 'found': found, - 'timestamp': timestamp if timestamp is not None else res_timestamp, - 'node': node_parent, - 'field': field} - - threads = [] - if cache is None: - cache = {} - - #CS-9233 if the node does not have children, get the value of source field for the node itself - if not children: - return fetch_fm_rec(time_context, - descendants, - config.fields[field]['source'][0], - segment, - config, - timestamp_info, - db=db, - cache=cache, - year_accepted_from=year_accepted_from - ) - - - for source_field, child in product(source_fields, children): - child_descendants = (child, {grandkid: parent for grandkid, parent in grandchildren.iteritems() - if parent == child}, {}) - t = threading.Thread(target=fetch_fm_rec, - args=(time_context, - child_descendants, - source_field, - segment, - config, - timestamp_info, - db, - cache, - None, - None, - year_accepted_from - )) - threads.append(t) - t.start() - for t in threads: - t.join() - - val = sum([try_float(get_nested(cache, [(period, child, source_field, segment, timestamp), 'val'], 0)) for - source_field, child in product(source_fields, children)]) - - return {'period': period, - 'segment': segment, - 'val': val, - 'by': 'system', - 'how': 'sum_of_children', - 'found': found, - 'timestamp': timestamp, - 'node': node_parent, - 'field': field} - - -def _bulk_period_conditional_rec(time_context, - descendants_list, - fields, - segment, - config, - timestamp_info, - db, - cache, - round_val=True): - """ - fetch many child sum records for multiple nodes and fields - returns nothing, just puts records in cache - """ - # NOTE: much of this logic is duplicated in _fetch_formula_rec - # if you are changing logic here, be sure to also change in the non bulk version of the function - period, comp_periods = time_context.fm_period, time_context.submission_component_periods - timestamp, _, _, _ = timestamp_info - try: - t = sec_context.details - year_accepted_from = int(t.get_flag('fm_latest_migration', 'year', 0)) - except: - year_accepted_from = 0 - for descendants, field in product(descendants_list, fields): - node, _, _ = descendants - res = _fetch_period_conditional_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=year_accepted_from) - if round_val: - res['val'] = round(res['val']) - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - return cache - -def _fetch_period_conditional_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - year_accepted_from=None - ): - """ - fetch record that is either a regular field or historic field depending on if period is historic - """ - node, _, _ = descendants - period, comp_periods = time_context.fm_period, time_context.component_periods - relative_periods = time_context.relative_periods - timestamp, _, _, _ = timestamp_info - hist_field = config.fields[field]['hist_field'] - period_info = time_context.period_info - all_quarters = time_context.all_quarters - if year_accepted_from is None: - try: - t = sec_context.details - year_accepted_from = int(t.get_flag('fm_latest_migration', 'year', 0)) - except: - year_accepted_from = 0 - - def get_comp_and_fields_condition(comp_period): - timestamp_condtion = False - if not timestamp: - #timestamp none means, get me the last value in that quarter, i.e get me the value on the eoq date - timestamp_condtion = True - else: - #when the timestamp is not None, we check if the timestamp is the eoq timestamp. - #If it is the eoq timestamp and rel_period is 'h', we use the hist field, otherwise we don't. - # [:19] because some epochs have milliseconds and some don't - epoch_timestamp = epoch(timestamp) - if ((len(comp_periods) > 1 or 'Q' in comp_period)) and (get_eoq_for_period(period, period_info)[:19] == str(epoch_timestamp)[:19]): - if str(get_eom(epoch_timestamp))[:19] == str(epoch_timestamp)[:19]: - timestamp_condtion = False - else: - timestamp_condtion = True - elif 'Q' not in comp_period: - component_periods_boundries = time_context.component_periods_boundries - eom_timestamp = component_periods_boundries[comp_period]['end'] - # For a monthly tenant, suppose the months are may, june and july - # we want to check quarterly view as of 20th july, so, we will get hist_field for may and june, and field for july - # then eom_timestamp of may will be lesser than 20th july i.e. int(eom_timestamp) <= int(timestamp) - # if we are checking for july month - if int(eom_timestamp) <= int(timestamp) or str(epoch(eom_timestamp))[:19] == str(epoch_timestamp)[:19]: - timestamp_condtion = True - return timestamp_condtion - - comps_and_fields = [(comp_period, hist_field if rel_period == 'h' and get_comp_and_fields_condition(comp_period) else field) - for comp_period, rel_period in zip(comp_periods, relative_periods)] - - hist_field_config = config.fields.get(hist_field, {}) - hist_field_is_cumulative = hist_field_config.get('is_cumulative', False) - if hist_field_is_cumulative and len(comps_and_fields) > 1: - updated_comp_period = "" - updated_comp_field = "" - for i, (comp_period, comp_field) in enumerate(comps_and_fields): - if comp_period > updated_comp_period: - updated_comp_period = comp_period - updated_comp_field = comp_field - - comps_and_fields = [(updated_comp_period, updated_comp_field)] - - threads = [] - if cache is None: - cache = {} - - for i, (comp_period, comp_field) in enumerate(comps_and_fields): - comp_qtr=all_quarters[i] - # TODO: holy shit this is gross and too much time knowledge in the fm service - tc = time_context._asdict() - tc.update({'fm_period': comp_period, - 'component_periods': [comp_period], - 'submission_component_periods': [comp_period], - 'close_periods': [comp_period], - 'deal_period': comp_qtr, - 'relative_periods': [], - 'period_info': period_info}) - comp_context = time_context_tuple(**tc) - t = threading.Thread(target=fetch_fm_rec, - args=(comp_context, - descendants, - comp_field, - segment, - config, - timestamp_info, - db, - cache, - None, - None, - year_accepted_from - )) - threads.append(t) - t.start() - for t in threads: - t.join() - val = sum([try_float(get_nested(cache, [(comp_period, node, comp_field, segment, timestamp), 'val'], 0)) - for comp_period, comp_field in comps_and_fields]) - overwritten_val = overwrite_fields(config, field, comp_periods, period, node, timecontext=time_context) - if overwritten_val is not None: - val = overwritten_val - return {'period': period, - 'segment': segment, - 'val': val, - 'by': 'system', - 'how': 'true_up', - 'found': True, - 'timestamp': timestamp, - 'node': node, - 'field': field} - -def _bulk_fetch_user_entered_recs_v2(time_context, - descendants_list, - fields, - segments, - config, - timestamp_info, - db, - cache, - get_all_segments=False, - round_val=True, - found=None, - nodes_and_eligible_segs_len=None, - includefuturetimestamp=False, - exclude_empty=None, - updated_since=None, - skip=None, - limit=None): - """ - fetch many user entered records for multiple nodes and fields - returns nothing, just puts records in cache - """ - # NOTE: much of this logic is duplicated in _fetch_user_entered_rec - # if you are changing logic here, be sure to also change in the non bulk version of the function - try: - found = found if found is not None else False - except: - found = False - try: - node_children_check = {node: any(node_children) for node, node_children, _ in descendants_list} - nodes = node_children_check.keys() - period, comp_periods = time_context.fm_period, time_context.submission_component_periods - timestamp, _, _, cutoff_timestamp = timestamp_info - FM_COLL = read_from_collection(period[:4], timestamp=timestamp, call_from='_bulk_fetch_user_entered_recs', - config=config) - fm_data_collection = db[FM_COLL] - how = 'sum_of_users' if comp_periods[0] != period else 'fm_upload' - - if len(period) == 4 and config.quarter_editable: # add quarters as well incase of monthly tenants - comp_periods = time_context.all_quarters - match = {'period': {'$in': comp_periods}, - 'node': {'$in': nodes}, - 'field': {'$in': fields}} - if updated_since is not None: - match['timestamp'] = {'$gte': int(updated_since)} - if exclude_empty is not None and exclude_empty is True: - match['val'] = {'$nin': [0.0, 0, None]} - if not get_all_segments: - match['segment'] = {'$in': segments} - if timestamp is not None: - match['timestamp'] = {'$lte': timestamp} - if includefuturetimestamp: - match['timestamp'] = {'$gte': timestamp} - - sort = {'timestamp': 1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'field': '$field'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$last': '$val'}, - 'comments': {'$last': '$comments'}} - - regroup = {'_id': {'node': '$_id.node', - 'field': '$_id.field'}, - 'val': {'$sum': '$val'}, - 'by': {'$last': '$by'}, - 'timestamp': { - '$last': '$timestamp' - }, - 'comments': {'$last': '$comments'} - } - - group['_id']['segment'] = '$segment' - regroup['_id']['segment'] = '$_id.segment' - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$sort': sort}, - {'$group': regroup}] - if FM_COLL == FM_LATEST_DATA_COLL: - regroup = {'_id': {'node': '$node', - 'field': '$field'}, - 'val': {'$sum': '$val'}, - 'by': {'$last': '$by'}, - 'timestamp': { - '$last': '$timestamp' - }} - regroup['_id']['segment'] = '$segment' - pipeline = [{'$match': match}, - {'$group': regroup}] - if skip is not None or limit is not None: - pipeline.append({'$skip': int(skip) if skip else 0}) - if limit: - pipeline.append({'$limit': int(limit)}) - if config.debug: - logger.info('bulk_fetch_user_entered_rec_v2 pipeline: %s from %s' % (pipeline, FM_COLL)) - - recs = fm_data_collection.aggregate(pipeline, allowDiskUse=True) - return recs - - except Exception as e: - logger.exception(e) - raise e - -def _bulk_fetch_user_entered_recs(time_context, - descendants_list, - fields, - segments, - config, - timestamp_info, - db, - cache, - get_all_segments=False, - round_val=True, - found=None, - nodes_and_eligible_segs_len=None, - includefuturetimestamp=False): - """ - fetch many user entered records for multiple nodes and fields - returns nothing, just puts records in cache - """ - # NOTE: much of this logic is duplicated in _fetch_user_entered_rec - # if you are changing logic here, be sure to also change in the non bulk version of the function - try: - found = found if found is not None else False - except: - found = False - try: - node_children_check = {node: any(node_children) for node, node_children, _ in descendants_list} - nodes = node_children_check.keys() - period, comp_periods = time_context.fm_period, time_context.submission_component_periods - timestamp, _, _, cutoff_timestamp = timestamp_info - FM_COLL = read_from_collection(period[:4], timestamp=timestamp, call_from='_bulk_fetch_user_entered_recs', - config=config) - fm_data_collection = db[FM_COLL] - how = 'sum_of_users' if comp_periods[0] != period else 'fm_upload' - - if len(period) == 4 and config.quarter_editable: # add quarters as well incase of monthly tenants - comp_periods = time_context.all_quarters - - if set(fields).intersection(set(config.weekly_data)): - comp_periods = [period] - - if set(fields).intersection(set(config.quarterly_high_low_fields)) and "Q" in period: - comp_periods = [period] - - match = {'period': {'$in': comp_periods}, - 'node': {'$in': nodes}, - 'field': {'$in': fields}} - if not get_all_segments: - match['segment'] = {'$in': segments} - if timestamp is not None: - match['timestamp'] = {'$lte': timestamp} - if includefuturetimestamp: - match['timestamp'] = {'$gte': timestamp} - - sort = {'timestamp': 1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'field': '$field'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$last': '$val'}, - 'comments':{'$last': '$comments'}} - - regroup = {'_id': {'node': '$_id.node', - 'field': '$_id.field'}, - 'val': {'$sum': '$val'}, - 'by': {'$last': '$by'}, - 'timestamp': { - '$last': '$timestamp' - }, - 'comments':{'$last': '$comments'} - } - - group['_id']['segment'] = '$segment' - regroup['_id']['segment'] = '$_id.segment' - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$sort': sort}, - {'$group': regroup}] - if FM_COLL == FM_LATEST_DATA_COLL: - regroup = {'_id': {'node': '$node', - 'field': '$field'}, - 'val': {'$sum': '$val'}, - 'by': {'$last': '$by'}, - 'timestamp': { - '$last': '$timestamp' - }} - regroup['_id']['segment'] = '$segment' - pipeline = [{'$match': match}, - {'$group': regroup}] - - if config.debug: - logger.info('bulk_fetch_user_entered_rec pipeline: %s from %s' % (pipeline, FM_COLL)) - - recs = fm_data_collection.aggregate(pipeline, allowDiskUse=True) - - db_recs = {(rec['_id']['node'], rec['_id']['field'], rec['_id']['segment']): rec - for rec in recs} - rollup_segments = config.rollup_segments - if nodes_and_eligible_segs_len is None: - nodes_and_eligible_segs_len = {} - if config.partially_segmented: - for node in nodes: - nodes_and_eligible_segs_len[node] = len(config.get_segments(epoch().as_epoch(), node)) - for node, field in product(nodes, fields): - aggregated_val = 0 - len_of_segments = len(segments) - # if the node has only one segment, aggregated_val condition should be passed. THis is in case the tenant has segments, - # but the current node doesn't have segments - if len_of_segments == 1 and 'all_deals' in segments: - # value in all_deals is summation of segments. Hence all segments need to be looped through - # for non-segmented tenants ['all_deals'] will be retained by default - segments = [segment for segment in config.segments] - len_of_segments = len(segments) - len_of_eligible_segments = nodes_and_eligible_segs_len.get(node) - for segment in segments: - - val = get_nested(db_recs, [(node, field, segment), 'val'], 0) - overwritten_val = overwrite_fields(config, field, comp_periods, period, node, timecontext=time_context) - if overwritten_val is not None: - val = overwritten_val - if ((not node_children_check[node]) or (len_of_segments == 1) or (len_of_eligible_segments == 1) or (len_of_segments > 1 and segment != "all_deals")) and segment in rollup_segments: - aggregated_val += val - cache[(period, node, field, segment, timestamp)] = {'period': period, - 'segment': segment, - 'val': round(val) if round_val and not config.fields[field].get('format') == 'percentage' else val, - 'by': get_nested(db_recs, [(node, field, segment), 'by'], 'system'), - 'how': how, - 'found': found if found else (node, field, segment) in db_recs, - 'timestamp': get_nested(db_recs, [(node, field, segment), 'timestamp'], 0), - 'node': node, - 'field': field} - if config.fields[field].get('commentable') == True: - comments = get_nested(db_recs, [(node, field, segment), 'comments'], '') - cache[(period, node, field, segment, timestamp)]['comments'] = comments - - if 'all_deals' in segments and config.is_special_pivot_segmented(node): - prev_val = cache[(period, node, field, 'all_deals', timestamp)]['val'] - cache[(period, node, field, 'all_deals', timestamp)]['val'] = 0.0 if \ - (config.segment_percentage_rollup and config.fields[field].get('format') == 'percentage') \ - else (round(aggregated_val) - if round_val and not config.fields[field].get('format') == 'percentage' else aggregated_val) - if (prev_val is not None and field in config.forecast_service_editable_fields): - cache[(period, node, field, 'all_deals', timestamp)]['val'] = prev_val - return cache - except Exception as e: - logger.exception(e) - raise e - -def _bulk_fetch_deal_rollup_recs(time_context, - descendants_list, - fields, - segments, - config, - timestamp_info, - db, - cache, - round_val=True, - get_all_segments=False, - is_dr_deal_fields=False, - dr_from_fm_coll_for_snapshot=None, - found=None, - eligible_nodes_for_segs=None, - call_from=None, - trend=False - ): - """ - fetch many deal rollup records for multiple nodes and fields - returns nothing, just puts records in cache - """ - # NOTE: much of this logic is duplicated in _fetch_deal_rollup_rec - # if you are changing logic here, be sure to also change in the non bulk version of the function - nodes = [node for node, _, _ in descendants_list] - seg_sum_fields = config.segment_amount_fields - seg_filter = {} - period, comp_periods = time_context.fm_period, time_context.component_periods - timestamp, dlf_expiration, deal_expiration, _ = timestamp_info - FM_COLL = read_from_collection(period[:4], timestamp=timestamp, field_type='DR', - call_from='bulk_fetch_deal_rollup_recs', config=config) - fm_data_collection = db[FM_COLL] - monthly_period_realtime_update = config.monthly_period_realtime_update - - - fields_not_found = {} - try: - dr_from_fm_coll_for_snapshot = dr_from_fm_coll_for_snapshot if dr_from_fm_coll_for_snapshot is not None else sec_context.details.get_flag('deal_rollup', 'snapshot_dr_from_fm_coll', False) - except: - dr_from_fm_coll_for_snapshot = False - - try: - found = found if found is not None else False - except: - found = False - - is_monthly = config.periods_config.monthly_fm - if (((len(comp_periods) == 1 and len(segments) == 1 and (not is_monthly or ( - 'Q' not in comp_periods[0] and not monthly_period_realtime_update)) and is_dr_deal_fields) or \ - (dr_from_fm_coll_for_snapshot)) and trend) or \ - (timestamp and not is_same_day(EpochClass(), epoch(timestamp))) or \ - (len(period) == 4 and config.read_from_fm_collection_for_year): - dlf_fields = [] - fm_fields_ = [] - for field in fields: - if config.deal_rollup_fields[field].get('dlf') or any(filt.get('op') == 'dlf' if isinstance(filt, dict) else \ - False for filt in config.deal_rollup_fields[field]['filter']): - dlf_fields.append(field) - else: - fm_fields_.append(field) - for field_type in ['dlf', 'deal_fields']: - fields_ = None - if field_type == 'dlf': - fields_ = dlf_fields - expiration_timestamp = dlf_expiration - else: - fields_ = fm_fields_ - expiration_timestamp = deal_expiration - if fields_: - if timestamp and timestamp < expiration_timestamp: - query_expiration_timestamp = 0 - else: - query_expiration_timestamp = expiration_timestamp - match = {'period': {'$in': comp_periods}, - 'node': {'$in': nodes}, - 'field': {'$in': fields_}} - if not get_all_segments: - match['segment'] = {'$in': segments} - if timestamp is not None: - match['timestamp'] = {'$lte': timestamp} - sort = {'timestamp': 1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'segment': '$segment', - 'field': '$field'}, - 'val': {'$last': '$val'}} - regroup = {'_id': {'node': '$_id.node', - 'field': '$_id.field', - 'segment': '$_id.segment'}, - 'val': {'$sum': '$val'}} - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$group': regroup}] - if FM_COLL == FM_LATEST_COLL: - regroup = {'_id': {'node': '$node', - 'field': '$field', - 'segment': '$segment'}, - 'val': {'$sum': '$val'}} - pipeline = [{'$match': match}, - {'$group': regroup}] - - if config.debug: - logger.info('bulk fetch_deal_rollup_rec pipeline: %s from %s' % (pipeline, FM_COLL)) - recs = fm_data_collection.aggregate(pipeline, allowDiskUse=True) - - db_recs = {(rec['_id']['node'], rec['_id']['field'], rec['_id']['segment']): rec - for rec in recs} - rollup_segments = config.rollup_segments - - for node, field in product(nodes, fields_): - aggregated_val = 0 - for segment in segments: - val = get_nested(db_recs, [(node, field, segment), 'val'], None) - if val is None: - fields_not_found[(node, field)] = expiration_timestamp - val = 0 - overwritten_val = overwrite_fields(config, field, comp_periods, period, node, timecontext=time_context) - if overwritten_val is not None: - val = overwritten_val - if segment in rollup_segments: - aggregated_val += val - cache[(period, node, field, segment, timestamp)] = {'period': period, - 'segment': segment, - 'val': round(val) if round_val else val, - 'by': get_nested(db_recs, [(node, field, segment), 'by'], 'system'), - 'how': 'deal_rollup', - 'found': found if found else (node, field, segment) in db_recs, - 'timestamp': get_nested(db_recs, [(node, field, segment), 'timestamp'], timestamp), - 'node': node, - 'field': field} - if 'all_deals' in segments: - prev_val = cache[(period, node, field, 'all_deals', timestamp)]['val'] - cache[(period, node, field, 'all_deals', timestamp)]['val'] = round(aggregated_val) if round_val else aggregated_val - - if (prev_val is not None and field in config.forecast_service_editable_fields): - cache[(period, node, field, 'all_deals', timestamp)]['val'] = prev_val - return cache - - fields_and_nodes = {} - if fields_not_found: - for node, field in fields_not_found.keys(): - expiration_timestamp = fields_not_found.get((node,field), 0) - if timestamp and expiration_timestamp and abs(timestamp - time_context.now_timestamp) > ONE_DAY: - pass - else: - if not fields_and_nodes.get(field): - fields_and_nodes[field] = [] - fields_and_nodes[field].append(node) - if not fields_and_nodes: - return cache - elif timestamp and not is_same_day(EpochClass(), epoch(timestamp)): - return cache - prev_periods = prev_periods_allowed_in_deals() - - if fields_and_nodes: - for field in fields_and_nodes: - nodes = fields_and_nodes[field] - fetch_dr_from_deals(config, seg_filter, seg_sum_fields, segments, [field], nodes, time_context, - period, timestamp, round_val=round_val, cache=cache, db=db, - prev_periods=prev_periods) - return cache - - return fetch_dr_from_deals(config, seg_filter, seg_sum_fields, segments, fields, nodes, time_context, - period, timestamp, round_val=round_val, cache=cache, db=db, - prev_periods=prev_periods) - - -def fetch_dr_from_deals(config, - seg_filter, - seg_sum_fields, - segments, - fields, - nodes, - time_context, - period, - timestamp, - round_val=True, - cache=None, - db=None, - prev_periods=[]) : - crits_and_ops = [schema['crit_and_ops'] - for field, schema in config.deal_rollup_fields.iteritems() if field in fields] - - seg_crit_and_ops = [ - (filt_name, - {'$and': [filt, seg_filter]}, - list(set([(seg_sum_fields.get(segment) if seg_sum_fields.get(segment) else sum_field, rendered_field, op) - for (sum_field, rendered_field, op) in ops - for segment in segments])), - config.deal_rollup_fields[filt_name]['filter'], - any(filt.get('op') == 'dlf' if isinstance(filt, dict) else False for filt in config.deal_rollup_fields[filt_name]['filter']) or \ - config.deal_rollup_fields[filt_name].get('dlf')) - for filt_name, filt, ops in crits_and_ops - ] - if len(period) == 4: - period_and_close_periods = time_context.period_and_close_periods_for_year or \ - get_period_and_close_periods(period, False, deal_svc=True) - else: - period_and_close_periods = [(time_context.deal_period, time_context.close_periods)] - - totals = fetch_many_dr_from_deals(period_and_close_periods, - nodes, - seg_crit_and_ops, - config.deal_config, - db=db, - return_seg_info=True, - timestamp=time_context.deal_timestamp, - prev_periods=prev_periods or time_context.prev_periods, - actual_view=True if len(period)==4 else False) - - for segment in segments: - segment_ = segment - if segment != 'all_deals': - segment_filter_ = config.segment_filters.get(segment) - deal_config = config.deal_config - seg_field = deal_config.segment_field - if seg_field and segment_filter_ and segment_filter_.get(seg_field) and segment_filter_.get(seg_field).get( - "$in"): - segment_ = segment_filter_.get(seg_field).get("$in") - for node, field in product(nodes, fields): - val = 0 - if isinstance(segment_,list): - for seg in segment_: - if seg_sum_fields.get(segment): - val += get_nested(totals, [(field, seg, node), seg_sum_fields[segment]],0) - else: - val += get_nested(totals, [(field, seg, node), config.fields[field]['sum_field']], 0) - else: - val += get_nested(totals, [(field, segment_, node), seg_sum_fields[segment] if seg_sum_fields.get(segment) - else config.fields[field]['sum_field']], 0) - cache[(period, node, field, segment, timestamp)] = {'period': period, - 'segment': segment, - 'val': round(val) if round_val else val, - 'by': 'system', - 'how': 'deal_rollup', - 'found': True, - 'timestamp': timestamp, - 'node': node, - 'field': field} - - return cache - - -def _bulk_fetch_prnt_dr_recs(config, - fields, - time_context, - descendants_list, - segments, - timestamp_info, - db, - cache, - timestamp, - eligible_nodes_for_segs, - round_val=True, - get_all_segments=True, - node=None, - prev_periods=[]): - """ - fetch records for each segment, field, node, timestamp for deals matching a filter. - Fetching of recording happens one at a time and the fetched records and added in the cache. - At the end we return the cache - - Arguments: - time_context {time_context} -- fm period, components of fm period - deal period, close periods of deal period - deal expiration timestamp, - relative periods of component periods - ('2020Q2', ['2020Q2'], - '2020Q2', ['201908', '201909', '201910'], - 1556074024910, - ['h', 'c', 'f']) - descendants_list {list} -- list of tuples of - (node, [children], [grandchildren]) - fetches data for each node - using children/grandkids to compute sums - [('A', ['B, 'C'], ['D', 'E'])] - fields {list} -- list of field names - ['commit', 'best_case'] - segments {list} -- list of segment names - ['all_deals', 'new', 'upsell'] - config -- instance of Config - timestamps {list} -- list of epoch timestamps to get data as of - [1556074024910, 1556075853732, ...] - - Keyword Arguments: - recency_window {int} -- window to reject data from before timestamp - recency_window (default: {None}) - if None, will accept any record, regardless of how stale - ONE_DAY - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - - Returns: - cache - """ - prnt_node= descendants_list[0][0] - if node: - nodes = [node] - prnt_node = node - nodes = [node for node, _, _ in descendants_list] - seg_sum_fields = config.segment_amount_fields - seg_filter = {} - period, comp_periods = time_context.fm_period, time_context.component_periods - timestamp, dlf_expiration, deal_expiration, _ = timestamp_info - FM_COLL = read_from_collection(period[:4], timestamp=timestamp, field_type='DR', - call_from='_bulk_fetch_prnt_dr_recs', config=config) - fm_data_collection = db[FM_COLL] - if cache is None: - cache = {} - db = db if db else sec_context.tenant_db - if timestamp and not is_same_day(EpochClass(), epoch(timestamp)): - dlf_fields = [] - fm_fields_ = [] - for field in fields: - if config.prnt_deal_rollup_fields[field].get('dlf') or \ - any(filt.get('op') == 'dlf' if isinstance(filt, dict) else False - for filt in config.prnt_deal_rollup_fields[field]['filter']): - dlf_fields.append(field) - else: - fm_fields_.append(field) - for field_type in ['dlf', 'deal_fields']: - fields_ = None - if field_type == 'dlf': - fields_ = dlf_fields - expiration_timestamp = dlf_expiration - else: - fields_ = fm_fields_ - expiration_timestamp = deal_expiration - if fields_: - if timestamp and timestamp < expiration_timestamp: - query_expiration_timestamp = 0 - else: - query_expiration_timestamp = expiration_timestamp - match = {'period': {'$in': comp_periods}, - 'node': {'$in': nodes}, - 'field': {'$in': fields_}} - if not get_all_segments: - match['segment'] = {'$in': segments} - if timestamp is not None: - match['timestamp'] = {'$lte': timestamp} - sort = {'timestamp': 1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'segment': '$segment', - 'field': '$field'}, - 'val': {'$last': '$val'}} - regroup = {'_id': {'node': '$_id.node', - 'field': '$_id.field', - 'segment': '$_id.segment'}, - 'val': {'$sum': '$val'}} - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$group': regroup}] - if FM_COLL == FM_LATEST_COLL: - regroup = {'_id': {'node': '$node', - 'field': '$field', - 'segment': '$segment'}, - 'val': {'$sum': '$val'}} - pipeline = [{'$match': match}, - {'$group': regroup}] - - if config.debug: - logger.info('bulk fetch_prnt_dr_recs pipeline: %s from %s' % (pipeline, FM_COLL)) - recs = fm_data_collection.aggregate(pipeline, allowDiskUse=True) - - db_recs = {(rec['_id']['node'], rec['_id']['field'], rec['_id']['segment']): rec for rec in recs} - rollup_segments = config.rollup_segments - - for node, field in product(nodes, fields_): - aggregated_val = 0 - for segment in segments: - val = get_nested(db_recs, [(node, field, segment), 'val'], None) - if val is None: - val = 0 - if segment in rollup_segments and (segment != 'all_deals' or \ - (segment == 'all_deals' and len(segments) == 1)): - aggregated_val += val - cache[(period, node, field, segment, timestamp)] = {'period': period, - 'segment': segment, - 'val': round(val) if round_val else val, - 'by': get_nested(db_recs, [(node, field, segment), 'by'], 'system'), - 'how': 'deal_rollup', - 'found': True, - 'timestamp': get_nested(db_recs, [(node, field, segment), 'timestamp'], timestamp), - 'node': node, - 'field': field} - if 'all_deals' in segments: - prev_val = cache[(period, node, field, 'all_deals', timestamp)]['val'] - cache[(period, node, field, 'all_deals', timestamp)]['val'] = round(aggregated_val) if round_val else aggregated_val - - if (prev_val is not None and field in config.forecast_service_editable_fields): - cache[(period, node, field, 'all_deals', timestamp)]['val'] = prev_val - return cache - else: - timestamps = [None] - if not eligible_nodes_for_segs: - for segment in config.segments: - if segment == config.primary_segment: - continue - eligible_nodes_for_segs[segment] = [dd['node'] for dd in fetch_eligible_nodes_for_segment(time_context.now_timestamp, segment, - db=db)] - - prev_periods = prev_periods if prev_periods else prev_periods_allowed_in_deals() - for timestamp, field, segment, descendants in product(timestamps, fields, segments, descendants_list): - if segment == 'all_deals' or descendants[0] in eligible_nodes_for_segs.get(segment, []): - fetch_deal_rollup_for_prnt_dr_rec(time_context, - descendants, - field, - segment, - config, - timestamp_info, - db, - cache, - prnt_node, - prev_periods=prev_periods) - - return cache - - -def fetch_deal_rollup_for_prnt_dr_rec(time_context,descendants,field,segment,config,timestamp_info,db,cache,prnt_node, - prev_periods=[]): - """ - fetch record that is sum of deal amount field for deals matching a filter. - We will use the parent of the node for fetching the information and the filter will be - applied to the parent of the node. If the node itself is a parent node then the filter will - be applied to that particular node only - - Arguments: - time_context {time_context} -- fm period, components of fm period - deal period, close periods of deal period - deal expiration timestamp, - relative periods of component periods - ('2020Q2', ['2020Q2'], - '2020Q2', ['201908', '201909', '201910'], - 1556074024910, - ['h', 'c', 'f']) - descendants {tuple} -- (node, [children], [grandchildren]) - fetches data for node - using children/grandkids to compute sums - ('A', ['B, 'C'], ['D', 'E']) - field {str} -- field name - 'commit' - segment {str} -- segment name - 'all deals' - config {Config} -- instance of Config - - Keyword Arguments: - timestamp_info {tuple} -- (ts to get data as of, - ts dlf DR recs expired as of, - ts deal DR recs expired as of, - ts to reject data from before (None to accept any level of staleness)) - if None passed, gets most recent record - (default: {None}) - (1556074024910, 1556074024910, 1556074024910) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - (pass in to avoid overhead when fetching many times) - cache {dict} -- dict to hold records fetched by (default: {None}) - (used to memoize fetching many recs) - prnt_node {str} -- parent of the particular node. Value will same as node when it's already the - parent. prnt_node is required for prnt_DR fields - - Returns: - dict -- a single fm record - """ - # - node, _, _ = descendants - current_segment_dtls = config.segments[segment] - period, comp_periods = time_context.fm_period, time_context.component_periods - timestamp, dlf_expiration, deal_expiration, _ = timestamp_info - found = True - sum_field = config.segment_amount_fields.get(segment, config.fields[field]['sum_field']) - seg_filter = config.segment_filters.get(segment) - name, crit, ops = config.fields[field]['crit_and_ops'] - crit = {'$and': [crit, seg_filter]} - if segment and segment != 'all_deals' and sum_field != config.fields[field]['sum_field']: - ops = [(sum_field, sum_field, '$sum')] - # Overriding the segment field with the overrided field in config - if 'sum_field_override' in current_segment_dtls and field in current_segment_dtls['sum_field_override']: - if segment != "all_deals": - ovveride_field = current_segment_dtls['sum_field_override'][field] - name, crit, ops = config.fields[ovveride_field]['crit_and_ops'] - sum_field = config.fields[ovveride_field]['sum_field'] - crit = {'$and': [crit, seg_filter]} - else: - total_value = 0 - # If sum_field_override in all_deals segment, then we are overriding the field with sum of all the segment values for that field. - for seg, segment_dtls in config.segments.items(): - if seg != "all_deals": - seg_filter = config.segment_filters.get(seg) - ovveride_field = segment_dtls['sum_field_override'][ - field] if 'sum_field_override' in segment_dtls else field - name, crit, ops = config.fields[ovveride_field]['crit_and_ops'] - sum_field = config.fields[ovveride_field]['sum_field'] if 'sum_field_override' in segment_dtls else \ - segment_dtls['sum_field'] - crit = {'$and': [crit, seg_filter]} - val = fetch_prnt_DR_deal_totals((time_context.deal_period, time_context.close_periods), - node, - ops, - config.deal_config, - prnt_node, - filter_criteria=crit, - timestamp=timestamp if timestamp else time_context.deal_timestamp, - db=db, - cache=cache, - prev_periods=prev_periods or time_context.prev_periods).get(sum_field, 0) - total_value += len(val) if isinstance(val, list) else val - res = {'period': period, - 'segment': segment, - 'val': total_value, - 'by': 'system', - 'how': 'deal_rollup', - 'found': found, - 'timestamp': timestamp, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - return res - - try: - val = fetch_prnt_DR_deal_totals((time_context.deal_period, time_context.close_periods), - node, - ops, - config.deal_config, - prnt_node, - filter_criteria=crit, - timestamp=None, - db=db, - cache=cache, - prev_periods=prev_periods or time_context.prev_periods).get(sum_field, 0) - except: - val, found = 0, False - - res = {'period': period, - 'segment': segment, - 'val': len(val) if isinstance(val,list) else val, - 'by': 'system', - 'how': 'deal_rollup', - 'found': found, - 'timestamp': timestamp, - 'node': node, - 'field': field} - if cache is not None: - cache[(period, node, field, segment, timestamp)] = res - return res - -def _bulk_fetch_ai_recs(time_context, - descendants_list, - fields, - segments, - config, - timestamp_info, - db, - cache, - get_all_segments=False, - round_val=True, - bucket_fields=[], - bucket_forecast_field=[]): - """ - fetch many prediction records for multiple nodes and fields - returns nothing, just puts records in cache - """ - # NOTE: much of this logic is duplicated in _fetch_ai_rec - # if you are changing logic here, be sure to also change in the non bulk version of the function - try: - nodes = [node for node, _, _ in descendants_list] - period = time_context.fm_period - timestamp, _, _, _ = timestamp_info - FM_COLL = read_from_collection(period[:4], timestamp=timestamp, call_from='_bulk_fetch_ai_recs', - config=config) - fm_data_collection = db[FM_COLL] - - # NOTE: taking a step backwards here until we have augustus monthly - # ai numbers are just taken straight from gbm, and are no longer the sum of component periods - # this might cause some inconsistencies for tenants with monthly predictions, so we should be cautious here - if bucket_forecast_field and len(bucket_forecast_field) > 0: - match = {'period': {'$in': [period]}, - 'node': {'$in': nodes}, - 'field': {'$in': bucket_forecast_field}} - if not get_all_segments: - match['segment'] = {'$in': segments} - if timestamp is not None: - match['timestamp'] = {'$lte': timestamp} - sort = {'timestamp': 1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'segment': '$segment', - 'field': '$field'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$last': '$val'}} - regroup = {'_id': {'node': '$_id.node', - 'field': '$_id.field'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$sum': '$val'}} - - regroup['_id']['segment'] = '$_id.segment' - else: - match = {'period': {'$in': [period]}, - 'node': {'$in': nodes}, - 'field': {'$in': fields}} - if not get_all_segments: - match['segment'] = {'$in': segments} - if timestamp is not None: - match['timestamp'] = {'$lte': timestamp} - sort = {'timestamp': 1} - group = {'_id': {'period': '$period', - 'node': '$node', - 'segment': '$segment', - 'field': '$field'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$last': '$val'}} - regroup = {'_id': {'node': '$_id.node', - 'field': '$_id.field'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$sum': '$val'}} - - regroup['_id']['segment'] = '$_id.segment' - - pipeline = [{'$match': match}, - {'$sort': sort}, - {'$group': group}, - {'$group': regroup}] - if FM_COLL == FM_LATEST_DATA_COLL: - regroup = {'_id': {'node': '$node', - 'field': '$field'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$sum': '$val'}} - regroup['_id']['segment'] = '$segment' - pipeline = [{'$match': match}, - {'$group': regroup}] - if config.debug: - logger.info('bulk_fetch_ai_rec pipeline: %s from %s' % (pipeline, FM_COLL)) - - db_recs = {(rec['_id']['node'], rec['_id']['field'], rec['_id']['segment']): rec - for rec in fm_data_collection.aggregate(pipeline, allowDiskUse=True)} - rollup_segments = config.rollup_segments - - for node, field in product(nodes, fields): - aggregated_val = 0 - for segment in segments: - val = get_nested(db_recs, [(node, field, segment), 'val'], 0) - #nan check - val = val if val==val else 0 - if field in bucket_fields: - val = get_nested(db_recs, [(node, bucket_forecast_field[0], field), 'val'], 0) - if segment in rollup_segments: - aggregated_val += val - cache[(period, node, field, segment, timestamp)] = {'period': period, - 'segment': segment, - 'val': round(val) if round_val else val, - 'by': get_nested(db_recs, [(node, field, segment), 'by'], 'system'), - 'how': 'ai', - 'found': (node, field, segment) in db_recs, - 'timestamp': get_nested(db_recs, [(node, field, segment), 'timestamp'], 0), - 'node': node, - 'field': field} - - if cache.get((period, node, field, 'all_deals', timestamp), None) is None or config.rollup_ai_segments: - cache[(period, node, field, 'all_deals', timestamp)] = {} - cache[(period, node, field, 'all_deals', timestamp)]['val'] = round(aggregated_val) if round_val else aggregated_val - return cache - except Exception as e: - logger.exception(e) - raise e - -def fetch_fm_rec_in_period(descendants, - fields, - segments, - config, - eligible_nodes_for_segs, - fm_rec_period, - cache, - timestamp = None, - time_context_list = [] - ): - - logger.info('Processing the data for the period: {}.....'.format(fm_rec_period)) - time_context = get_time_context(fm_rec_period, config=None, quarter_editable=config.quarter_editable) - db = sec_context.tenant_db - res = bulk_fetch_recs_by_timestamp(time_context, - descendants, - fields, - segments, - config, - eligible_nodes_for_segs, - [timestamp], - db = db, - time_context_list = time_context_list - ) - - cache.update(res) - return cache - -def _fetch_fm_rec(args): - # dumb overhead function because pool.map only takes a single argument and cant do lambdas - return fetch_fm_rec(*args) - -def _bulk_fetch_user_entered_recs_pool(args): - # overhead function because pool.map only takes a single argument and cant do lambdas - return _bulk_fetch_user_entered_recs(*args) - -def _bulk_fetch_ai_recs_pool(args): - # overhead function because pool.map only takes a single argument and cant do lambdas - return _bulk_fetch_ai_recs(*args) - -def handle_dr_fields_pool(args): - # overhead function because pool.map only takes a single argument and cant do lambdas - return handle_dr_fields(*args) - -def _bulk_fetch_prnt_dr_recs_pool(args): - # overhead function because pool.map only takes a single argument and cant do lambdas - return _bulk_fetch_prnt_dr_recs(*args) - -def _bulk_period_conditional_rec_pool(args): - # overhead function because pool.map only takes a single argument and cant do lambdas - return _bulk_period_conditional_rec(*args) - -def _bulk_fetch_formula_recs_pool(args): - # overhead function because pool.map only takes a single argument and cant do lambdas - return _bulk_fetch_formula_recs(*args) - -# fetch cached records from snapshot -def fetch_cached_snapshot(period, node, mobile=None, config=None, db=None, segment=None, force_retrieve_cache=False, - timestamp=None): - if not db: - db = sec_context.tenant_db - config = config if config else FMConfig() - allowed_periods = config.snapshot_config.get("allowed_periods", []) - yearly_view = config.snapshot_config.get('yearly_view', False) - if yearly_view: - if len(period) == 6 and 'Q' not in period: - # This code can be used to find the year to which it belongs in case we have month as a period - for quarter, months in get_available_quarters_and_months(PeriodsConfig()).iteritems(): - if period in months: - qtr = quarter - current_year = qtr[0:4] - elif len(period) == 6: - # This code can be used to find the year to which it belongs in case we have quarter as a period - current_year = period[0:4] - else: - current_year = period - if period != current_year: - quarter_and_its_component_period = get_period_and_close_periods(current_year, deal_svc=True) - qtr_comp_period_lst = [] - for sublist in quarter_and_its_component_period: - qtr_comp_period_lst.append(sublist[0]) - for mth in sublist[1]: - qtr_comp_period_lst.append(mth) - allowed_periods += qtr_comp_period_lst - else: - allowed_periods.append(period) - if period not in allowed_periods: - periods_to_prefetch = get_periods_editable(config.periods_config, - future_qtr_editable=config.future_qtrs_prefetch_count, - past_qtr_editable=config.past_qtrs_prefetch_count) - for per in periods_to_prefetch: - allowed_periods.append(per) - _, component_periods = get_period_and_component_periods(per) - allowed_periods += component_periods - if config.deal_config.weekly_fm: - weeks = [week_prd.mnemonic for week_prd in weekly_periods(per)] - allowed_periods += weeks - is_cached = False - if period in allowed_periods: - is_cached = True - if not is_cached: - return {} - coll = SNAPSHOT_ROLE_COLL if config.has_roles else SNAPSHOT_COLL - if timestamp and config.snapshot_config.get("snapshot_historical", False): - coll = SNAPSHOT_HIST_ROLE_COLL if config.has_roles else SNAPSHOT_HIST_COLL - snapshot_coll = db[coll] - if mobile: - coll = MOBILE_SNAPSHOT_ROLE_COLL if config.has_roles else MOBILE_SNAPSHOT_COLL - snapshot_coll = db[coll] - if segment: - criteria = {"node": node, "period": period, "segment": segment} - else: - criteria = {"node": node, "period": period, "segment": 'all_deals'} - if mobile: - projections = {"mobile_snapshot_data": 1, "last_updated_time": 1, "mobile_snapshot_stale": 1, "how": 1} - else: - projections = {"snapshot_data": 1, "last_updated_time": 1, "snapshot_stale": 1, "how": 1} - if not mobile and timestamp and config.snapshot_config.get("snapshot_historical", False): - try: - timestamp = int(timestamp) - dt = epoch(timestamp).as_datetime() - as_of_date = dt.strftime("%Y-%m-%d") - except: - as_of_date = '' - criteria['as_of_date'] = as_of_date - if config.has_roles: - user_role = sec_context.get_current_user_role() - if user_role not in config.get_roles: - user_role = DEFAULT_ROLE - criteria.update({'role': user_role}) - response = snapshot_coll.find_one(criteria, projections) - result = {} - if response: - # mobile cached data - if force_retrieve_cache: - if response.get('snapshot_data'): - result = response['snapshot_data'] - result['updated_at'] = response['last_updated_time'] - return result - - if mobile: - if response.get('mobile_snapshot_stale', False): - if response.get('how', 'chipotle') == 'ui_update': - return {} - if response.get('mobile_snapshot_data'): - result = add_collection_nodes(response['mobile_snapshot_data'], config) - result['updated_at'] = response['last_updated_time'] - return result - - if response.get('how', 'chipotle') == 'UE_update' and \ - not response.get('snapshot_stale'): - if response.get('snapshot_data'): - result = response['snapshot_data'] - result['updated_at'] = response['last_updated_time'] - logger.info("returning from cache as data was regenerated as part of conditional snapshot for UE update") - return result - - # web cached data - if response.get('snapshot_stale', False): - if response.get('how', 'chipotle') == 'ui_update' and \ - not config.snapshot_config.get('show_cached_only', False): - return {} - if response.get('snapshot_data'): - result = response['snapshot_data'] - result['updated_at'] = response['last_updated_time'] - if response.get('how', 'chipotle') == 'ui_update' and response.get('snapshot_stale', False): - result['data_stale'] = True - return result - - - -def add_collection_nodes(mobile_snapshot_data, config): - data = {} - column_data = mobile_snapshot_data.get('data') - for column in config.columns: - data[column] = column_data - - mobile_snapshot_data['data'] = data - return mobile_snapshot_data - - -def remove_collection_nodes(resp): - cached_resp = {} - if len(resp.get('data', {}).values()) > 0: - data = resp.get('data', {}).values().pop() - cached_resp['data'] = data - cached_resp['metadata'] = resp.get('metadata') - return cached_resp - -# fetch cached records for forecast_summary from snapshot -def fetch_cached_forecast_summary(period, node, segment, config=None, db=None, - current_data_only=False, new_home_page=False): - if not db: - db = sec_context.tenant_db - config = config if config else FMConfig() - allowed_periods = config.snapshot_config.get("allowed_periods", []) - show_stale_limit = config.snapshot_config.get("show_stale_limit", 15) - yearly_view = config.snapshot_config.get('yearly_view', False) - if yearly_view: - if len(period) == 6 and 'Q' not in period: - # This code can be used to find the year to which it belongs in case we have month as a period - for quarter, months in get_available_quarters_and_months(PeriodsConfig()).iteritems(): - if period in months: - qtr = quarter - current_year = qtr[0:4] - elif len(period) == 6: - # This code can be used to find the year to which it belongs in case we have quarter as a period - current_year = period[0:4] - else: - current_year = period - if period != current_year: - quarter_and_its_component_period = get_period_and_close_periods(current_year, deal_svc=True) - qtr_comp_period_lst = [] - for sublist in quarter_and_its_component_period: - qtr_comp_period_lst.append(sublist[0]) - for mth in sublist[1]: - qtr_comp_period_lst.append(mth) - allowed_periods += qtr_comp_period_lst - else: - allowed_periods.append(period) - if period not in allowed_periods: - periods_to_prefetch = get_periods_editable(config.periods_config, - future_qtr_editable=config.future_qtrs_prefetch_count, - past_qtr_editable=config.past_qtrs_prefetch_count) - for per in periods_to_prefetch: - allowed_periods.append(per) - _, component_periods = get_period_and_component_periods(per) - allowed_periods += component_periods - is_cached = False - if period in allowed_periods: - is_cached = True - if not is_cached: - return {} - coll = SNAPSHOT_ROLE_COLL if config.has_roles else SNAPSHOT_COLL - snapshot_coll = db[coll] - criteria = {"node": node, "period": period, "segment": segment} - if config.has_roles: - user_role = sec_context.get_current_user_role() - if user_role not in config.get_roles: - user_role = DEFAULT_ROLE - criteria.update({'role': user_role}) - projections = {"forecast_summary_data": 1, "last_updated_time": 1, "forecast_summary_stale": 1, "how": 1} - if new_home_page: - projections = {"new_homepage_fs": 1, "last_updated_time": 1, "new_homepage_fs_stale": 1, "how": 1} - if config.debug: - logger.info("fetch_cached_forecast_summary criteria %s projection %s" % (criteria, - projections)) - response = snapshot_coll.find_one(criteria, projections) - result = {} - if response: - if new_home_page: - if response.get('new_homepage_fs_stale', False): - if response.get('how', 'chipotle') == 'ui_update': - return {} - elif ((get_now().as_datetime() - epoch( - response['last_updated_time']).as_datetime()).total_seconds()) / 60 > int(show_stale_limit): - return {} - if response.get('new_homepage_fs'): - result = response['new_homepage_fs'] - result['updated_at'] = response['last_updated_time'] - else: - if response.get('forecast_summary_stale', False): - if response.get('how', 'chipotle') == 'ui_update': - return {} - elif ((get_now().as_datetime() - epoch(response['last_updated_time']).as_datetime()).total_seconds()) / 60 > int(show_stale_limit): - return {} - if response.get('forecast_summary_data'): - result = response['forecast_summary_data'] - # to be removed on additing data for new_home_page - if result.get("views_data") and not (result.get("current") and result.get("changes")): - return {} - # this is to fix the records. - if not current_data_only and not result.get("changes"): - return {} - result['update_at'] = response['last_updated_time'] - return result - - -def get_all_source_fields(config, field): - source_fields = config.fields[field].get('source', []) - child_source_fields = [] - - for child_field in source_fields: - child_source_fields.extend(get_all_source_fields(config, child_field)) - - return source_fields + child_source_fields - - -def fetch_nextq_pipeline(timestamp, - node, - fields=None, - db=None): - """ - fetch next quarter pipeline data that occured. - Arguments: - node {str} -- Global#00538000005ac6T - timestamp {str} -- 2023-10-30 - Keyword Arguments: - db {pymongo.database.Database} -- instance of tenant_db - (default: {None}) - if None, will create one - Returns: - next quarter pipeline aggregation - """ - nextq_collection = db[NEXTQ_COLL] if db else sec_context.tenant_db[NEXTQ_COLL] - - match = {'node' : node, 'timestamp': timestamp} - pipeline = [{'$match': match}] - logger.info("PIPELINE FOR NEXT QUARTER : {}".format(pipeline)) - result = nextq_collection.aggregate(pipeline, allowDiskUse=True) - logger.info("result {}".format(result)) - - return result - -def fetch_waterfall_fm_sync_data(periods, nodes, field, segment): - fm_data_collection = sec_context.tenant_db[FM_COLL] - criteria = { - 'period': {'$in': periods}, - 'node': {'$in': nodes}, - 'field': field, - 'segment': segment - } - sort = { - 'timestamp': 1 - } - group = {'_id': {'period': '$period', - 'node': '$node', - 'segment': '$segment', - 'field': '$field'}, - 'timestamp': {'$last': '$timestamp'}, - 'by': {'$last': '$by'}, - 'val': {'$last': '$val'}} - # regroup = {'_id': {'node': '$_id.node', - # 'field': '$_id.field'}, - # 'timestamp': {'$last': '$timestamp'}, - # 'by': {'$last': '$by'}, - # 'val': {'$sum': '$val'}} - # - # regroup['_id']['segment'] = '$_id.segment' - - pipeline = [{'$match': criteria}, - {'$sort': sort}, - {'$group': group}, - ] - - return {(rec['_id']['period'], rec['_id']['node'], rec['_id']['field'], rec['_id']['segment']): rec['val'] - for rec in fm_data_collection.aggregate(pipeline, allowDiskUse=True)} - -def convert_excel_date_to_eod_timestamp(excel_date): - time = xl2datetime_ttz(excel_date) - final_date = time.replace(hour=23, minute=59, second=59) - timestamp = datetime2epoch(final_date) - return timestamp - -def fetch_fields_by_type(config, time_context, descendants, fields, ignore_recency_for): - adhoc_field_mapping = {} - fields_by_type = {} - for field in set(fields): - if 'hist_field' in config.fields[field] and 'h' in time_context.relative_periods and not config.show_raw_data_in_trend: - # historic period, switch to using true up field if it exists - field_type = 'PC' - if config.quarter_editable: - if not all(elem == 'h' for elem in time_context.relative_periods): - field_type = config.fields[field]['type'] - else: - field_type = config.fields[field]['type'] - if field_type == 'NC' and 'hist_field' in config.fields[field] and config.show_raw_data_in_trend: # get raw field in case of NC - if descendants: - raw_field, _ = config.fields[field]['source'] - else: - _,raw_field = config.fields[field]['source'] - adhoc_field_mapping[raw_field]=field - field_type = config.fields[raw_field]['type'] - field = raw_field - - if field_type not in fields_by_type: - fields_by_type[field_type] = [] - if field not in fields_by_type[field_type]: - fields_by_type[field_type].append(field) - #else: - # fields_by_type[field_type].append(field) - - if adhoc_field_mapping: - for raw_field,field in adhoc_field_mapping.items(): - if field in fields: - fields.remove(field) - if field in ignore_recency_for: - ignore_recency_for.remove(field) - ignore_recency_for.append(raw_field) - fields.append(raw_field) - - if config.config.get('bucket_fields'): - field_set, remove_set = set(fields_by_type['AI']), set(config.config.get('bucket_fields')) - fields_by_type['AI'] = list(field_set - remove_set) - - return fields_by_type - - -def fetch_qtd_actual_data(period, node, as_of, config, time_context, read_weeks, historical_timestamp = None): - week_nums = len(read_weeks) - qtd_actual_timestamps = {} - if week_nums == 0: - return {}, qtd_actual_timestamps - waterfall_config = config.config.get('waterfall_config', {}) - QTDActual = waterfall_config.get('QTDActual', {}) - waterfall_pivot = node.split('#')[0] if '#' in node else None - if not waterfall_pivot or (waterfall_pivot not in QTDActual): - return default_dynamics_data(week_nums), qtd_actual_timestamps - - pivot_config = QTDActual[waterfall_pivot] - segment = pivot_config.get('segment', None) - field = pivot_config.get('field', None) - multiplier = pivot_config.get('multiplier', 1000) - if not segment or not field: - return default_dynamics_data(week_nums), qtd_actual_timestamps - - descendants = fetch_descendants(as_of, [node], levels=2, include_children=False) - descendants = [(rec['node'], rec['descendants'][0], rec['descendants'][1]) for rec in descendants] - ignore_recency_for = config.ignore_recency_for_fields - fields_by_type = fetch_fields_by_type(config, time_context, descendants, [field], ignore_recency_for) - cache = {} - fm_recs = {} - timestamps = [] - data = {} - epoch_now = epoch().as_epoch() - current_year, current_month, current_day = epoch2datetime(epoch_now).timetuple()[:3] - current_eod_xl = get_eod(epoch(current_year, current_month, current_day)).as_xldate() - if historical_timestamp: - current_eod_xl = epoch(historical_timestamp).as_xldate() - for week_range in read_weeks: - begin_date_timestamp = convert_excel_date_to_eod_timestamp(week_range['begin']) - timestamps.append(begin_date_timestamp) - if week_range['begin'] >= current_eod_xl: - break - - if len(timestamps) == len(read_weeks): - timestamps.append(epoch(read_weeks[-1]['end']).as_epoch()) - - try: - fm_recs = bulk_fetch_fm_recs_history(time_context, - descendants, - [field], - [segment], - config, - timestamps, - cache=cache, - ignore_recency_for=ignore_recency_for, - fields_by_type=fields_by_type, - get_all_segments=False) - except Exception as e: - logger.error("Failed! %s" % (e)) - - no_of_req_weeks = len(timestamps) - for week_range in read_weeks: - if no_of_req_weeks: - begin_date_timestamp = timestamps[len(timestamps)-no_of_req_weeks] - data[week_range['label']] = fm_recs[(period, node, field, segment, begin_date_timestamp)]['val'] / multiplier - qtd_actual_timestamps[week_range['label']] = fm_recs[(period, node, field, segment, begin_date_timestamp)]['timestamp'] - no_of_req_weeks -= 1 - else: - data[week_range['label']] = 0 - - if no_of_req_weeks: - end_date_timestamp = timestamps[len(timestamps) - no_of_req_weeks] - data['week14'] = fm_recs[(period, node, field, segment, end_date_timestamp)]['val'] / multiplier - qtd_actual_timestamps['week14'] = fm_recs[(period, node, field, segment, end_date_timestamp)][ - 'timestamp'] - - return data, qtd_actual_timestamps - -def fetch_prev_qtd_data(period, node, config): - weeks = weekly_periods(period) - weeks.sort(key = lambda x:x.begin) - week_nums = len(weeks) - - waterfall_config = config.config.get('waterfall_config', {}) - Prev_QTD = waterfall_config.get('Prev_QTD', {}) - waterfall_pivot = node.split('#')[0] if '#' in node else None - if not waterfall_pivot or (waterfall_pivot not in Prev_QTD): - return default_dynamics_data(week_nums) - - pivot_config = Prev_QTD[waterfall_pivot] - segment = pivot_config.get('segment', None) - field = pivot_config.get('field', None) - multiplier = pivot_config.get('multiplier', 1000) - - cache = {} - for week in weeks: - week_mnemonic = week.mnemonic - time_context = get_time_context(week_mnemonic) - descendants = (node, [], []) - fetch_fm_rec(time_context, descendants,field, segment, config, cache=cache) - - res = {} - for i in range(1, 14): - week_key = "week"+str(i) - try: - val = float(cache[(weeks[min(i, 12)].mnemonic, node, field, segment, None)]['val']) - except Exception: - val = 0 - res[week_key] = val / multiplier - - return res - - -def fetch_immediate_childs_status(immediate_childs, nodes_label, period, y_week, field, db = None): - try: - waterfall_data_collection = db[WATERFALL_COLL] if db else sec_context.tenant_db[WATERFALL_COLL] - criteria = { - 'node':{'$in': immediate_childs}, - 'period': period, - 'y_week': y_week, - 'field': field - } - projection = { - 'node': 1, - 'is_submitted':1 - } - pipeline = [ - {'$match':criteria}, - {'$project': projection}, - { - '$group':{ - '_id': '$node', - 'is_submitted': {'$first': '$is_submitted'} - } - } - ] - - documents = list(waterfall_data_collection.aggregate(pipeline)) - final_data = {} - for doc in documents: - node_id = doc.get('_id') - is_submitted = doc.get('is_submitted', False) - if is_submitted is None: - is_submitted = False - final_data[node_id] = { - 'label': nodes_label.get(node_id), - 'is_submitted': is_submitted - } - for node_id , label in nodes_label.items(): - if node_id not in final_data: - final_data[node_id] = { - 'label': label, - 'is_submitted': False - } - return final_data - except Exception as ex: - logger.exception(ex) - return None - - -def fetch_immediate_child_overview(node, period, y_week, field, as_of, db = None): - try: - descendants = fetch_descendants(as_of, [node], levels=2, include_children=False) - descendants = [(rec['node'], rec['descendants'][0], rec['descendants'][1], rec['parent']) for rec in descendants] - if not descendants[0][2]: - return [] - immediate_childs = list(descendants[0][1].keys()) - nodes_label = fetch_labels(as_of, immediate_childs) - final_data = fetch_immediate_childs_status(immediate_childs, nodes_label, period, y_week, field) - - final_output = [{'node': node_id, 'label': data['label'], 'is_submitted': data['is_submitted'] } - for node_id , data in final_data.items()] - - return final_output - except Exception as ex: - logger.exception(ex) - return None - -def fetch_outweek_data_submit_status(node, period, y_week, field, db = None): - try: - result_node = node.rsplit('#', 1)[1] - if result_node == 'Global' or result_node == '!': - return {'is_submitted': None} - - waterfall_data_collection = db[WATERFALL_COLL] if db else sec_context.tenant_db[WATERFALL_COLL] - criteria = { - 'node':node, - 'period': period, - 'y_week': y_week, - 'field': field, - 'is_submitted': {'$exists': True} - } - result = waterfall_data_collection.find_one(criteria, {'is_submitted': 1}) - is_submitted_status = result.get('is_submitted') if result else False - if is_submitted_status not in [True, False]: - return {'is_submitted': False} - return {'is_submitted': is_submitted_status} - except Exception as ex: - logger.exception(ex) - return None - -def fetch_track_data(period, nodes, db = None, track_field= 'track'): - try: - waterfall_data_collection = db[WATERFALL_COLL] if db else sec_context.tenant_db[WATERFALL_COLL] - criteria = {'period': period, 'field': track_field, 'node':{'$in': nodes}, 'y_week': None} - cursor = waterfall_data_collection.find(criteria, { '_id': 0, 'node': 1, 'val': 1, 'x_week': 1} ) - default_track_data = default_dynamics_data() - data = {} - for doc in cursor: - node_id = doc['node'] - value = doc['val'] - week = doc['x_week'] - if node_id not in data: - data[node_id] = default_track_data.copy() - if week not in default_track_data: - continue - data[node_id][week] = value - - return data - except Exception as ex: - logger.exception(ex) - return None - - -def fetch_waterfall_data(period, node, as_of, db=None, read_weeks=[], - config={}, time_context={}, - week_nums=13, is_mx_track=False, - historical_timestamp=None): - waterfall_data_collection = db[WATERFALL_COLL] if db else sec_context.tenant_db[WATERFALL_COLL] - if historical_timestamp: - waterfall_data_collection = db[WATERFALL_HISTORY_COLL] if db else sec_context.tenant_db[WATERFALL_HISTORY_COLL] - criteria = {'period': period, 'node': node} - if historical_timestamp: - criteria.update({ - 'last_updated_at': {'$lte': historical_timestamp} - }) - sort = { - 'last_updated_at' : -1 - } - group = { - "_id": { - "node": "$node", - "period": "$period", - "field": "$field", - "x_week": "$x_week", - "y_week": "$y_week" - }, - "latest_record": {"$first": "$$ROOT"} - } - replace_root = {"$replaceRoot": {"newRoot": "$latest_record"}} - - pipeline = [ - {'$match': criteria}, - {'$sort': sort}, - {'$group' : group}, - replace_root - ] - - logger.info("pipeline for waterfall historical fetch is %s", pipeline) - cursor = waterfall_data_collection.aggregate(pipeline) - else: - cursor = waterfall_data_collection.find(criteria) - data = {'track': {}, 'weeksData': {}, 'rollover': {}} - if is_mx_track: - data.update({'mx_track': {}}) - - # Fetch timestamp of each weekdata record separately - weekdata_last_updated = {} - weekdata_is_submitted = {} - - for doc in cursor: - field = doc['field'] - x_week = doc['x_week'] - y_week = doc['y_week'] - val = doc['val'] - last_updated = doc['last_updated_at'] - is_submitted = doc.get('is_submitted', False) - - if field == 'track': - data['track'][x_week] = val - elif field == 'mx_track': - if is_mx_track: - data['mx_track'][x_week] = val - elif field == 'weeksData': - if y_week not in data['weeksData']: - data['weeksData'][y_week] = {} - if y_week not in weekdata_last_updated: - weekdata_last_updated[y_week] = {} - if y_week not in weekdata_is_submitted: - weekdata_is_submitted[y_week] = {} - data['weeksData'][y_week][x_week] = val - weekdata_last_updated[y_week][x_week] = last_updated - weekdata_is_submitted[y_week][x_week] = is_submitted - elif field == 'rollover': - data['rollover'][y_week] = val - - logger.info("FINAL DATA TO RESTRUCTURE : {}".format(data)) - - restructured_data = {'track': {}, 'weeksData': {}, 'rollover': {}} - if is_mx_track: - restructured_data.update({'mx_track': {}}) - qtd_actual_data, qtd_actual_timestamps = fetch_qtd_actual_data(period, node, as_of, config, time_context, - read_weeks=read_weeks, - historical_timestamp = historical_timestamp) - restructured_data['QTDActual'] = qtd_actual_data - last_updated_qtd_actual_timestamp = None - - prev_qtd_data = fetch_prev_qtd_data(period, node, config) - restructured_data['prev_QTD'] = prev_qtd_data - - for i in range(week_nums): - week_key = "week{}".format(i+1) - if week_key in qtd_actual_timestamps and qtd_actual_timestamps[week_key] > epoch(read_weeks[i]['begin']).as_epoch(): - last_updated_qtd_actual_timestamp = qtd_actual_timestamps[week_key] - - if last_updated_qtd_actual_timestamp: - last_updated_qtd_actual_timestamp = epoch(last_updated_qtd_actual_timestamp).as_epoch() - - # Initialize track with zeros for all weeks - for i in range(1, week_nums + 1): # weeks range from 1 to 13 - week_key = "week{}".format(i) - restructured_data['track'][week_key] = data['track'].get(week_key, 0) - - # Initialize mx_track with zeros for all weeks - if is_mx_track: - for i in range(1, week_nums + 1): # weeks range from 1 to 13 - week_key = "week{}".format(i) - restructured_data['mx_track'][week_key] = data['mx_track'].get(week_key, 0) - - # Initialize rollover with zeros for all weeks - for i in range(week_nums + 1): # weeks range from 1 to 13 - week_key = "week{}".format(i) - restructured_data['rollover'][week_key] = data['rollover'].get(week_key, 0) - - # declare var to check whether current intersection is editable - is_current_intersection_editable = True - - # Restructure weeksData - - for y_idx in range(week_nums + 1): - y_week = "week{}".format(y_idx) - restructured_week = OrderedDict() - start_week = 1 - end_week = week_nums + 1 - - # week numbers start from 1 - for x_idx in range(start_week, end_week): - week_key = "week{}".format(x_idx) - if x_idx < y_idx: - y_prev_week = "week{}".format(y_idx-1) - restructured_week[week_key] = data['weeksData'][y_prev_week][week_key] - continue - restructured_week[week_key] = data['weeksData'].get(y_week, {}).get(week_key, 0) - - # Intersection cell manipulation - if y_idx in range(1, week_nums + 1) and y_week == week_key: - now = get_now() - wk_len = len(read_weeks) - if y_idx <= wk_len: - # Current week is always Thursday to next Wednesday - active_week_begin = epoch(read_weeks[y_idx - 1]['begin']).as_datetime() + timedelta(days=3) - active_week_end = epoch(read_weeks[min(y_idx, wk_len-1)]['end']).as_datetime() - timedelta(days=4) - if y_idx == wk_len: - active_week_end = epoch(read_weeks[-1]['end']).as_datetime() - is_active_week = active_week_begin <= now.as_datetime() <= active_week_end - if historical_timestamp: - is_active_week = active_week_begin <= epoch(historical_timestamp).as_datetime() <= active_week_end - - # If value is already submitted, continue - if weekdata_is_submitted.get(y_week, {}).get(week_key, False): - if is_active_week: - is_current_intersection_editable = False - - if y_idx <= len(read_weeks) and now.as_xldate() > read_weeks[y_idx-1]['end']: - # Find timestamps of data - week_data_timestamp = weekdata_last_updated.get(y_week, {}).get(week_key, 0) - next_week_key = "week{}".format(x_idx + 1) - currentQTD = restructured_data['QTDActual'].get(week_key, 0) - nextQTD = restructured_data['QTDActual'].get(next_week_key, 0) - nextQTD_timestamp = qtd_actual_timestamps.get(next_week_key, 0) - nextMonday_timestamp = epoch(read_weeks[min(y_idx, wk_len-1)]['begin']).as_epoch() - if y_idx == wk_len: - nextMonday_timestamp = epoch(read_weeks[min(y_idx, wk_len - 1)]['begin']).as_epoch() + 7*24*3600*1000 - - # If weeksData value is overridden after nextQTD value population - # we need to show weeksData value directly, however, shouldn't be editable - now_epoch = now.as_epoch() - if historical_timestamp: - now_epoch = historical_timestamp - if nextMonday_timestamp <= nextQTD_timestamp < now_epoch and nextQTD > 0 and week_data_timestamp > nextQTD_timestamp: - if is_active_week: - is_current_intersection_editable = False - continue - # if "file is loaded", i.e., if QTD value for next Monday is present, - # show the difference between next week start and current week start - # make intersection non-editable if it is current week - elif nextMonday_timestamp <= nextQTD_timestamp < now_epoch and nextQTD > 0: - # For week1, value has to be ignored - if week_key == 'week1': - currentQTD = 0 - restructured_week[week_key] = nextQTD - currentQTD - if is_active_week: - is_current_intersection_editable = False - elif y_idx == wk_len: - restructured_week[week_key] = nextQTD - currentQTD - - data['weeksData'][y_week] = restructured_week - restructured_data['weeksData'][y_week] = data['weeksData'][y_week] - - - config = FMConfig() - waterfall_config = config.config.get('waterfall_config', {}) - fm_to_waterfall_sync = waterfall_config.get('fm_to_waterfall_sync', {}) - waterfall_pivot = node.split('#')[0] if '#' in node else None - if not waterfall_pivot or (waterfall_pivot not in fm_to_waterfall_sync): - - restructured_data['dynamicsData'] = default_dynamics_data(week_nums) - return restructured_data, is_current_intersection_editable, last_updated_qtd_actual_timestamp - - pivot_config = fm_to_waterfall_sync[waterfall_pivot] - segment = pivot_config.get('segment', None) - field = pivot_config.get('field', None) - multiplier = pivot_config.get('multiplier', 1000) - if not segment or not field: - restructured_data['dynamicsData'] = default_dynamics_data(week_nums) - return restructured_data, is_current_intersection_editable, last_updated_qtd_actual_timestamp - - sum_field = config.segment_amount_fields.get(segment) - seg_filter = config.segment_filters.get(segment) - _, crit, _ = config.fields[field]['crit_and_ops'] - crit = {'$and': [crit, seg_filter]} - close_date_field = config.deal_config.close_date_field - - restructured_data['dynamicsData'] = get_waterfall_weekly_totals(period, node, read_weeks, close_date_field, sum_field, crit, multiplier) - - return restructured_data, is_current_intersection_editable, last_updated_qtd_actual_timestamp - -def default_dynamics_data(week_nums=13): - data = {} - for i in range(1, week_nums+1): - data['week{}'.format(i)] = 0 - return data - -def transform_week_string(input_str): - # Match the week number in the string - import re - match = re.match(r'week(\d+)', input_str) - if match: - week_number = int(match.group(1)) - return 'week{}'.format(week_number - 1) - else: - raise ValueError("Input string does not match the expected format") - - - -def get_forecast_schedule(nodes, db=None, get_dict={}): - try: - forecast_schedule_coll = db[FORECAST_SCHEDULE_COLL] if db else sec_context.tenant_db[FORECAST_SCHEDULE_COLL] - criteria = {'node_id': {'$in': nodes}} - projections = {"recurring": 1, - "unlockPeriod": 1, - "unlockFreq": 1, - "unlockDay": 1, - "unlocktime": 1, - "lockPeriod": 1, - "lockFreq": 1, - "lockDay": 1, - "locktime": 1, - "timeZone": 1, - "node_id": 1, - "lock_on": 1, - "unlock_on": 1, - "status": 1, - "non_recurring_timestamp": 1, - "status_non_recurring": 1 - } - response = forecast_schedule_coll.find(criteria, projections) - if get_dict: - return {res['node_id']: res for res in response} - return [res for res in response] - except Exception as ex: - logger.exception(ex) - return {'success': False, 'Error': str(ex)} - - -def get_requests(user_id=None, user_node=None, db=None): - try: - forecast_req_coll = db[FORECAST_UNLOCK_REQUESTS] if db else sec_context.tenant_db[FORECAST_UNLOCK_REQUESTS] - criteria = {} - if user_id is not None: - criteria = {"user_id": user_id} - if user_node: - criteria['user_node'] = user_node - response = forecast_req_coll.find(criteria, {"user_id": 1, "user_node": 1, - 'email': 1, 'action': 1}) - return {res['user_id']: res for res in response} - except Exception as ex: - logger.exception(ex) - return {'success': False, 'Error': str(ex)} - - -def get_user_level_schedule(user_id=None, user_node=None, db=None, get_user_by_list=None): - try: - user_level_schedule = db[USER_LEVEL_SCHEDULE] if db else sec_context.tenant_db[USER_LEVEL_SCHEDULE] - criteria = {} - if user_id is not None: - criteria = {"user_id": user_id} - if user_node: - criteria['node_id'] = user_node - response = user_level_schedule.find(criteria, {"user_id": 1, "email": 1, - 'node_id': 1, 'forecastTimestamp': 1, - 'forecastWindow': 1}) - result = [] - if get_user_by_list: - for res in response: - if res['user_id'] not in result: - result[res['user_id']] = [res] - else: - result[res['user_id']].append(res) - return result - - return {res['user_id'] + "_" + res['node_id']: res for res in response} - except Exception as ex: - logger.exception(ex) - return {'success': False, 'Error': str(ex)} - -def is_valid_week_string(input_str): - import re - match = re.match(r'week(\d+)', input_str) - if not match: - raise ValueError("Input string does not match the expected format") - -def find_req_periods(period, week_period = None): - req_periods = [] - if week_period is not None: - req_periods.append(week_period) - else: - quarter_weeks_range = weekly_periods(period) - for week_range in quarter_weeks_range: - req_periods.append(week_range.mnemonic) - return req_periods - -def find_recs_from_weekly_forecast_fm_coll(nodes, period, week_period, segments, field, db=None, coll_suffix=None): - coll_name = WEEKLY_FORECAST_FM_COLL - fm_weekly_forecast_coll = '_'.join([coll_name, coll_suffix]) if coll_suffix else coll_name - fm_weekly_forecast_collection = db[coll_name] if db else sec_context.tenant_db[fm_weekly_forecast_coll] - req_periods = find_req_periods(period, week_period) - - criteria = { - 'node': {'$in': nodes}, - 'period': {'$in': req_periods}, - 'segment' : {'$in': segments}, - 'field': field - } - projection = { - 'node': 1, - 'segment': 1, - 'val': 1, - 'period': 1, - '_id': 0 - } - pipeline = [ - {'$match':criteria}, - {'$project': projection} - ] - documents = list(fm_weekly_forecast_collection.aggregate(pipeline)) - return documents - - -def update_fm_weekly_snapshot_for_historical_avg(nodes, period, week_period, config, fm_weekly_segments, fm_weekly_snapshot): - historical_field = 'historical_avg' - fm_weekly_forecast_waterfall_config = config.config.get('weekly_forecast_waterfall_config', {}) - field_label = fm_weekly_forecast_waterfall_config.get('historical_avg', {}).get('label', historical_field) - field_format = fm_weekly_forecast_waterfall_config.get('historical_avg', {}).get('format', 'percentage') - field_editable = False - documents = find_recs_from_weekly_forecast_fm_coll(nodes, period, week_period, fm_weekly_segments, historical_field) - for doc in documents: - required_keys = ['segment', 'val', 'period'] - if not all(k in doc for k in required_keys): - continue - seg = doc['segment'] - value = doc['val'] - mnemonic_period = doc['period'] - node_key = doc['node'] - node_seg_key = node_key + '_' + seg - if historical_field not in fm_weekly_snapshot[node_seg_key]['fields']: - fm_weekly_snapshot[node_seg_key]['fields'][historical_field] = { - 'weeks_period_data_val': {}, - 'editable': field_editable, - 'format': field_format, - 'label': field_label - } - fm_weekly_snapshot[node_seg_key]['fields'][historical_field]['weeks_period_data_val'][mnemonic_period] = value - - -def find_previous_quarters(current_quarter, prev_years): - year = int(current_quarter[:4]) - quarter = current_quarter[4:] - previous_quarters = ["{}{}".format(year - i, quarter) for i in range(1, prev_years + 1)] - return previous_quarters - -def find_previous_week_quarters(current_week_quarter, prev_years): - year = int(current_week_quarter[:4]) - week_quarter = current_week_quarter[4:] - previous_week_quarters = ["{}{}".format(year - i, week_quarter) for i in range(1, prev_years + 1)] - return previous_week_quarters - - -def get_crm_schedule(db=None): - try: - crm_schedule = db[CRM_SCHEDULE] if db else sec_context.tenant_db[CRM_SCHEDULE] - criteria = {} - response = crm_schedule.find(criteria) - if response: - return [res for res in response] - else: - return [] - except Exception as ex: - logger.exception(ex) - return {'success': False, 'Error': str(ex)} - - -def get_regional_admin_details(node_id, db=None): - try: - admin_mapping_coll = db[ADMIN_MAPPING] if db else sec_context.tenant_db[ADMIN_MAPPING] - criteria = {'node_id': node_id} - response = admin_mapping_coll.find(criteria) - return {res['node_id']: res for res in response} - except Exception as ex: - logger.exception(ex) - return {'success': False, 'Error': str(ex)} - -def get_fm_weekly_records(period, descendants, config, eligible_nodes_for_segs, fm_weekly_fields, fm_weekly_segments, cache = {}): - req_periods = find_req_periods(period) - non_DR_fields, cur_DR_fields = [], [] - time_context = get_time_context(period, config=None, quarter_editable=config.quarter_editable) - fields_by_type, _ = get_fields_by_type(fm_weekly_fields, config, time_context) - - for _field in fm_weekly_fields: - if 'DR' in fields_by_type and _field in fields_by_type['DR']: - cur_DR_fields.append(_field) - else: - non_DR_fields.append(_field) - - if cur_DR_fields: - time_context_list = [get_time_context(fm_rec_period, config=None, quarter_editable=config.quarter_editable) for fm_rec_period in req_periods] - fetch_fm_rec_in_period(descendants, cur_DR_fields, fm_weekly_segments, config, eligible_nodes_for_segs, req_periods[0], cache, time_context_list = time_context_list) - - if non_DR_fields: - for fm_rec_period in req_periods: - fetch_fm_rec_in_period(descendants, non_DR_fields, fm_weekly_segments, config, eligible_nodes_for_segs, fm_rec_period, cache) - - return cache - - -def get_segments_in_weekly_forecast_for_user(config, node, period, segment = None): - fm_weekly_forecast_waterfall_config = config.config.get('weekly_forecast_waterfall_config',{}) - as_of = get_period_as_of(period) - is_pivot_special = False - if node and node.split('#')[0] in config.get_special_pivots: - is_pivot_special = True - effective_user = sec_context.get_effective_user() - user_edit_dims = effective_user.edit_dims if effective_user else None - user_segments = config.get_segments(as_of, node, is_pivot_special= is_pivot_special) - weekly_segments_list = list(fm_weekly_forecast_waterfall_config.get('segments', [])) - req_segments = [] - - for _segment in weekly_segments_list: - if _segment in user_segments and (not user_edit_dims or _segment in user_edit_dims): - req_segments.append(_segment) - - if segment: - req_segments = [segment] - if 'segment_tree' in fm_weekly_forecast_waterfall_config: - segment_keys_path = segment.split(".") - req_segment_dict = get_nested(fm_weekly_forecast_waterfall_config["segment_tree"], segment_keys_path) - req_segments = get_all_keys(req_segment_dict) - - return req_segments - - -def get_weekly_forecast_parameters(config, req_segments): - fm_weekly_forecast_waterfall_config = config.config.get('weekly_forecast_waterfall_config',{}) - fm_weekly_fields = list(fm_weekly_forecast_waterfall_config.get('weekly_fields_source', [])) - required_fields = list(fm_weekly_forecast_waterfall_config.get('fields_order', [])) - fields_execution_order = list(fm_weekly_forecast_waterfall_config.get('backend_fields_order', [])) - req_segments_map = {segment: config.segments.get(segment, {}) for segment in (req_segments or [])} - fm_quarter_fields = list(fm_weekly_forecast_waterfall_config.get('quarter_fields_source', [])) - total_weeks_fields = list(fm_weekly_forecast_waterfall_config.get('total_weeks_source', [])) - return fm_weekly_fields, fm_quarter_fields, required_fields, req_segments_map, fields_execution_order, total_weeks_fields - - -def find_recursive_fm_weekly_require_field_val(mnemonic_period, - node_id, - require_field, - weekly_segment, - timestamp, - weekly_fields_source, - quarter_fields_source, - total_weeks_source, - backend_restore_weekly_field_source, - calculation_cache, - weekly_waterfall_config, - sorted_weekly_periods_data, - fm_weekly_recs, - is_leaf): - - - cache_key = (mnemonic_period, node_id, require_field, weekly_segment, timestamp) - if cache_key in calculation_cache: - return calculation_cache[cache_key] - # Fetch field configuration - field_config = weekly_waterfall_config.get('fields', {}).get(require_field, {}) - require_field_function = field_config.get('function', '') - - #this sources are used to calculate the require field value - - if 'schema_dependent_function' in field_config: - target_schema = "rep" if is_leaf else "grid" - require_field_function = field_config['schema_dependent_function'][target_schema]['function'] - is_default_function = 'default' in field_config['schema_dependent_function'][target_schema] - default_function = field_config['schema_dependent_function'][target_schema].get('default', '') - is_relative_period = 'relative_period' in field_config['schema_dependent_function'][target_schema] - relative_periods = field_config['schema_dependent_function'][target_schema].get('relative_period', []) - else: - require_field_function = field_config.get('function', '') - is_default_function = 'default' in field_config - default_function = field_config.get('default', '') - is_relative_period = 'relative_period' in field_config - relative_periods = field_config.get('relative_period', []) - - derived_grid_source = [] - - # Handle derived grid source - if 'derived_grid_source' in field_config: - for derived_grid_field in field_config['derived_grid_source']: - derived_grid_field_val = find_recursive_fm_weekly_require_field_val( - mnemonic_period, node_id, derived_grid_field, weekly_segment, timestamp, - weekly_fields_source, quarter_fields_source, total_weeks_source, backend_restore_weekly_field_source, - calculation_cache, weekly_waterfall_config, sorted_weekly_periods_data, fm_weekly_recs, is_leaf - ) - derived_grid_source.append(derived_grid_field_val) - - # Evaluate the require_field function - try: - require_field_val = eval(require_field_function) - except ZeroDivisionError: - require_field_val = 0 - - # Determine relative period - cur_relative_period = sorted_weekly_periods_data.get(mnemonic_period, {}).get('relative_period', None) - is_editable = field_config.get('editable', False) - is_target_field_found = True if require_field_val != 0 else False - if is_editable: - target_field = field_config.get('target_field', require_field) - is_target_field_found = fm_weekly_recs.get((mnemonic_period, node_id, target_field, weekly_segment, timestamp), {}).get('found', False) - - - if is_default_function and ( - (not is_target_field_found and not is_relative_period) or - (is_relative_period and cur_relative_period in relative_periods) - ): - - try: - require_field_val = eval(default_function) - except ZeroDivisionError: - require_field_val = 0 - - - # Cache and return the result - calculation_cache[cache_key] = require_field_val - return require_field_val - - -def get_weekly_periods_in_months(qtr_period): - all_monthly_periods = find_monthly_periods(qtr_period) - all_weekly_periods = find_req_periods(qtr_period) - month_week_map = {} - req_data = {} - for week_period in all_weekly_periods: - month_no = int(week_period[4:6]) - month_week_map.setdefault(month_no, []).append(week_period) - - for month_period in all_monthly_periods: - month_no = int(month_period[4:6]) - req_data[month_period] = month_week_map[month_no] - - return req_data - - -def get_source_list_for_weekly_forecast(req_period, node_id, weekly_segment, timestamp, fm_recs, fm_fields, all_fields_source): - fields_source = [0] * len(all_fields_source) - - for fm_field in fm_fields: - fm_weekly_field_val = fm_recs.get((req_period, node_id, fm_field, weekly_segment, timestamp), {}).get('val', 0) - field_index = all_fields_source.index(fm_field) - fields_source[field_index] = fm_weekly_field_val - - return fields_source - - -def get_source_list_for_aggregate_fields_in_periods(req_periods, node_id, weekly_segment, timestamp, fm_recs, fm_fields, all_fields_source): - fields_source = [0] * len(all_fields_source) - - for fm_field in fm_fields: - - period_sum = 0 - for _period in req_periods: - cache_key = (_period, node_id, fm_field, weekly_segment, timestamp) - period_sum += fm_recs.get(cache_key, 0) - - field_index = all_fields_source.index(fm_field) - fields_source[field_index] = period_sum - - return fields_source - -def get_children_of_nodes(nodes, as_of_timestamp, levels = 2): - parent_child_map = {} - descendants = fetch_descendants(as_of_timestamp, nodes , levels) - descendants = [(rec['node'], rec['descendants'][0], rec['descendants'][1]) for rec in descendants] - for key, dict1, _ in descendants: - childs = list(dict1.keys()) - parent_child_map[key] = childs - - return parent_child_map - - -def is_weekly_forecast_field_editable(weekly_segment, field, weekly_waterfall_config, config): - is_editable = weekly_waterfall_config.get('fields', {}).get(field, {}).get('editable', False) - segment_editable = 'segment_func' not in config.segments.get(weekly_segment, {}) - segment_dependent_editability = weekly_waterfall_config.get('fields', {}).get(field, {}).get('segment_dependent_editability', False) - - if not is_editable: - return False - - # If the field is editable and its editability is segment-dependent, check for disabled edit fields or segment_func - if segment_dependent_editability: - if not segment_editable: - return False - - target_field = weekly_waterfall_config.get('fields', {}).get(field, {}).get('target_field', field) - disable_edit_fields = config.segments.get(weekly_segment, {}).get('disable_edit_fields', []) - if target_field in disable_edit_fields: - return False - - return True - - -def process_weekly_snapshot_field(node_id, weekly_segment, mnemonic_period, cur_field, cur_field_val, fm_weekly_snapshot, - all_quarter_fields_source, quarter_fields_source, weekly_waterfall_config, config, sorted_weekly_periods_data): - - segment_label = config.segments.get(weekly_segment, {}).get('label', weekly_segment) - node_segment_key = node_id + "_" + weekly_segment - require_field_label = weekly_waterfall_config.get('fields', {}).get(cur_field, {}).get('label','').format(segment= segment_label) - require_field_format = weekly_waterfall_config.get('fields', {}).get(cur_field, {}).get('format', 'amount') - require_field_editable = is_weekly_forecast_field_editable(weekly_segment, cur_field, weekly_waterfall_config, config) - - if cur_field not in fm_weekly_snapshot[node_segment_key]['fields']: - fm_weekly_snapshot[node_segment_key]['fields'][cur_field] = { - 'weeks_period_data_val': {}, - 'editable': require_field_editable, - 'format': require_field_format, - 'label': require_field_label - } - - if 'monthly_forecast' in weekly_waterfall_config.get('fields', {}).get(cur_field, {}): - monthly_forecast_field = weekly_waterfall_config.get('fields', {}).get(cur_field, {}).get('monthly_forecast') - field_index = all_quarter_fields_source.index(monthly_forecast_field) - monthly_forecast_val = quarter_fields_source[field_index] - fm_weekly_snapshot[node_segment_key]['fields'][cur_field]['monthly_forecast'] = monthly_forecast_val - - if 'clickable_field' in weekly_waterfall_config.get('fields', {}).get(cur_field, {}): - clickable_field = weekly_waterfall_config.get('fields', {}).get(cur_field, {}).get('clickable_field') - fm_weekly_snapshot[node_segment_key]['fields'][cur_field]['clickable_field'] = clickable_field - - - relative_period_val = sorted_weekly_periods_data.get(mnemonic_period, {}).get('relative_period', None) - render_periods = weekly_waterfall_config.get('fields', {}).get(cur_field, {}).get('render_period', ['h', 'f', 'c']) - if relative_period_val in render_periods: - fm_weekly_snapshot[node_segment_key]['fields'][cur_field]['weeks_period_data_val'][mnemonic_period] = cur_field_val - - -def is_leaf_node(nodes, period): - is_leaf_map = {} - for _node_id in nodes: - as_of = get_period_as_of(period) - children = [x['node'] for x in fetch_children(as_of, [_node_id])] - is_leaf = not children - is_leaf_map[_node_id] = is_leaf - return is_leaf_map - -def process_weekly_forecast_cumulative_field(node, segment, restore_field, weekly_periods_val, fm_weekly_recs, sorted_weekly_periods_data): - - weekly_periods = list(sorted_weekly_periods_data.keys()) - running_sum = 0 - for week_period in weekly_periods: - running_sum += weekly_periods_val[week_period] - - fm_weekly_recs[(week_period, node, restore_field, segment, None)] = {'period': week_period, - 'segment': segment, - 'val': running_sum, - 'by': 'system', - 'how': 'backend process', - 'found': True, - 'timestamp': None, - 'node': node, - 'field': restore_field} - - - -def get_weekly_segment_snapshot_values_cache(all_nodes, fields_execution_order, req_periods, req_segments, period, - fm_weekly_recs, fm_weekly_fields, total_weeks_fields, fm_quarterly_recs, - fm_quarter_fields, config, calculation_cache, weekly_waterfall_config): - - all_weekly_fields_source = list(weekly_waterfall_config.get('weekly_fields_source', [])) - all_quarter_fields_source = list(weekly_waterfall_config.get('quarter_fields_source', [])) - all_total_weeks_source = list(weekly_waterfall_config.get('total_weeks_source', [])) - all_backend_restore_weekly_field_source = list(weekly_waterfall_config.get('backend_restore_weekly_field_source', [])) - backend_restore_fields = list(weekly_waterfall_config.get('backend_restore_weekly_field_source', [])) - sorted_weekly_periods_data = fetch_weekly_periods(period) - is_leaf_map = is_leaf_node(all_nodes, period) - - # Process nodes and segments to calculate required field values - for node_id, weekly_segment, timestamp in product(all_nodes, req_segments, [None]): - is_leaf = is_leaf_map[node_id] - total_weeks_source = [0] * len(all_total_weeks_source) - # Process each field in execution order - for cur_field in fields_execution_order: - total_quarter_val = 0 - weekly_periods_val = {} - field_config = weekly_waterfall_config.get('fields', {}).get(cur_field, {}) - is_cumulative_field = 'cumulative_process' in field_config and 'backend_restore_field' in field_config.get('cumulative_process', {}) - # Iterate over required periods to calculate field values - for mnemonic_period in req_periods: - # Fetch data sources for weekly and quarterly fields - - weekly_fields_source = get_source_list_for_weekly_forecast( - mnemonic_period, node_id, weekly_segment, timestamp, fm_weekly_recs, fm_weekly_fields, all_weekly_fields_source) - - quarter_fields_source = get_source_list_for_weekly_forecast( - period, node_id, weekly_segment, timestamp, fm_quarterly_recs, fm_quarter_fields, all_quarter_fields_source) - - backend_restore_weekly_field_source = get_source_list_for_weekly_forecast( - mnemonic_period, node_id, weekly_segment, timestamp, fm_weekly_recs, backend_restore_fields, all_backend_restore_weekly_field_source - ) - - cur_field_val = find_recursive_fm_weekly_require_field_val( - mnemonic_period, node_id, cur_field, weekly_segment, timestamp, - weekly_fields_source, quarter_fields_source, total_weeks_source, backend_restore_weekly_field_source, calculation_cache, - weekly_waterfall_config, sorted_weekly_periods_data, fm_weekly_recs, is_leaf) - - total_quarter_val += cur_field_val - weekly_periods_val[mnemonic_period] = cur_field_val - - if cur_field in total_weeks_fields: - field_index = all_total_weeks_source.index(cur_field) - total_weeks_source[field_index] = total_quarter_val - - if is_cumulative_field: - restore_field = field_config.get('cumulative_process').get('backend_restore_field') - process_weekly_forecast_cumulative_field(node_id, weekly_segment, restore_field, weekly_periods_val, fm_weekly_recs, sorted_weekly_periods_data) - - -def create_fm_weekly_snapshot(nodes, req_segments, req_periods, required_fields, period, fm_quarterly_recs, fm_quarter_fields , - include_children_data, parent_child_map, nodes_labels, calculation_cache, - fm_weekly_snapshot, config): - - weekly_waterfall_config = config.config.get('weekly_forecast_waterfall_config',{}) - all_quarter_fields_source = list(weekly_waterfall_config.get('quarter_fields_source', [])) - sorted_weekly_periods_data = fetch_weekly_periods(period) - for node_id, weekly_segment, timestamp in product(nodes, req_segments, [None]): - segment_label = config.segments.get(weekly_segment, {}).get('label', weekly_segment) - node_segment_key = node_id + "_" + weekly_segment - - if node_segment_key not in fm_weekly_snapshot: - fm_weekly_snapshot[node_segment_key] = {'fields': {}, 'segment': weekly_segment, 'label': segment_label} - - for cur_field in required_fields: - childs_rollup_weekly = {} - is_rollup_field = 'child_rollup' in weekly_waterfall_config.get('fields', {}).get(cur_field, {}) - - for mnemonic_period in req_periods: - - cur_field_val = calculation_cache.get((mnemonic_period, node_id, cur_field, weekly_segment, timestamp), 0) - - quarter_fields_source = get_source_list_for_weekly_forecast( - period, node_id, weekly_segment, timestamp, fm_quarterly_recs, fm_quarter_fields, all_quarter_fields_source) - - require_field_format = weekly_waterfall_config.get('fields', {}).get(cur_field, {}).get('format', 'amount') - process_weekly_snapshot_field( - node_id, weekly_segment, mnemonic_period, cur_field, cur_field_val, fm_weekly_snapshot, - all_quarter_fields_source, quarter_fields_source, weekly_waterfall_config, config, sorted_weekly_periods_data) - - - if is_rollup_field and include_children_data: - cur_child_field = weekly_waterfall_config.get('fields', {}).get(cur_field, {}).get('child_rollup', {}).get('field', cur_field) - if 'child_data' not in fm_weekly_snapshot[node_segment_key]['fields'][cur_field]: - fm_weekly_snapshot[node_segment_key]['fields'][cur_field]['child_data'] = {'editable': False, 'format': require_field_format} - - - for cur_child in parent_child_map[node_id]: - child_segment_key = cur_child + "_" + weekly_segment - if child_segment_key not in fm_weekly_snapshot[node_segment_key]['fields'][cur_field]['child_data']: - fm_weekly_snapshot[node_segment_key]['fields'][cur_field]['child_data'][child_segment_key] = { - 'label': nodes_labels[cur_child], 'weeks_period_data_val': {}} - - cur_child_val = calculation_cache.get((mnemonic_period, cur_child, cur_child_field, weekly_segment, timestamp), 0) - - if mnemonic_period not in childs_rollup_weekly: - childs_rollup_weekly[mnemonic_period] = 0 - childs_rollup_weekly[mnemonic_period] += cur_child_val - - fm_weekly_snapshot[node_segment_key]['fields'][cur_field]['child_data'][child_segment_key]['weeks_period_data_val'][mnemonic_period] = cur_child_val - - if is_rollup_field and include_children_data: - fm_weekly_snapshot[node_segment_key]['fields'][cur_field]['child_rollup_data'] = childs_rollup_weekly - - -def get_parent_child_metadata(nodes, as_of_timestamp, include_children_data = False): - parent_child_map = {} - if include_children_data: - parent_child_map = get_children_of_nodes(nodes, as_of_timestamp) - - all_nodes = list(nodes) - for key, value in parent_child_map.items(): - all_nodes.extend(value) - - return parent_child_map, all_nodes - - -def get_weekly_forecast_cache(period, - nodes , - fm_weekly_fields, - fm_quarter_fields, - fields_execution_order, - total_weeks_fields, - req_segments, - req_segments_map, - req_periods, - as_of_timestamp, - config, - eligible_nodes_for_segs, - weekly_waterfall_config, - week_period = None, - include_children_data = False): - - # Initialize required data structures for storing records and cache - fm_quarterly_recs, fm_weekly_recs, calculation_cache = {}, {}, {} - parent_child_map, all_nodes = get_parent_child_metadata(nodes, as_of_timestamp, include_children_data) - descendants = [(node_id, {}, {}) for node_id in all_nodes] - - - get_fm_weekly_records(period, descendants, config, eligible_nodes_for_segs, - fm_weekly_fields, req_segments_map, fm_weekly_recs) - - fetch_fm_rec_in_period(descendants, fm_quarter_fields, req_segments_map, config, - eligible_nodes_for_segs, period, fm_quarterly_recs) - - get_weekly_segment_snapshot_values_cache(all_nodes, fields_execution_order, req_periods, req_segments, period, - fm_weekly_recs, fm_weekly_fields, total_weeks_fields, fm_quarterly_recs, - fm_quarter_fields, config, calculation_cache, weekly_waterfall_config) - - return fm_quarterly_recs, fm_weekly_recs, calculation_cache - - -def fetch_weekly_segment_snapshot(period, - nodes , - fm_quarter_fields, - required_fields, - req_segments, - fm_quarterly_recs, - fm_weekly_recs, - calculation_cache, - req_periods, - as_of_timestamp, - config, - week_period = None, - include_children_data = False): - - # Initialize required data structures for storing records and cache - fm_weekly_snapshot = {} - - is_historical_avg = False - if 'historical_avg' in required_fields: - is_historical_avg = True - required_fields.remove('historical_avg') - - parent_child_map, all_nodes = get_parent_child_metadata(nodes, as_of_timestamp, include_children_data) - nodes_labels = fetch_labels(as_of_timestamp, all_nodes) - - create_fm_weekly_snapshot(nodes, req_segments, req_periods, required_fields, period, fm_quarterly_recs, fm_quarter_fields , - include_children_data, parent_child_map, nodes_labels, calculation_cache, - fm_weekly_snapshot, config) - - if is_historical_avg and 'historical_avg' in config.config.get('weekly_forecast_waterfall_config', {}): - update_fm_weekly_snapshot_for_historical_avg(nodes, period, week_period, config, req_segments, fm_weekly_snapshot) - - return {'data': fm_weekly_snapshot} - - -def update_weekly_segment_snapshot_metadata(node, req_segments, segment, data, as_of_timestamp, include_children_data, config): - config_data = config.config - fm_weekly_config = config_data.get('weekly_forecast_waterfall_config', {}) - - # Generate segment order based on config and available data - display_segments = config_data.get('segments', {}).get('display', []) - segment_order = ["{}_{}".format(node, seg) for seg in display_segments if "{}_{}".format(node, seg) in data.get('data', {})] - - # Extract ordered fields present in snapshot data - node_segment_key = "{}_{}".format(node, req_segments[0]) - config_fields_order = fm_weekly_config.get('fields_order', []) - fields_in_snapshot = data.get('data', {}).get(node_segment_key, {}).get('fields', {}).keys() - fields_order = [field for field in config_fields_order if field in fields_in_snapshot] - - weekly_snapshot_metadata = {'column_order': fields_order, 'segment_order': segment_order} - - if include_children_data: - parent_child_map = get_children_of_nodes([node], as_of_timestamp) - weekly_snapshot_metadata['child_key'] = parent_child_map.get(node, []) - - if 'segment_tree' in fm_weekly_config: - segment_keys_path = (segment or 'segment_tree').split(".") - req_segment_dict = get_nested(fm_weekly_config['segment_tree'], segment_keys_path, fm_weekly_config['segment_tree']) - weekly_snapshot_metadata['segment_tree'] = modify_dict_keys(req_segment_dict, node) - - data['metadata'] = weekly_snapshot_metadata - - -def find_months_from_weekly_forecast_periods(req_periods): - """Extract unique months from the data.""" - months = set() - for key in req_periods: - if len(key) == 9 and 'W' in key: - month = int(key[4:6]) - months.add(month) - - months = sorted(months) - start_index = 0 - for index in range(len(months)-1): - if months[index+1]-months[index]>1: - start_index = index+1 - break - sorted_month_order = months[start_index:] + months[:start_index] - return sorted_month_order - - -def create_key_from_weekly_mnemonic(period, mnemonic_key, custom_month_order): - - if len(mnemonic_key) == 9 and 'W' in mnemonic_key: - year = int(period[0:4]) - month_no = int(mnemonic_key[4:6]) - month = custom_month_order[month_no-1].capitalize() - month_key = month[0:3] - week_no = mnemonic_key[7:] - return "FY{}-{}-WK{}".format(year, month_key, week_no) - - elif mnemonic_key in custom_month_order: - return mnemonic_key.capitalize() - - return mnemonic_key - - - -def find_field_names_from_weekly_forecast_export_coll(period, cursor, current_months, custom_month_order): - """Generate field names based on available months and weeks.""" - fieldnames = ['Name', 'label'] - weeks_in_months = {} - - # Process the first record to identify weeks - for record in cursor: - for key in record: - if len(key) == 9 and 'W' in key: - month_no = int(key[4:6]) - new_key = create_key_from_weekly_mnemonic(period, key, custom_month_order) - if month_no not in weeks_in_months: - weeks_in_months[month_no] = [] - weeks_in_months[month_no].append(new_key) - break - - # Add week and month field names - for month_no in current_months: - weeks_in_months[month_no] = sorted(weeks_in_months[month_no]) - month_key = custom_month_order[month_no-1].capitalize() - fieldnames.extend(weeks_in_months[month_no]) - fieldnames.append(month_key) - - fieldnames.extend([ - 'quarter-total', - 'Monthly Forecast', - 'Difference (Monthly-Weekly)' - ]) - - return fieldnames - - -def reorder_weekly_forecast_segment_rows(node, period, segment_rows, ordered_segment_rows, config): - segment_order = config.config.get('segments', {}).get('display', {}) - lookup_dict = {} - for key in segment_rows.keys(): - segment = key.split('_')[-1] - lookup_dict[segment] = key - - for segment in segment_order: - if segment in lookup_dict: - key = lookup_dict[segment] - ordered_segment_rows.append((key, segment_rows[key])) - - waterfall_config = config.config.get('weekly_forecast_waterfall_config', {}) - as_of_timestamp = get_period_as_of(period) - all_fields = list(waterfall_config.get('fields_order', [])) - req_segments = get_segments_in_weekly_forecast_for_user(config, node, period) - parent_child_map = get_children_of_nodes([node], as_of_timestamp) - child_labels = fetch_labels(as_of_timestamp, parent_child_map.get(node, [])) - - fields_label_order = {} - for seg in req_segments: - seg_key = "{}_{}".format(node, seg) - segment_label = config.segments.get(seg, {}).get('label', seg) - for field in all_fields: - if field in waterfall_config.get('fields', {}): - field_label = waterfall_config['fields'].get(field, {}).get('label', field).format(segment=segment_label) - fields_label_order.setdefault(seg_key, []).append(field_label) - if 'child_rollup' in waterfall_config['fields'][field]: - fields_label_order[seg_key].append("{}_Team Rollup".format(field_label)) - for _child in parent_child_map.get(node, []): - fields_label_order[seg_key].append(child_labels.get(_child, _child)) - elif field in waterfall_config: - field_label = waterfall_config.get(field, {}).get('label', field) - fields_label_order.setdefault(seg_key, []).append(field_label) - - for key, seg_rows in ordered_segment_rows: - ordered_labels = fields_label_order.get(key, []) - label_index_map = {label: i for i, label in enumerate(ordered_labels)} - seg_rows[:] = sorted( - seg_rows, - key=lambda x: label_index_map.get(x["label"], float("inf")), - ) - - - -def reformat_weekly_forecast_export_data(node, period, cursor, custom_month_order, segment_header, ordered_segment_rows , other_rows, as_of, config): - """Reformat forecast data to include appropriate headers and formatting.""" - segment_rows = {} - for record in cursor: - rec_copy = record.copy() - rec_format = rec_copy.get('quarter-total-format', 'amount') - node_key = rec_copy.get('node') - labels = fetch_labels(as_of, [node_key]) - record['Name'] = labels[node_key] - for key, value in rec_copy.items(): - # Process weekly data - if len(key) == 9 and 'W' in key: - value = value * 100 if rec_format == 'percentage' else value - val = round(float(value), 2) - new_key = create_key_from_weekly_mnemonic(period, key, custom_month_order) - del record[key] - record[new_key] = val - - # Process monthly data - elif key in custom_month_order: - value = value * 100 if rec_format == 'percentage' else value - val = round(float(value), 2) - new_key = key.capitalize() - del record[key] - record[new_key] = val - - # Process quarter total - elif key == 'quarter-total': - value = value * 100 if rec_format == 'percentage' else value - val = round(float(value), 2) - record[key] = val - - elif key == 'monthly_forecast': - new_key = 'Monthly Forecast' - val = round(float(value), 2) - del record[key] - record[new_key] = val - - elif key == 'monthly_difference_in_amount': - new_key = 'Difference (Monthly-Weekly)' - val = round(float(value), 2) - del record[key] - record[new_key] = val - - # Categorize the processed record - if 'segment' in rec_copy: - if rec_copy['type'] == 'segment-header': - segment_header[rec_copy['segment']] = record - else: - segment_rows.setdefault(rec_copy['segment'], []).append(record) - else: - other_rows.append(record) - - reorder_weekly_forecast_segment_rows(node, period, segment_rows, ordered_segment_rows, config) - -def fetch_weekly_forecast_export_coll(node, period, req_segments, db=None): - node_segments_key = [] - for segment in req_segments: - node_segments_key.append(node+ "_" +segment) - weekly_forecast_export_coll = db[WEEKLY_FORECAST_EXPORT_COLL] if db else sec_context.tenant_db[WEEKLY_FORECAST_EXPORT_COLL] - criteria = { - 'period': period, - 'node': node, - '$or': [ - {'segment': {'$in': node_segments_key}}, - {'segment': {'$exists': False}} - ] - } - cursor = list(weekly_forecast_export_coll.find(criteria)) - return cursor - - -def fetch_cached_export(period, timestamp=None, config=None, db=None): - as_of = timestamp if timestamp is not None else get_period_as_of(period) - dt = epoch(as_of).as_datetime() - as_of_date = dt.strftime("%Y-%m-%d") - coll = EXPORT_ALL + "_" + str(period) - export_all_coll = sec_context.tenant_db[coll] - criteria = {'export_date': {'$lte': as_of_date}} - file_name = list(export_all_coll.find(criteria).sort([('export_date', -1)]).limit(1))[0]['file_name'] - return file_name - - -def create_csv_and_save_in_bucket(period, bucket_name, fieldnames, data_rows, file_name): - - current_timestamp = epoch().as_epoch() - csv_buffer = BytesIO() - writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames) - writer.writeheader() - - for row in data_rows: - writer.writerow({key: row.get(key, None) for key in fieldnames}) - - s3.put_object( - Bucket = bucket_name, - Key = file_name, - Body = csv_buffer.getvalue(), # Get the content from the buffer - ContentType ='text/csv' - ) - logger.info("File '{}' with {} total rows created successfully and saved to S3 bucket '{}' for period {}.".format(file_name, len(data_rows), bucket_name, period)) - - file_path = "https://{}.s3.us-east-1.amazonaws.com/{}".format(bucket_name, file_name) - return {"success": True, "data": {"updated_at": current_timestamp, "file_path": file_path}} - - -def get_weekly_edw_data(period = None): - try: - return sec_context.details.get_flag('fm_svc', WEEKLY_EDW_DATA, {}) - except : - return None - - -def get_weekly_edw_process_status(period = None): - try: - return sec_context.details.get_flag('fm_svc', WEEKLY_EDW_PROCESS_STATUS, "Not Started") - except: - return None - - -def get_edw_data(): - try: - return sec_context.details.get_flag('fm_svc',EDW_DATA, []) - except: - return None - - -def set_edw_data(edw_data): - try: - logger.info("EDW flag has been set",edw_data) - return sec_context.details.set_flag('fm_svc',EDW_DATA,edw_data) - except: - logger.error("An error occurred while setting the edw_data value.") - return None - - -def get_edw_process_update(): - try: - return sec_context.details.get_flag('fm_svc',EDW_PROCESS_UPDATE, 'Started') - except: - return None - - -def set_edw_process_update(edw_update): - try: - logger.info("EDW flag has been set",edw_update) - return sec_context.details.set_flag('fm_svc',EDW_PROCESS_UPDATE,edw_update) - except: - logger.error("An error occurred while setting the edw_data value.") - return None - -def get_weekly_edw_process_start_time(): - try: - timestamp = epoch().as_epoch() - return sec_context.details.get_flag('fm_svc', WEEKLY_EDW_PROCESS_START_TIME, timestamp) - except: - return None - -def get_weekly_forecast_export_all(period): - try: - return sec_context.details.get_flag('fm_svc', WEEKLY_FORECAST_EXPORT_ALL.format(period), {}) - except: - return None - -def get_all_keys(data): - """Recursively collects all keys in the dictionary.""" - keys = [] - for key, value in data.items(): - keys.append(key) - if isinstance(value, dict): - keys.extend(get_all_keys(value)) - return keys - -def modify_dict_keys(data, prefix_key): - if isinstance(data, dict): - return { - "{}_{}".format(prefix_key, key): modify_dict_keys(value, prefix_key) - for key, value in data.items() - } - return data - -def fetch_weekly_periods(period): - periodconfig = PeriodsConfig() - quarter_week_range = weekly_periods(period) - now = get_now(periodconfig) - now_dt = now.as_datetime() - weekly_periods_data = {} - for week_range in quarter_week_range: - week_info = render_period(week_range, now_dt, periodconfig) - weekly_periods_data[week_range.mnemonic] = week_info - - sorted_weekly_periods_data = OrderedDict(sorted(weekly_periods_data.items(), key=lambda item: item[1]['end'])) - return sorted_weekly_periods_data - - -def fetch_expired_nodes(node, as_of, period): - exp_from_date , exp_to_date = get_period_begin_end(period) - exp_boundary_dates = (exp_from_date, exp_to_date) - exp_child_order = [x['node'] for x in fetch_children(as_of, [node], period = period, boundary_dates = exp_boundary_dates)] - latest_child_nodes = [x['node'] for x in fetch_children(as_of, [node], period = period)] - latest_child_nodes = list(set(latest_child_nodes)) - expired_nodes = set(exp_child_order) - set(latest_child_nodes) - expired_nodes = list(expired_nodes) - return expired_nodes - - -def get_children_periods_mapping(node, period): - children_periods_map = {} - # Get all the periods for the quarter - qtr_periods = get_quarter_period(period) - - # Get the boundary dates for the quarter for fetching the children - for qtr in qtr_periods: - qtr_as_of = get_period_as_of(qtr) - qtr_from_date, _ = fetch_boundry(qtr_as_of, drilldown=True) - _, qtr_to_date = get_period_begin_end(qtr) - qtr_boundary_dates = (qtr_from_date, qtr_to_date) - children = [x['node'] for x in fetch_children(qtr_as_of, [node], period = qtr, boundary_dates=qtr_boundary_dates)] - children = list(set(children)) - children_periods_map[qtr] = children - - return children_periods_map - - -def fetch_cached_snapshot_in_period(period, node, config=None, db=None, segment=None): - if not db: - db = sec_context.tenant_db - - config = config if config else FMConfig() - - coll = SNAPSHOT_ROLE_COLL if config.has_roles else SNAPSHOT_COLL - snapshot_coll = db[coll] - if segment: - criteria = {"node": node, "period": period, "segment": segment} - else: - criteria = {"node": node, "period": period, "segment": 'all_deals'} - - projections = {"snapshot_data": 1, "last_updated_time": 1, "snapshot_stale": 1, "how": 1} - - if config.has_roles: - user_role = sec_context.get_current_user_role() - if user_role not in config.get_roles: - user_role = DEFAULT_ROLE - criteria.update({'role': user_role}) - - response = snapshot_coll.find_one(criteria, projections) - result = {} - if response: - if response.get('snapshot_data'): - result = response['snapshot_data'] - result['updated_at'] = response['last_updated_time'] - return result - - -def find_field_format(field, segment, config): - field_format = config.config.get('fields', {}).get(field, {}).get('format', None) - if field_format: - return field_format - percentage_override_segments = config.percentage_override_segments - count_override_segments = config.count_override_segments - if segment in count_override_segments: - field_format = 'count' - elif segment in percentage_override_segments: - field_format = 'percentage' - else: - field_format = 'amount' - - return field_format - - -def fetch_active_nodes(waterfall_config, waterfall_pivot, node, period): - waterfall_pivot_max_depth = waterfall_config.get('max_depth', {}) - leaf_level = 12 - if waterfall_pivot in waterfall_pivot_max_depth: - leaf_level = waterfall_pivot_max_depth[waterfall_pivot]['leaf_depth'] - root_node = node.rsplit('#', 1)[0] + '#!' - as_of = get_period_as_of(period) - subtree_height = find_all_subtree_height(as_of, root_node, leaf_level) - - descendants = set() - excluded_node_set = set() - if waterfall_pivot in waterfall_pivot_max_depth: - if 'exclude_node_and_descendants' in waterfall_pivot_max_depth[waterfall_pivot]: - exclude_node_and_descendants = waterfall_pivot_max_depth[waterfall_pivot]['exclude_node_and_descendants'] - excluded_node_set.update(exclude_node_and_descendants) - excluded_descendants = fetch_descendants(as_of, - nodes=exclude_node_and_descendants, - levels=12, - include_hidden=False, - include_children=False, - drilldown=True) - for rec in excluded_descendants: - if rec.get("descendants"): - for desc in rec['descendants']: - if desc: - for cnode, _ in desc.items(): - excluded_node_set.add(cnode) - - for node, height in subtree_height.items(): - if height >= 1 and node not in excluded_node_set: - descendants.add(node) - - return list(descendants) - - -def fetch_qtd_performance_gard_parameters(config, req_segments, period): - qtd_gard_config = config.config.get('qtd_performance_config', {}) - weekly_forecast_config = qtd_gard_config.get('weekly_forecast_config', {}) - qtd_weekly_fields = list(weekly_forecast_config.get('weekly_fields_source', [])) - fields_execution_order = list(weekly_forecast_config.get('backend_fields_order', [])) - req_segments_map = {segment: config.segments.get(segment, {}) for segment in req_segments} - qtd_quarter_fields = list(weekly_forecast_config.get('quarter_fields_source', [])) - total_weeks_fields = list(weekly_forecast_config.get('total_weeks_source', [])) - as_of_timestamp = get_period_as_of(period) - weekly_periods = find_req_periods(period) - req_fields = list(qtd_gard_config.get('field_schema', {}).keys()) - return qtd_weekly_fields, qtd_quarter_fields, req_segments_map, fields_execution_order, \ - total_weeks_fields, as_of_timestamp, weekly_periods, req_fields - - -def fetch_nodes_for_gard_dashboard(node, node_level, subnode_level, as_of): - depth = int(subnode_level) - int(node_level) - descendants = list(fetch_descendants(as_of, [node], depth)) - - - children = {rec['node']: list(rec['descendants'][0].keys()) for rec in descendants}.get(node, []) - lth_grand_children = {rec['node']: list(rec['descendants'][depth - 1].keys()) for rec in descendants}.get(node, []) - - return lth_grand_children + (children if depth > 1 else []) + [node] - - -def fetch_hierarchy_order_for_gard_dashboard(node, node_level, subnode_level, as_of, period, boundary_dates=None): - hierarchy_order = [{node: []}] - depth = int(subnode_level) - int(node_level) - children = [x['node'] for x in - fetch_children(as_of, [node], period, boundary_dates=boundary_dates)] - - if depth == 1: - return hierarchy_order + [{child: []} for child in children] - descendants = list(fetch_descendants(as_of, [node], depth)) - - root_node = node.rsplit('#', 1)[0] + '#!' - lth_grand_children = {rec['node']: list(rec['descendants'][depth - 1].keys()) for rec in descendants}.get(node, []) - map_data = find_map_in_nodes_and_lth_grand_children(as_of, root_node, children, lth_grand_children) - - return hierarchy_order + [{child: map_data.get(child, [])} for child in children] - - -def get_monthly_sum_from_weekly(node_id, weekly_periods, weekly_field, segment, timestamp, - month_rank, qtd_gard_weekly_recs, qtd_gard_calculation_cache): - - sorted_months = find_months_from_weekly_forecast_periods(weekly_periods) - monthly_sum = {} - - for week_period in weekly_periods: - cache_key = (week_period, node_id, weekly_field, segment, timestamp) - week_val = qtd_gard_calculation_cache.get(cache_key, 0) - month = int(week_period[4:6]) - monthly_sum[month] = monthly_sum.get(month, 0) + week_val - - #rank is 1 based - if len(sorted_months) >= month_rank: - return monthly_sum[sorted_months[month_rank - 1]] - else: - return 0 - -def fetch_qtd_performance_gard_snapshot(req_nodes, - req_fields, - period, - segment, - weekly_periods, - qtd_gard_weekly_recs, - qtd_gard_quarterly_recs, - qtd_gard_calculation_cache, - config, - labels): - - qtd_gard_snapshot = {} - field_schema = config.config.get('qtd_performance_config', {}).get('field_schema', {}) - - for node_id , field, timestamp in product(req_nodes, req_fields, [None]): - field_config = field_schema.get(field, {}) - val = 0 - if node_id not in qtd_gard_snapshot: - qtd_gard_snapshot[node_id] = {'fields': {}} - - if 'value_type' in field_config and field_config['value_type'] == 'quarterly': - val = qtd_gard_quarterly_recs.get((period, node_id, field, segment, timestamp), {}).get('val', 0) - - elif 'value_type' in field_config and field_config['value_type'] == 'monthly_sum' and 'weekly_field' in field_config and 'month_rank' in field_config: - weekly_field = field_config['weekly_field'] - month_rank = field_config['month_rank'] - val = get_monthly_sum_from_weekly(node_id, weekly_periods, weekly_field, segment, timestamp, - month_rank, qtd_gard_weekly_recs, qtd_gard_calculation_cache) - - qtd_gard_snapshot[node_id]['fields'][field] = {'value': val} - if 'label' not in qtd_gard_snapshot[node_id]: - qtd_gard_snapshot[node_id]['label'] = labels.get(node_id) - - return {'data':qtd_gard_snapshot} - - - -def fetch_qtd_performance_product_snapshot(node_id, - req_fields, - period, - segments, - weekly_periods, - qtd_gard_weekly_recs, - qtd_gard_quarterly_recs, - qtd_gard_calculation_cache, - config): - - qtd_product_snapshot = {} - field_schema = config.config.get('qtd_performance_config', {}).get('field_schema', {}) - - for segment, field, timestamp in product(segments, req_fields, [None]): - field_config = field_schema.get(field, {}) - val = 0 - if segment not in qtd_product_snapshot: - qtd_product_snapshot[segment] = {'fields': {}} - - if 'value_type' in field_config and field_config['value_type'] == 'quarterly': - val = qtd_gard_quarterly_recs.get((period, node_id, field, segment, timestamp), {}).get('val', 0) - - elif 'value_type' in field_config and field_config['value_type'] == 'monthly_sum' and 'weekly_field' in field_config and 'month_rank' in field_config: - weekly_field = field_config['weekly_field'] - month_rank = field_config['month_rank'] - val = get_monthly_sum_from_weekly(node_id, weekly_periods, weekly_field, segment, timestamp, - month_rank, qtd_gard_weekly_recs, qtd_gard_calculation_cache) - - qtd_product_snapshot[segment]['fields'][field] = {'value': val} - if 'label' not in qtd_product_snapshot[segment]: - qtd_product_snapshot[segment]['label'] = config.segments[segment].get('label', segment) - - return {'data':qtd_product_snapshot} - - -def update_qtd_performance_gard_metadata(data, hierarchy_order, config): - qtd_gard_metadata = {} - qtd_gard_config = config.config.get('qtd_performance_config', {}) - qtd_gard_metadata["column_order"] = qtd_gard_config.get('column_order', []) - field_schema = {} - for field, field_config in qtd_gard_config.get('field_schema', {}).items(): - field_schema[field] = { - 'label': field_config.get('label', field), - 'format': field_config.get('format', 'amount') - } - qtd_gard_metadata["field_schema"] = field_schema - qtd_gard_metadata['hierarchy_order'] = hierarchy_order - - data['metadata'] = qtd_gard_metadata - - -def update_qtd_performance_product_metadata(data, config): - qtd_product_metadata = {} - qtd_product_config = config.config.get('qtd_performance_config', {}) - qtd_product_metadata["column_order"] = qtd_product_config.get('column_order', []) - field_schema = {} - for field, field_config in qtd_product_config.get('field_schema', {}).items(): - field_schema[field] = { - 'label': field_config.get('label', field), - 'format': field_config.get('format', 'amount') - } - qtd_product_metadata["field_schema"] = field_schema - segments_order = qtd_product_config.get('segment_order', []) - qtd_product_metadata['segment_order'] = segments_order - segments = [] - for segment_order in segments_order: - for i in segment_order: - segments.append(i) - segments += segment_order[i] - segments_breakdown = [] - for segment in segments: - segments_breakdown.append({"key": segment, - "segment": segment, - "label": config.segments[segment].get('label', segment), - 'segment_id': config.segments[segment].get('filter')}) - qtd_product_metadata['segment_breakdown'] = segments_breakdown - data['metadata'] = qtd_product_metadata - - -def fetch_pipeline_gard_parameters(config, req_segments, period): - pipeline_config = config.config.get('pipeline_config', {}) - req_fields = pipeline_config.get('fields', {}) - req_segments_map = {segment: config.segments.get(segment, {}) for segment in req_segments} - as_of_timestamp = get_period_as_of(period) - weekly_periods = find_req_periods(period) - return req_segments_map, as_of_timestamp, weekly_periods, req_fields - - -def get_pipeline_data(time_context, - period, - nodes , - fields, - req_segments, - as_of_timestamp, - config, - eligible_nodes_for_segs, - include_children_data = False): - - parent_child_map, all_nodes = get_parent_child_metadata(nodes, as_of_timestamp, include_children_data) - descendants = [(node_id, {}, {}) for node_id in all_nodes] - fm_recs = {} - fm_recs.update(bulk_fetch_recs_by_timestamp(time_context, - descendants, - fields, - req_segments, - config, - eligible_nodes_for_segs, - [None])) - return fm_recs - - -def fetch_pipeline_gard_snapshot(req_nodes, - req_fields, - period, - segment, - fm_recs, - labels): - - pipeline_gard_snapshot = {} - - for node_id , field, timestamp in product(req_nodes, req_fields, [None]): - if node_id not in pipeline_gard_snapshot: - pipeline_gard_snapshot[node_id] = {'fields': {}} - - val = fm_recs.get((period, node_id, field, segment, timestamp), {}).get('val', 0) - - pipeline_gard_snapshot[node_id]['fields'][field] = {'value': val} - if 'label' not in pipeline_gard_snapshot[node_id]: - pipeline_gard_snapshot[node_id]['label'] = labels.get(node_id) - - return {'data': pipeline_gard_snapshot} - - -def update_pipeline_gard_metadata(data, hierarchy_order, config): - pipeline_gard_metadata = {} - pipeline_config = config.config.get('pipeline_config', {}) - pipeline_gard_metadata["column_order"] = pipeline_config.get('column_order', []) - field_schema = {} - for field, field_config in pipeline_config.get('field_schema', {}).items(): - field_schema[field] = { - 'label': field_config.get('label', field), - 'format': field_config.get('format', 'amount') - } - pipeline_gard_metadata["field_schema"] = field_schema - pipeline_gard_metadata['hierarchy_order'] = hierarchy_order - - data['metadata'] = pipeline_gard_metadata - - -def fetch_pipeline_product_snapshot(node_id, - config, - req_fields, - period, - segments, - fm_recs): - - pipeline_product_snapshot = {} - - for segment, field, timestamp in product(segments, req_fields, [None]): - if segment not in pipeline_product_snapshot: - pipeline_product_snapshot[segment] = {'fields': {}} - - val = fm_recs.get((period, node_id, field, segment, timestamp), {}).get('val', 0) - - pipeline_product_snapshot[segment]['fields'][field] = {'value': val} - if 'label' not in pipeline_product_snapshot: - pipeline_product_snapshot[segment]['label'] = config.segments[segment].get('label', segment) - - return {'data': pipeline_product_snapshot} - - -def update_pipeline_product_metadata(data, config): - pipeline_product_metadata = {} - pipeline_config = config.config.get('pipeline_config', {}) - pipeline_product_metadata["column_order"] = pipeline_config.get('column_order', []) - field_schema = {} - for field, field_config in pipeline_config.get('field_schema', {}).items(): - field_schema[field] = { - 'label': field_config.get('label', field), - 'format': field_config.get('format', 'amount') - } - pipeline_product_metadata["field_schema"] = field_schema - segments_order = pipeline_config.get('segment_order', []) - pipeline_product_metadata['segment_order'] = segments_order - segments = [] - for segment_order in segments_order: - for i in segment_order: - segments.append(i) - segments += segment_order[i] - segments_breakdown = [] - for segment in segments: - segments_breakdown.append({"key": segment, - "segment": segment, - "label": config.segments[segment].get('label', segment), - 'segment_id': config.segments[segment].get('filter')}) - pipeline_product_metadata['segment_breakdown'] = segments_breakdown - data['metadata'] = pipeline_product_metadata - - -def find_monthly_periods_info(q_period): - - periodconfig = PeriodsConfig() - quarter_month_range = monthly_periods(q_period) - now = get_now(periodconfig) - now_dt = now.as_datetime() - monthly_periods_data = {} - for month_range in quarter_month_range: - month_info = render_period(month_range, now_dt, periodconfig) - monthly_periods_data[month_range.mnemonic] = month_info - - return monthly_periods_data - - -def find_quarterly_periods_info(q_periods): - - avail_qs = [prd for prd in get_all_periods__m('Q') if prd.mnemonic in q_periods] - periodconfig = PeriodsConfig() - now = get_now(periodconfig) - now_dt = now.as_datetime() - - quarterly_periods_data = {} - for quarter_range in avail_qs: - quarter_range_info = render_period(quarter_range, now_dt, periodconfig) - quarterly_periods_data[quarter_range.mnemonic] = quarter_range_info - - return quarterly_periods_data - - -def find_monthly_periods(q_period): - req_periods = [] - quarter_month_range = monthly_periods(q_period) - for month_range in quarter_month_range: - req_periods.append(month_range.mnemonic) - return req_periods - - -def get_fm_monthly_records(period, descendants, config, eligible_nodes_for_segs, fm_monthly_fields, segment_map, cache = {}): - - monthly_periods = find_monthly_periods(period) - non_DR_fields, cur_DR_fields = [], [] - time_context = get_time_context(period, config=None, quarter_editable=config.quarter_editable) - fields_by_type, _ = get_fields_by_type(fm_monthly_fields, config, time_context) - - for _field in fm_monthly_fields: - if 'DR' in fields_by_type and _field in fields_by_type['DR']: - cur_DR_fields.append(_field) - else: - non_DR_fields.append(_field) - - if cur_DR_fields: - time_context_list = [get_time_context(fm_rec_period, config=None, quarter_editable=config.quarter_editable) for fm_rec_period in monthly_periods] - fetch_fm_rec_in_period(descendants, cur_DR_fields, segment_map, config, eligible_nodes_for_segs, monthly_periods[0], cache, time_context_list = time_context_list) - - if non_DR_fields: - for fm_rec_period in monthly_periods: - fetch_fm_rec_in_period(descendants, non_DR_fields, segment_map, config, eligible_nodes_for_segs, fm_rec_period, cache) - - return cache - -def find_recursive_fm_monthly_require_field_val(mnemonic_period, - node_id, - require_field, - weekly_segment, - timestamp, - weekly_forecast_config, - sorted_monthly_periods_data, - monthly_fields_source, - quarter_fields_source, - weekly_aggregate_monthly_source, - monthly_calculation_cache, - is_leaf): - - - cache_key = (mnemonic_period, node_id, require_field, weekly_segment, timestamp) - - if cache_key in monthly_calculation_cache: - return monthly_calculation_cache[cache_key] - - field_config = weekly_forecast_config.get('fields', {}).get(require_field, {}) - - if 'schema_based' in field_config['monthly_info']: - target_schema = "rep" if is_leaf else "grid" - require_field_function = field_config['monthly_info']['schema_based'][target_schema]['function'] - is_default_function = 'default' in field_config['monthly_info']['schema_based'][target_schema] - default_function = field_config['monthly_info']['schema_based'][target_schema].get('default', '') - is_relative_period = 'relative_period' in field_config['monthly_info']['schema_based'][target_schema] - relative_period_values = field_config['monthly_info']['schema_based'][target_schema].get('relative_period', []) - - # Evaluate the require_field function - try: - require_field_val = eval(require_field_function) - except ZeroDivisionError: - require_field_val = 0 - - # Determine relative period - cur_relative_period = sorted_monthly_periods_data.get(mnemonic_period, {}).get('relative_period', None) - - if is_default_function and is_relative_period and cur_relative_period in relative_period_values: - - try: - require_field_val = eval(default_function) - except ZeroDivisionError: - require_field_val = 0 - - - # Cache and return the result - monthly_calculation_cache[cache_key] = require_field_val - return require_field_val - - -def get_monthly_forecast_cache(period, - nodes, - req_segments, - fields_execution_order, - fm_monthly_fields, - segment_map, - config, - eligible_nodes_for_segs, - weekly_calculation_cache, - fm_quarterly_recs, - weekly_forecast_config): - - fm_monthly_recs, monthly_calculation_cache = {}, {} - descendants = [(node_id, {}, {}) for node_id in nodes] - get_fm_monthly_records(period, descendants, config, eligible_nodes_for_segs, fm_monthly_fields, segment_map, fm_monthly_recs) - sorted_monthly_periods_data = find_monthly_periods_info(period) - all_monthly_fields_source = weekly_forecast_config.get('monthly_fields_source', []) - monthly_fields = weekly_forecast_config.get('monthly_fields_source', []) - all_quarter_fields_source = weekly_forecast_config.get('quarter_fields_source', []) - quarter_fields = weekly_forecast_config.get('quarter_fields_source', []) - all_weekly_aggregate_monthly_source = weekly_forecast_config.get('weekly_aggregate_monthly_source', []) - weekly_aggregate_fields = weekly_forecast_config.get('weekly_aggregate_monthly_source', []) - weeks_in_month = get_weekly_periods_in_months(period) - monthly_periods = find_monthly_periods(period) - is_leaf_map = is_leaf_node(nodes, period) - - - for node_id, weekly_segment in product(nodes, req_segments): - is_leaf = is_leaf_map[node_id] - for require_field in fields_execution_order: - field_config = weekly_forecast_config.get('fields', {}).get(require_field, {}) - is_monthly_info_field = 'monthly_info' in field_config - if not is_monthly_info_field: - continue - - for month_mnemonic in monthly_periods: - - req_weeks_periods = weeks_in_month[month_mnemonic] - monthly_fields_source = get_source_list_for_weekly_forecast(month_mnemonic, node_id, weekly_segment, None, - fm_monthly_recs, monthly_fields, all_monthly_fields_source) - - quarter_fields_source = get_source_list_for_weekly_forecast(period, node_id, weekly_segment, None, - fm_quarterly_recs, quarter_fields, all_quarter_fields_source) - - weekly_aggregate_monthly_source = get_source_list_for_aggregate_fields_in_periods(req_weeks_periods, node_id, weekly_segment, None, - weekly_calculation_cache, weekly_aggregate_fields, all_weekly_aggregate_monthly_source) - - find_recursive_fm_monthly_require_field_val(month_mnemonic, - node_id, - require_field, - weekly_segment, - None, - weekly_forecast_config, - sorted_monthly_periods_data, - monthly_fields_source, - quarter_fields_source, - weekly_aggregate_monthly_source, - monthly_calculation_cache, - is_leaf) - - - - return fm_monthly_recs, monthly_calculation_cache - - -def get_monthly_forecast_parameters(config, q_period): - fm_monthly_fields = config.config.get('weekly_forecast_waterfall_config', {}).get('monthly_fields_source') - monthly_periods = find_monthly_periods(q_period) - return fm_monthly_fields, monthly_periods - - -def is_monthly_forecast_field_editable(weekly_segment, target_schema, month_status, field_config, weekly_waterfall_config, config): - is_editable = month_status in field_config['monthly_info']["schema_based"][target_schema].get('editable', []) - - if not is_editable: - return False - - target_field = field_config['monthly_info']['schema_based'][target_schema].get(month_status) - segment_editable = 'segment_func' not in config.segments.get(weekly_segment, {}) - segment_dependent_editability = field_config.get('segment_dependent_editability', False) - - - # If the field is editable and its editability is segment-dependent, check for disabled edit fields or segment_func - if segment_dependent_editability: - if not segment_editable: - return False - - disable_edit_fields = config.segments.get(weekly_segment, {}).get('disable_edit_fields', []) - if target_field in disable_edit_fields: - return False - - return True - - - -def update_months_data_in_weekly_snapshot( - period, nodes, required_fields, req_segments, - monthly_calculation_cache, monthly_periods, snapshot_data, config): - - monthly_periods_info = find_monthly_periods_info(period) - weekly_waterfall_config = config.config.get('weekly_forecast_waterfall_config', {}) - field_configs = weekly_waterfall_config.get('fields', {}) - is_leaf_map = is_leaf_node(nodes, period) - - for node_id, segment_id in product(nodes, req_segments): - node_seg_key = "{}_{}".format(node_id, segment_id) - is_leaf = is_leaf_map[node_id] - node_data = snapshot_data['data'].setdefault(node_seg_key, {}).setdefault('fields', {}) - - for field_id in required_fields: - field_config = field_configs.get(field_id, {}) - monthly_info_config = field_config.get('monthly_info', {}) - - if not monthly_info_config: - continue - - field_data = node_data.setdefault(field_id, {}) - - for month_mnemonic in monthly_periods: - month_status = monthly_periods_info.get(month_mnemonic, {}).get('relative_period') - - if 'schema_based' in monthly_info_config: - target_schema = "rep" if is_leaf else "grid" - cache_key = (month_mnemonic, node_id, field_id, segment_id, None) - value = monthly_calculation_cache.get(cache_key, 0) - - is_editable = is_monthly_forecast_field_editable( - segment_id, target_schema, month_status, field_config, - weekly_waterfall_config, config - ) - - if month_status in monthly_info_config.get('render_period', ['h', 'f', 'c']): - months_data = field_data.setdefault('monthly_info', {}) - months_data[month_mnemonic] = {'value': value, 'editable': is_editable} - if is_editable: - target_field = monthly_info_config['schema_based'][target_schema].get(month_status) - months_data[month_mnemonic]['target_editable_field'] = target_field - - - -def find_recursive_fm_quarterly_require_field_val(mnemonic_period, - node_id, - require_field, - weekly_segment, - timestamp, - weekly_forecast_config, - quarter_period_info, - total_weeks_source, - total_months_source, - quarter_fields_source, - quarterly_calculation_cache, - is_leaf): - - - - cache_key = (mnemonic_period, node_id, require_field, weekly_segment, timestamp) - - if cache_key in quarterly_calculation_cache: - return quarterly_calculation_cache[cache_key] - - field_config = weekly_forecast_config.get('fields', {}).get(require_field, {}) - - if 'schema_based' in field_config['quarterly_info']: - target_schema = "rep" if is_leaf else "grid" - require_field_function = field_config['quarterly_info']['schema_based'][target_schema]['function'] - is_default_function = 'default' in field_config['quarterly_info']['schema_based'][target_schema] - default_function = field_config['quarterly_info']['schema_based'][target_schema].get('default', '') - is_relative_period = 'relative_period' in field_config['quarterly_info']['schema_based'][target_schema] - relative_period_values = field_config['quarterly_info']['schema_based'][target_schema].get('relative_period', []) - - # Evaluate the require_field function +def get_forecast_schedule(nodes, db=None, get_dict={}): try: - require_field_val = eval(require_field_function) - except ZeroDivisionError: - require_field_val = 0 - - # Determine relative period - cur_relative_period = quarter_period_info.get(mnemonic_period, {}).get('relative_period', None) - - if is_default_function and is_relative_period and cur_relative_period in relative_period_values: - - try: - require_field_val = eval(default_function) - except ZeroDivisionError: - require_field_val = 0 - - - # Cache and return the result - quarterly_calculation_cache[cache_key] = require_field_val - return require_field_val - - -def get_quarterly_forecast_cache(period, - nodes, - req_segments, - fields_execution_order, - config, - weekly_calculation_cache, - monthly_calculation_cache, - fm_quarterly_recs, - weekly_forecast_config): - - quarterly_calculation_cache = {} - all_total_months_source = weekly_forecast_config.get('total_months_source', []) - total_months_fields = weekly_forecast_config.get('total_months_source', []) - all_quarter_fields_source = weekly_forecast_config.get('quarter_fields_source', []) - quarter_fields = weekly_forecast_config.get('quarter_fields_source', []) - all_total_weeks_source = weekly_forecast_config.get('total_weeks_source', []) - total_weeks_fields = weekly_forecast_config.get('total_weeks_source', []) - req_weeks_periods = find_req_periods(period) - monthly_periods = find_monthly_periods(period) - quarter_period_info = find_quarterly_periods_info([period]) - is_leaf_map = is_leaf_node(nodes, period) - - - for node_id, weekly_segment in product(nodes, req_segments): - is_leaf = is_leaf_map[node_id] - for require_field in fields_execution_order: - - field_config = weekly_forecast_config.get('fields', {}).get(require_field, {}) - is_quarterly_info_field = 'quarterly_info' in field_config - if not is_quarterly_info_field: - continue - - quarter_fields_source = get_source_list_for_weekly_forecast(period, node_id, weekly_segment, None, - fm_quarterly_recs, quarter_fields, all_quarter_fields_source) - - total_weeks_source = get_source_list_for_aggregate_fields_in_periods(req_weeks_periods, node_id, weekly_segment, None, - weekly_calculation_cache, total_weeks_fields, all_total_weeks_source) - - total_months_source = get_source_list_for_aggregate_fields_in_periods(monthly_periods, node_id, weekly_segment, None, - monthly_calculation_cache, total_months_fields, all_total_months_source) - - find_recursive_fm_quarterly_require_field_val(period, - node_id, - require_field, - weekly_segment, - None, - weekly_forecast_config, - quarter_period_info, - total_weeks_source, - total_months_source, - quarter_fields_source, - quarterly_calculation_cache, - is_leaf) - - return quarterly_calculation_cache - - -def update_quarterly_data_in_weekly_snapshot(period, nodes, required_fields, req_segments, - quarterly_calculation_cache, snapshot_data, config): - - - weekly_waterfall_config = config.config.get('weekly_forecast_waterfall_config', {}) - quarterly_periods_info = find_quarterly_periods_info([period]) - field_configs = weekly_waterfall_config.get('fields', {}) - - for node_id, segment_id in product(nodes, req_segments): - node_seg_key = "{}_{}".format(node_id, segment_id) - node_data = snapshot_data['data'].setdefault(node_seg_key, {}).setdefault('fields', {}) - - for field_id in required_fields: - field_config = field_configs.get(field_id, {}) - quarterly_info_config = field_config.get('quarterly_info', {}) - - if not quarterly_info_config: - continue - - field_data = node_data.setdefault(field_id, {}) - quarter_status = quarterly_periods_info.get(period, {}).get('relative_period') - cache_key = (period, node_id, field_id, segment_id, None) - value = quarterly_calculation_cache.get(cache_key, 0) - - if quarter_status in quarterly_info_config.get('render_period', ['h', 'f', 'c']): - field_data['quarter-total'] = value - - -def get_snapshot_window_feature_data(node, config): - status = None - forecastTimestamp = None - user_det = sec_context.get_effective_user() - userid = user_det.user_id - user_level_schedule = get_user_level_schedule(user_id=userid, user_node=node) - as_of = epoch().as_epoch() - fm_schedule = get_forecast_schedule([node], get_dict=True) - forecastWindow = "unlock" - user_node = node - if not fm_schedule: - return {"forecast_status": config.forecast_window_default, - "forecastTimestamp": as_of} - if fm_schedule.get(user_node).get("recurring"): - lockPeriod = fm_schedule[user_node].get('lockPeriod', 'month') - lockFreq = fm_schedule[user_node].get('lockFreq', 'month') - unlockDay = int(fm_schedule[user_node].get('unlockDay', 0)) - unlocktime = fm_schedule[user_node]['unlocktime'].split(":") if 'unlocktime' in fm_schedule[user_node] \ - else ["00", "00"] - lockDay = int(fm_schedule[user_node]['lockDay']) if 'lockDay' in fm_schedule[user_node] \ - else 0 - locktime = fm_schedule[user_node]['locktime'].split(":") if 'locktime' in fm_schedule[user_node] \ - else ["00", "00"] - timeZone_original = fm_schedule.get(user_node)['timeZone'] if 'timeZone' in fm_schedule[user_node] \ - else "PST" - from micro_fm_app.fm_service.api.forecast_schedule import timezone_map - timeZone = timezone_map[timeZone_original.upper()] - lock_on = None - unlock_on = None - current_year = datetime.datetime.now().year - current_month = datetime.datetime.now().month - if lockPeriod == 'month' or (lockPeriod == 'quarter' and lock_on is None): - unlock_on = datetime.datetime(current_year, current_month, unlockDay, - int(unlocktime[0]), int(unlocktime[1]), 0) - zone = pytz.timezone(timeZone) - unlock_on = zone.localize(unlock_on) - if lockFreq == 'month': - lock_on = datetime.datetime(current_year, current_month, lockDay, - int(locktime[0]), int(locktime[1]), 0) - zone = pytz.timezone(timeZone) - lock_on = zone.localize(lock_on) - elif lockFreq == 'days' or lockFreq == 'weekdays': - lock_on = unlock_on + datetime.timedelta(lockDay) - - if as_of >= epoch(lock_on).as_epoch(): - forecastWindow = "lock" - forecastTimestamp = epoch(lock_on).as_epoch() - if as_of >= epoch(unlock_on).as_epoch() and epoch(unlock_on).as_epoch() > epoch(lock_on).as_epoch(): - forecastWindow = "unlock" - forecastTimestamp = epoch(unlock_on).as_epoch() - else: - forecastWindow = fm_schedule.get(user_node).get("status_non_recurring") - forecastTimestamp = fm_schedule.get(user_node).get("non_recurring_timestamp") - user_key = userid + "_" + user_node - if user_level_schedule and user_key in user_level_schedule: - if forecastTimestamp and forecastTimestamp <= user_level_schedule[user_key]['forecastTimestamp']: - forecastWindow = user_level_schedule[user_key]['forecastWindow'] - forecastTimestamp = user_level_schedule[user_key]['forecastTimestamp'] - return {"forecast_status": forecastWindow, - "forecastTimestamp": forecastTimestamp} - - -def get_node_descendants(node, as_of): - node_descendants = { - node_id for nth_level in list(fetch_descendants(as_of, [node], levels=20, drilldown=True))[0]['descendants'] - for node_id in nth_level.keys() - } - node_descendants.add(node) - return list(node_descendants) - -def _append_transformed_export_data(req_period, as_of_date, node_label, segment_label, field_label, - value, fieldnames, transformed_export_data, node_level = None): - - row_data = { - 'as_of_date': as_of_date, - 'Name': node_label, - 'Segment': segment_label, - 'Period': req_period, - 'Field': field_label, - 'Value': value - } - - if 'Level' in fieldnames and node_level: - row_data.update({'Level': 'Level {}'.format(node_level)}) - - transformed_export_data.append(row_data) - - -def transform_weekly_snapshot_data_into_export_format(q_period, custom_month_order, nodes_labels, nodes_level, - as_of_date, config, fieldnames, weekly_segment_snapshot): - """ - Transform and store weekly snapshot data into export format. - """ - transformed_export_data = [] - for key, dic in weekly_segment_snapshot['data'].items(): - node_key, segment_key = key.rsplit('_', 1)[0], dic['segment'] - node_label = nodes_labels.get(node_key) - node_level = nodes_level[node_key] - segment_label = config.segments.get(segment_key, {}).get('label') - - for field, field_dict in dic['fields'].items(): - total_period_data = {} - field_format = field_dict.get('format', 'amount') - - for week_period, val in field_dict['weeks_period_data_val'].items(): - # Format value based on field format - value = round(float(val * 100 if field_format == 'percentage' else val), 2) - req_period = create_key_from_weekly_mnemonic(q_period, week_period, custom_month_order) - _append_transformed_export_data( - req_period, as_of_date, node_label, segment_label, - field_dict['label'], value, fieldnames, transformed_export_data, node_level - ) - - # Get month name from week period and accumulate values - month_no = int(week_period[4:6]) - month_key = custom_month_order[month_no - 1] - total_period_data[month_key] = total_period_data.get(month_key, 0) + val - total_period_data['quarter-total'] = total_period_data.get('quarter-total', 0) + val + forecast_schedule_coll = db[FORECAST_SCHEDULE_COLL] if db else sec_context.tenant_db[FORECAST_SCHEDULE_COLL] + criteria = {'node_id': {'$in': nodes}} + projections = {"recurring": 1, + "unlockPeriod": 1, + "unlockFreq": 1, + "unlockDay": 1, + "unlocktime": 1, + "lockPeriod": 1, + "lockFreq": 1, + "lockDay": 1, + "locktime": 1, + "timeZone": 1, + "node_id": 1, + "lock_on": 1, + "unlock_on": 1, + "status": 1, + "non_recurring_timestamp": 1, + "status_non_recurring": 1 + } + response = forecast_schedule_coll.find(criteria, projections) + if get_dict: + return {res['node_id']: res for res in response} + return [res for res in response] + except Exception as ex: + logger.exception(ex) + return {'success': False, 'Error': str(ex)} - for cur_period, cur_val in total_period_data.items(): - value = round(float(cur_val * 100 if field_format == 'percentage' else cur_val), 2) - req_period = create_key_from_weekly_mnemonic(q_period, cur_period, custom_month_order) - _append_transformed_export_data( - req_period, as_of_date, node_label, segment_label, - field_dict['label'], value, fieldnames, transformed_export_data, node_level - ) - return transformed_export_data diff --git a/utils/mongo_writer.py b/utils/mongo_writer.py index 4c68ee0..ce50d6e 100644 --- a/utils/mongo_writer.py +++ b/utils/mongo_writer.py @@ -4,7 +4,7 @@ from pymongo import InsertOne, ReturnDocument, UpdateOne from pymongo.errors import BulkWriteError, DuplicateKeyError -from infra import (AUDIT_COLL, DEALS_COLL, DRILLDOWN_COLL, +from infra.constants import (AUDIT_COLL, DEALS_COLL, DRILLDOWN_COLL, DRILLDOWN_LEADS_COLL, HIER_COLL, HIER_LEADS_COLL) from infra.read import (fetch_ancestors, fetch_boundry, fetch_closest_boundaries, fetch_descendant_ids, @@ -496,42 +496,6 @@ def label_node(node, if not is_lead_service(service): _set_run_full_mode_flag(True) - -def order_nodes(ordered_nodes, - config, - signature='order_node', - drilldown=False, - as_to=None - ): - """ - order nodes for display purposes - - Arguments: - ordered_nodes {list} -- children nodes in desired sort order ['0050000FLN2C9I2', ] - config {HierConfig} -- config for hierarchy, from HierConfig ...? - - Keyword Arguments: - signature {str} -- what process triggered action - (default: {'order_node'}) - drilldown {bool} -- if True, write to drilldown collection - (default: {False}) - db {object} -- instance of tenant_db (default: {None}) - if None, will create one - """ - coll = HIER_COLL if not drilldown else DRILLDOWN_COLL - hier_collection = sec_context.tenant_db[coll] - - # TODO: obviously shitty, but does anyone care about this feature - # going backwards so we can sort highest priority to lowest ... - # because otherwise mongo puts all the nulls first - for i, node in enumerate(ordered_nodes[::-1]): - hier_collection.update({'node': node, - 'to': as_to}, {'$set': {'priority': i}}) - - # Enable flag to run the capturedrilldowntask - _set_run_full_mode_flag(True) - - def update_nodes_with_valid_to_timestamp(from_timestamp, drilldown=False, service=None): diff --git a/utils/string_utils.py b/utils/string_utils.py deleted file mode 100644 index e092fd1..0000000 --- a/utils/string_utils.py +++ /dev/null @@ -1,14 +0,0 @@ -import string -import random - - -def first15(s): - if s and len(s)>15: - return s[0:15] - else: - return s - -def random_string(size=8, chars=None): - if chars is None: - chars = string.ascii_letters + string.digits - return ''.join(random.choice(chars) for x in range(size)) diff --git a/utils/time_series_utils.py b/utils/time_series_utils.py index b91f78e..c4e4e66 100644 --- a/utils/time_series_utils.py +++ b/utils/time_series_utils.py @@ -7,26 +7,6 @@ logger = logging.getLogger("gnana.%s" % __name__) Event = namedtuple('Event', ['ts', 'event_type', 'data']) -def slice_timeseries(times, values, at, interpolate=False, use_fv=False, default=0): - """ - get value at 'at' time in timeseries - if 'at' not in timeseries, get value at closest prior time - """ - if not times or not values: - logger.warning("no timeseries data provided, is this a future period?") - return default - if at < times[0]: - return default if not use_fv else values[0] - if interpolate: - try: - return np.interp(at, times, values) - except ValueError: - idx = index_of(at, times) - return values[idx] - else: - idx = index_of(at, times) - return values[idx] - class EventAggregator: