From b5711e381f547dae94ce1018032b506daefe04f9 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 21 Sep 2022 17:38:19 +0530 Subject: [PATCH 01/79] initial changes for k2 integration --- newrelic/admin/record_deploy.py | 2 ++ newrelic/config.py | 6 ++++++ newrelic/core/config.py | 2 ++ newrelic/newrelic.ini | 13 +++++++++++-- setup.py | 1 + 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/newrelic/admin/record_deploy.py b/newrelic/admin/record_deploy.py index 3a6229de66..c4079b346c 100644 --- a/newrelic/admin/record_deploy.py +++ b/newrelic/admin/record_deploy.py @@ -139,6 +139,8 @@ def _args( settings.monitor_mode = False + settings.k2_enabled = False + initialize(config_file) host = settings.host diff --git a/newrelic/config.py b/newrelic/config.py index 456cd722dc..287be6c542 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,6 +328,7 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) + _process_setting(section, "k2_enabled", "getboolean", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) @@ -3080,6 +3081,11 @@ def initialize( _setup_agent_console() else: _settings.enabled = False + if _settings.k2_enabled: + log_message("Invoking K2 security module.", critical=True) + # run k2 agent + import k2_python_agent + k2_python_agent.init_k2({}, {}) def filter_app_factory(app, global_conf, config_file, environment=None): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 60520c1134..77a12ebcdc 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -578,6 +578,8 @@ def default_host(license_key): _settings.monitor_mode = _environ_as_bool("NEW_RELIC_MONITOR_MODE", True) +_settings.k2_enabled = _environ_as_bool("ENABLE_K2_MODE", True) + _settings.developer_mode = _environ_as_bool("NEW_RELIC_DEVELOPER_MODE", False) _settings.high_security = _environ_as_bool("NEW_RELIC_HIGH_SECURITY", False) diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index d06d8a2926..84f2d99a13 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -42,12 +42,18 @@ license_key = *** REPLACE ME *** # https://docs.newrelic.com/docs/apm/agents/manage-apm-agents/app-naming/use-multiple-names-app/ app_name = Python Application +# When "true", the agent enables the k2 security module +# This global switch is normally overridden for +# each environment below. It may also be set using the +# ENABLE_K2_MODE environment variable. +monitor_mode = true + # When "true", the agent collects performance data about your # application and reports this data to the New Relic UI at # newrelic.com. This global switch is normally overridden for # each environment below. It may also be set using the # NEW_RELIC_MONITOR_MODE environment variable. -monitor_mode = true +k2_enabled = true # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the @@ -241,15 +247,18 @@ distributed_tracing.enabled = true [newrelic:development] monitor_mode = false +k2_enabled = false [newrelic:test] monitor_mode = false +k2_enabled = false [newrelic:staging] app_name = Python Application (Staging) monitor_mode = true +k2_enabled = true [newrelic:production] monitor_mode = true - +k2_enabled = true # --------------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 5fdf2005fe..3f90f1a04f 100644 --- a/setup.py +++ b/setup.py @@ -156,6 +156,7 @@ def build_extension(self, ext): }, scripts=["scripts/newrelic-admin"], extras_require={"infinite-tracing": ["grpcio", "protobuf<4"]}, + install_requires=["k2_python_agent @ git+https://github.com/k2io/k2-python-agent@feature/newrelic_integration#egg=k2_python_agent"] ) if with_setuptools: From 20d5117dae5693cc6e920abf2e88240c9dec3936 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 21 Sep 2022 17:48:48 +0530 Subject: [PATCH 02/79] minor fix --- newrelic/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index 287be6c542..2bba619cd0 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3082,7 +3082,6 @@ def initialize( else: _settings.enabled = False if _settings.k2_enabled: - log_message("Invoking K2 security module.", critical=True) # run k2 agent import k2_python_agent k2_python_agent.init_k2({}, {}) From dfbcf0a8ffa8ea71d83ddb091b0803897f108017 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 22 Sep 2022 15:44:02 +0530 Subject: [PATCH 03/79] k2 agent would now init without any args --- newrelic/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index 2bba619cd0..a3bf07b2cf 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3084,7 +3084,7 @@ def initialize( if _settings.k2_enabled: # run k2 agent import k2_python_agent - k2_python_agent.init_k2({}, {}) + k2_python_agent.init_k2() def filter_app_factory(app, global_conf, config_file, environment=None): From f399c3a58422926da6069da792fccb31d985201f Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 27 Sep 2022 16:56:36 +0530 Subject: [PATCH 04/79] initial implementation of securty config, removal of switch based k2 startup --- newrelic/admin/record_deploy.py | 2 -- newrelic/config.py | 14 +++++++++----- newrelic/core/config.py | 13 ++++++++++--- newrelic/newrelic.ini | 26 +++++++++++++++----------- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/newrelic/admin/record_deploy.py b/newrelic/admin/record_deploy.py index c4079b346c..3a6229de66 100644 --- a/newrelic/admin/record_deploy.py +++ b/newrelic/admin/record_deploy.py @@ -139,8 +139,6 @@ def _args( settings.monitor_mode = False - settings.k2_enabled = False - initialize(config_file) host = settings.host diff --git a/newrelic/config.py b/newrelic/config.py index a3bf07b2cf..1c45e910eb 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,7 +328,11 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) - _process_setting(section, "k2_enabled", "getboolean", None) + _process_setting(section, "security.enabled", "getboolean", False) + _process_setting(section, "security.mode", "get", "RASP") + _process_setting(section, "security.validator_service_endpoint_url", "get", "") + _process_setting(section, "security.resource_service_endpoint_url", "get", "") + _process_setting(section, "security.accessor_token", "get", "") _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) @@ -3081,10 +3085,10 @@ def initialize( _setup_agent_console() else: _settings.enabled = False - if _settings.k2_enabled: - # run k2 agent - import k2_python_agent - k2_python_agent.init_k2() + + # run k2 agent + import k2_python_agent + k2_python_agent.init_k2() def filter_app_factory(app, global_conf, config_file, environment=None): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 77a12ebcdc..83efe8be5c 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -275,6 +275,9 @@ class ApplicationLoggingMetricsSettings(Settings): class ApplicationLoggingLocalDecoratingSettings(Settings): pass +class SecurityModuleSettings(Settings): + pass + class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -398,7 +401,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.infinite_tracing = InfiniteTracingSettings() _settings.event_harvest_config = EventHarvestConfigSettings() _settings.event_harvest_config.harvest_limits = EventHarvestConfigHarvestLimitSettings() - +_settings.security = SecurityModuleSettings() _settings.log_file = os.environ.get("NEW_RELIC_LOG", None) _settings.audit_log_file = os.environ.get("NEW_RELIC_AUDIT_LOG", None) @@ -578,8 +581,6 @@ def default_host(license_key): _settings.monitor_mode = _environ_as_bool("NEW_RELIC_MONITOR_MODE", True) -_settings.k2_enabled = _environ_as_bool("ENABLE_K2_MODE", True) - _settings.developer_mode = _environ_as_bool("NEW_RELIC_DEVELOPER_MODE", False) _settings.high_security = _environ_as_bool("NEW_RELIC_HIGH_SECURITY", False) @@ -823,6 +824,12 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) +_settings.security.enabled = False +_settings.security.mode = "RASP" +_settings.security.validator_service_endpoint_url = "" +_settings.security.resource_service_endpoint_url = "" +_settings.security.accessor_token = "" + def global_settings(): """This returns the default global settings. Generally only used diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 84f2d99a13..58a62f695d 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -42,18 +42,26 @@ license_key = *** REPLACE ME *** # https://docs.newrelic.com/docs/apm/agents/manage-apm-agents/app-naming/use-multiple-names-app/ app_name = Python Application -# When "true", the agent enables the k2 security module -# This global switch is normally overridden for -# each environment below. It may also be set using the -# ENABLE_K2_MODE environment variable. -monitor_mode = true - # When "true", the agent collects performance data about your # application and reports this data to the New Relic UI at # newrelic.com. This global switch is normally overridden for # each environment below. It may also be set using the # NEW_RELIC_MONITOR_MODE environment variable. -k2_enabled = true +monitor_mode = true + +# When enabled, the agent collects security data about your +# application and reports this data to the New Relic. +security.enable = true +# security module provides two modes IAST or RASP +# RASP stands for Runtime Application Self Protection +# while IAST for Interactive Application Security Testing +# Default mode is RASP +security.mode = RASP +# web-protect agent endpoint connection URLs +security.validator_service_endpoint_url = ws://localhost:54321 +security.resource_service_endpoint_url = http://localhost:54322 +# web-protect agent accessor token +security.accessor_token = # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the @@ -247,18 +255,14 @@ distributed_tracing.enabled = true [newrelic:development] monitor_mode = false -k2_enabled = false [newrelic:test] monitor_mode = false -k2_enabled = false [newrelic:staging] app_name = Python Application (Staging) monitor_mode = true -k2_enabled = true [newrelic:production] monitor_mode = true -k2_enabled = true # --------------------------------------------------------------------------- From c3232c915a303dad2fbb60ff3b823a7f99d38244 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 28 Sep 2022 13:02:42 +0530 Subject: [PATCH 05/79] minor fix with incorrect value for configuration mapper --- newrelic/config.py | 10 +++++----- newrelic/newrelic.ini | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 1c45e910eb..7d4fdea705 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,11 +328,11 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) - _process_setting(section, "security.enabled", "getboolean", False) - _process_setting(section, "security.mode", "get", "RASP") - _process_setting(section, "security.validator_service_endpoint_url", "get", "") - _process_setting(section, "security.resource_service_endpoint_url", "get", "") - _process_setting(section, "security.accessor_token", "get", "") + _process_setting(section, "security.enabled", "getboolean", None) + _process_setting(section, "security.mode", "get", None) + _process_setting(section, "security.validator_service_endpoint_url", "get", None) + _process_setting(section, "security.resource_service_endpoint_url", "get", None) + _process_setting(section, "security.accessor_token", "get", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 58a62f695d..3b64167fea 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -51,7 +51,7 @@ monitor_mode = true # When enabled, the agent collects security data about your # application and reports this data to the New Relic. -security.enable = true +security.enable = false # security module provides two modes IAST or RASP # RASP stands for Runtime Application Self Protection # while IAST for Interactive Application Security Testing From 206160e865931fa806b3adde6fc9da60b6b19cfa Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 30 Sep 2022 01:19:29 +0530 Subject: [PATCH 06/79] changes corresponding to latest k2 agent changes --- newrelic/config.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 7d4fdea705..ae16462aac 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3087,8 +3087,14 @@ def initialize( _settings.enabled = False # run k2 agent - import k2_python_agent - k2_python_agent.init_k2() + from k2_python_agent import AgentConfig, ModuleLoadAgent + config = AgentConfig() + config.set_base_config(_settings.security) + config.application_name = _settings.app_name + # TODO: replace with identified app id + config.application_id = _settings.application_id + + ModuleLoadAgent().initialise() def filter_app_factory(app, global_conf, config_file, environment=None): From dd7bd2d4fe0bd22ac58c5747e6a4ef209349455c Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 30 Sep 2022 02:31:37 +0530 Subject: [PATCH 07/79] minor fixes --- newrelic/config.py | 4 ++-- newrelic/core/config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index ae16462aac..e4b21d281b 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3092,9 +3092,9 @@ def initialize( config.set_base_config(_settings.security) config.application_name = _settings.app_name # TODO: replace with identified app id - config.application_id = _settings.application_id + config.application_id = None - ModuleLoadAgent().initialise() + ModuleLoadAgent(config).initialise() def filter_app_factory(app, global_conf, config_file, environment=None): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 83efe8be5c..63ca6f7ac3 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -824,7 +824,7 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) -_settings.security.enabled = False +_settings.security.enable = False _settings.security.mode = "RASP" _settings.security.validator_service_endpoint_url = "" _settings.security.resource_service_endpoint_url = "" From 7578dc00df8e2626329ae595ef25cbe656dfb1f7 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 30 Sep 2022 16:25:29 +0530 Subject: [PATCH 08/79] updated integration source for k2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f90f1a04f..c52cd3baa1 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): }, scripts=["scripts/newrelic-admin"], extras_require={"infinite-tracing": ["grpcio", "protobuf<4"]}, - install_requires=["k2_python_agent @ git+https://github.com/k2io/k2-python-agent@feature/newrelic_integration#egg=k2_python_agent"] + install_requires=["k2_python_agent @ git+https://github.com/k2io/k2-python-agent@newrelic_integration#egg=k2_python_agent"] ) if with_setuptools: From 857f3f9102c72c34baf5366c5a8393044f22c391 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 30 Sep 2022 18:05:51 +0530 Subject: [PATCH 09/79] Added customer id --- newrelic/config.py | 1 + newrelic/core/config.py | 1 + newrelic/newrelic.ini | 1 + 3 files changed, 3 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index e4b21d281b..886b321f20 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -333,6 +333,7 @@ def _process_configuration(section): _process_setting(section, "security.validator_service_endpoint_url", "get", None) _process_setting(section, "security.resource_service_endpoint_url", "get", None) _process_setting(section, "security.accessor_token", "get", None) + _process_setting(section, "security.customer_id", "get", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 63ca6f7ac3..3eb37299a3 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -829,6 +829,7 @@ def default_host(license_key): _settings.security.validator_service_endpoint_url = "" _settings.security.resource_service_endpoint_url = "" _settings.security.accessor_token = "" +_settings.security.customer_id = "" def global_settings(): diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 3b64167fea..e5e8f3e531 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -62,6 +62,7 @@ security.validator_service_endpoint_url = ws://localhost:54321 security.resource_service_endpoint_url = http://localhost:54322 # web-protect agent accessor token security.accessor_token = +security.customer_id = # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the From f0c30a14b815a5ac95e14f7aa99a98860791d027 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sat, 1 Oct 2022 18:42:33 +0530 Subject: [PATCH 10/79] initial changes for application id propagation --- newrelic/config.py | 48 +++++++++++++++++++++++++++++++++-------- newrelic/core/config.py | 2 +- newrelic/newrelic.ini | 11 ++++++++-- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 886b321f20..dfa14cee03 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -329,6 +329,7 @@ def _process_configuration(section): _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) _process_setting(section, "security.enabled", "getboolean", None) + _process_setting(section, "security.force_complete_disable", "getboolean", None) _process_setting(section, "security.mode", "get", None) _process_setting(section, "security.validator_service_endpoint_url", "get", None) _process_setting(section, "security.resource_service_endpoint_url", "get", None) @@ -3060,6 +3061,43 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) +def _generate_security_module_config(): + from k2_python_agent import AgentConfig + config = AgentConfig() + config.set_base_config(_settings.security) + # propogate app name and id + config.application_name = _settings.app_name + config.application_id = _settings.application_id + + return config + + +def _update_security_module(agent): + config = _generate_security_module_config() + agent.refresh_agent(config) + + +def _setup_security_module(): + """Initiates k2 security module and adds a + callback to agent startup to propagate NR config + """ + if _settings.security.force_complete_disable: + return + + # run security module + from k2_python_agent import AgentConfig, ModuleLoadAgent + from functools import partial as Partial + + config =_generate_security_module_config() + + security_module_agent = ModuleLoadAgent(config) + security_module_agent.initialise() + + # create a callback to reinitialise the security module + callback = Partial(_update_security_module, security_module_agent) + newrelic.core.agent.Agent.run_on_startup(callback) + + def initialize( config_file=None, environment=None, @@ -3087,15 +3125,7 @@ def initialize( else: _settings.enabled = False - # run k2 agent - from k2_python_agent import AgentConfig, ModuleLoadAgent - config = AgentConfig() - config.set_base_config(_settings.security) - config.application_name = _settings.app_name - # TODO: replace with identified app id - config.application_id = None - - ModuleLoadAgent(config).initialise() + _setup_security_module() def filter_app_factory(app, global_conf, config_file, environment=None): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 3eb37299a3..e1e1381021 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -824,6 +824,7 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) +_settings.security.force_complete_disable = False _settings.security.enable = False _settings.security.mode = "RASP" _settings.security.validator_service_endpoint_url = "" @@ -831,7 +832,6 @@ def default_host(license_key): _settings.security.accessor_token = "" _settings.security.customer_id = "" - def global_settings(): """This returns the default global settings. Generally only used directly in test scripts and test harnesses or when applying global diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index e5e8f3e531..1946eb3abd 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -52,6 +52,7 @@ monitor_mode = true # When enabled, the agent collects security data about your # application and reports this data to the New Relic. security.enable = false +security.force_complete_disable = false # security module provides two modes IAST or RASP # RASP stands for Runtime Application Self Protection # while IAST for Interactive Application Security Testing @@ -60,9 +61,15 @@ security.mode = RASP # web-protect agent endpoint connection URLs security.validator_service_endpoint_url = ws://localhost:54321 security.resource_service_endpoint_url = http://localhost:54322 + # web-protect agent accessor token -security.accessor_token = -security.customer_id = +# security.sec_home_path = +# security.sec_log_file_name = + +# disable vulnerability type flags +# security.detection.disable_rci = +# security.detection.disable_rxss = +# security.detection.disable_desearlization = # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the From 42d621a200e2948429d4cb6978d1bb8676eff1cc Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 2 Oct 2022 01:20:44 +0530 Subject: [PATCH 11/79] minor fix in application_id extraction --- newrelic/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index dfa14cee03..8b421b89d1 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3066,8 +3066,13 @@ def _generate_security_module_config(): config = AgentConfig() config.set_base_config(_settings.security) # propogate app name and id + agent_instance = newrelic.core.agent.agent_instance() + application = agent_instance.application(_settings.app_name) + configuration = application.configuration + + if configuration and hasattr(configuration, 'primary_application_id'): + config.application_id = configuration.primary_application_id config.application_name = _settings.app_name - config.application_id = _settings.application_id return config From e700fb2df80db2c75c897365d84b70e21a76c712 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 2 Oct 2022 01:23:42 +0530 Subject: [PATCH 12/79] minor fix --- newrelic/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 8b421b89d1..0c04fbca8b 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3068,9 +3068,8 @@ def _generate_security_module_config(): # propogate app name and id agent_instance = newrelic.core.agent.agent_instance() application = agent_instance.application(_settings.app_name) - configuration = application.configuration - - if configuration and hasattr(configuration, 'primary_application_id'): + if application: + configuration = application.configuration config.application_id = configuration.primary_application_id config.application_name = _settings.app_name From 1d5964b801a6086ab4d140c544ee0dfaeb696fe2 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 2 Oct 2022 11:58:17 +0530 Subject: [PATCH 13/79] guid propagation instead of application_id --- newrelic/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index 0c04fbca8b..c9c472f79e 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3070,7 +3070,7 @@ def _generate_security_module_config(): application = agent_instance.application(_settings.app_name) if application: configuration = application.configuration - config.application_id = configuration.primary_application_id + config.application_id = configuration.entity_guid config.application_name = _settings.app_name return config @@ -3079,6 +3079,7 @@ def _generate_security_module_config(): def _update_security_module(agent): config = _generate_security_module_config() agent.refresh_agent(config) + agent.connect() def _setup_security_module(): From 163645b09455c440a3d594be98afcec7102c29f5 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 3 Oct 2022 00:19:52 +0530 Subject: [PATCH 14/79] This implements propagation of all possible k2 config in NR's config --- newrelic/config.py | 23 ++++++- newrelic/core/config.py | 148 +++++++++++++++++++++++++++++++++++++--- newrelic/newrelic.ini | 47 ++++++++++--- 3 files changed, 197 insertions(+), 21 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index c9c472f79e..ba87d71bf4 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,13 +328,32 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) - _process_setting(section, "security.enabled", "getboolean", None) _process_setting(section, "security.force_complete_disable", "getboolean", None) + _process_setting(section, "security.enable", "getboolean", None) _process_setting(section, "security.mode", "get", None) _process_setting(section, "security.validator_service_endpoint_url", "get", None) _process_setting(section, "security.resource_service_endpoint_url", "get", None) _process_setting(section, "security.accessor_token", "get", None) - _process_setting(section, "security.customer_id", "get", None) + _process_setting(section, "security.customer_id", "getint", None) + _process_setting(section, "security.log_level", "get", None) + _process_setting(section, "security.sec_home_path", "get", None) + _process_setting(section, "security.sec_log_file_name", "get", None) + _process_setting(section, "security.detection.disable_rci", "getboolean", None) + _process_setting(section, "security.detection.disable_rxss", "getboolean", None) + _process_setting(section, "security.detection.disable_deserialization", "getboolean", None) + _process_setting(section, "security.policy.vulnerabilityScan.enabled", "getboolean", None) + _process_setting(section, "security.policy.vulnerabilityScan.iastScan.enabled", "getboolean", None) + _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.interval", "getint", None) + _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.batchSize", "getint", None) + _process_setting(section, "security.policy.protectionMode.enabled", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.ipBlocking.enabled", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.ipBlocking.attackerIpBlocking", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.ipBlocking.ipDetectViaXFF", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.ipBlocking.timeout", "getint", None) + _process_setting(section, "security.policy.protectionMode.apiBlocking.enabled", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.apiBlocking.protectAllApis", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.apiBlocking.protectAttackedApis", "getboolean", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index e1e1381021..a31a2b53dd 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -278,6 +278,30 @@ class ApplicationLoggingLocalDecoratingSettings(Settings): class SecurityModuleSettings(Settings): pass +class SecurityDetectionSettings(Settings): + pass + +class SecurityPolicySettings(Settings): + pass + +class SecurityPolicyVulnerabilityScanSettings(Settings): + pass + +class SecurityPolicyIASTSettings(Settings): + pass + +class SecurityPolicyIASTProbingSettings(Settings): + pass + +class SecurityPolicyprotectionModeSettings(Settings): + pass + +class SecurityPolicyIPBlockingSettings(Settings): + pass + +class SecurityPolicyAPIBlockingSettings(Settings): + pass + class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -401,7 +425,6 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.infinite_tracing = InfiniteTracingSettings() _settings.event_harvest_config = EventHarvestConfigSettings() _settings.event_harvest_config.harvest_limits = EventHarvestConfigHarvestLimitSettings() -_settings.security = SecurityModuleSettings() _settings.log_file = os.environ.get("NEW_RELIC_LOG", None) _settings.audit_log_file = os.environ.get("NEW_RELIC_AUDIT_LOG", None) @@ -824,13 +847,122 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) -_settings.security.force_complete_disable = False -_settings.security.enable = False -_settings.security.mode = "RASP" -_settings.security.validator_service_endpoint_url = "" -_settings.security.resource_service_endpoint_url = "" -_settings.security.accessor_token = "" -_settings.security.customer_id = "" +_settings.security = SecurityModuleSettings() +_settings.security.detection = SecurityDetectionSettings() +_settings.security.policy = SecurityPolicySettings() +_settings.security.policy.vulnerabilityScan = SecurityPolicyVulnerabilityScanSettings() +_settings.security.policy.vulnerabilityScan = SecurityPolicyIASTSettings() +_settings.security.policy.vulnerabilityScan = SecurityPolicyIASTProbingSettings() +_settings.security.policy.protectionMode = SecurityPolicyprotectionModeSettings() +_settings.security.policy.protectionMode.ipBlocking = SecurityPolicyIPBlockingSettings() +_settings.security.policy.protectionMode.ipBlocking = SecurityPolicyAPIBlockingSettings() + +_settings.security.force_complete_disable = _environ_as_bool( + "NEW_RELIC_SECURITY_FORCE_COMPLETE_DISABLE", + default=False +) +_settings.security.enable = _environ_as_bool( + "NEW_RELIC_SECURITY_ENABLE", + default=False +) +_settings.security.mode = os.environ.get( + "NEW_RELIC_SECURITY_MODE", + default="RASP" +) +_settings.security.validator_service_endpoint_url = os.environ.get( + "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_ENDPOINT_URL", + default=None +) +_settings.security.resource_service_endpoint_url = os.environ.get( + "NEW_RELIC_SECURITY_RESOURCE_SERVICE_ENDPOINT_URL", + default=None +) +_settings.security.accessor_token = os.environ.get( + "NEW_RELIC_SECURITY_ACCESSOR_TOKEN", + default=None +) +_settings.security.customer_id = os.environ.get( + "NEW_RELIC_SECURITY_CUSTOMER_ID", + default=None +) +_settings.security.log_level = os.environ.get( + "NEW_RELIC_SECURITY_LOG_LEVEL", + default="INFO" +) +_settings.security.sec_home_path = os.environ.get( + "NEW_RELIC_SECURITY_SEC_HOME_PATH", + default="/opt" +) +_settings.security.sec_log_file_name = os.environ.get( + "NEW_RELIC_SECURITY_SEC_LOG_FILE_NAME", + default="/temp/security_log" +) +_settings.security.detection.disable_rci = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_DISABLE_RCI", + default=False +) +_settings.security.detection.disable_rxss = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_DISABLE_RXSS", + default=False +) +_settings.security.detection.disable_deserialization = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_DISABLE_DESERIALIZATION", + default=False +) + +_settings.security.policy.vulnerabilityScan.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_ENABLED", + default=False +) +_settings.security.policy.vulnerabilityScan.iastScan.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_ENABLED", + default=False +) +_settings.security.policy.vulnerabilityScan.iastScan.probing.interval = _environ_as_int( + "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_INTERVAL", + default=1 +) +_settings.security.policy.vulnerabilityScan.iastScan.probing.batchSize = _environ_as_int( + "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_BATCHSIZE", + default=5 +) +_settings.security.policy.protectionMode.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_ENABLED", + default=False +) +_settings.security.policy.protectionMode.ipBlocking.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ENABLED", + default=False +) +_settings.security.policy.protectionMode.ipBlocking.attackerIpBlocking = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ATTACKERIPBLOCKING", + default=False +) +_settings.security.policy.protectionMode.ipBlocking.ipDetectViaXFF = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_IPDETECTVIAXFF", + default=False +) +_settings.security.policy.protectionMode.ipBlocking.timeout = _environ_as_int( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_TIMEOUT", + default=120 +) +_settings.security.policy.protectionMode.apiBlocking.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_ENABLED", + default=False +) +_settings.security.policy.protectionMode.apiBlocking.protectAllApis = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTALLAPIS", + default=False +) +_settings.security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTKNOWNVULNERABLEAPIS", + default=False +) +_settings.security.policy.protectionMode.apiBlocking.protectAttackedApis = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTATTACKEDAPIS", + default=False +) + def global_settings(): """This returns the default global settings. Generally only used diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 1946eb3abd..d985f87a02 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -49,27 +49,52 @@ app_name = Python Application # NEW_RELIC_MONITOR_MODE environment variable. monitor_mode = true +# When enabled, the agent's security module is not loaded +# this would mean changes on soft reloads won't work +security.force_complete_disable = false # When enabled, the agent collects security data about your # application and reports this data to the New Relic. -security.enable = false -security.force_complete_disable = false +security.enable = true # security module provides two modes IAST or RASP # RASP stands for Runtime Application Self Protection # while IAST for Interactive Application Security Testing # Default mode is RASP security.mode = RASP + # web-protect agent endpoint connection URLs -security.validator_service_endpoint_url = ws://localhost:54321 -security.resource_service_endpoint_url = http://localhost:54322 +security.validator_service_endpoint_url = +security.resource_service_endpoint_url = # web-protect agent accessor token -# security.sec_home_path = -# security.sec_log_file_name = - -# disable vulnerability type flags -# security.detection.disable_rci = -# security.detection.disable_rxss = -# security.detection.disable_desearlization = +security.accessor_token = +security.customer_id = + +# log level +security.log_level = INFO + +# security file settings +# security.sec_home_path = /opt +# security.sec_log_file_name = /tmp/security.log + +# vulnerabilty detection disable +# security.detection.disable_rci = false +# security.detection.disable_rxss = false +# security.detection.disable_deserialization = false + +# policy settings +# security.policy.vulnerabilityScan.enabled = false +# security.policy.vulnerabilityScan.iastScan.enabled = false +# security.policy.vulnerabilityScan.iastScan.probing.interval = 1 +# security.policy.vulnerabilityScan.iastScan.probing.batchSize = 5 +# security.policy.protectionMode.enabled = false +# security.policy.protectionMode.ipBlocking.enabled = false +# security.policy.protectionMode.ipBlocking.attackerIpBlocking = false +# security.policy.protectionMode.ipBlocking.ipDetectViaXFF = false +# security.policy.protectionMode.ipBlocking.timeout = 120 +# security.policy.protectionMode.apiBlocking.enabled = false +# security.policy.protectionMode.apiBlocking.protectAllApis = false +# security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = false +# security.policy.protectionMode.apiBlocking.protectAttackedApis = false # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the From 52e59e2cb6348d48483576345c59e029548ecad4 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 3 Oct 2022 15:05:43 +0530 Subject: [PATCH 15/79] changes to populate dictionary of policy and changes to reflect security.enable flag on startup --- newrelic/config.py | 12 ++++++++++-- newrelic/core/config.py | 30 ++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index ba87d71bf4..d598774b63 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3080,6 +3080,10 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) +def _generate_security_policy(): + return _settings.security.policy.to_deep_dict() + + def _generate_security_module_config(): from k2_python_agent import AgentConfig config = AgentConfig() @@ -3098,7 +3102,8 @@ def _generate_security_module_config(): def _update_security_module(agent): config = _generate_security_module_config() agent.refresh_agent(config) - agent.connect() + if _settings.security.enable: + agent.connect() def _setup_security_module(): @@ -3107,16 +3112,19 @@ def _setup_security_module(): """ if _settings.security.force_complete_disable: return - # run security module from k2_python_agent import AgentConfig, ModuleLoadAgent from functools import partial as Partial config =_generate_security_module_config() + policy = _generate_security_policy() security_module_agent = ModuleLoadAgent(config) security_module_agent.initialise() + security_module_agent.set_policy(policy) + if not _settings.security.enable: + security_module_agent.disable() # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) newrelic.core.agent.Agent.run_on_startup(callback) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index a31a2b53dd..81a9024d29 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -94,6 +94,16 @@ def __contains__(self, item): return hasattr(self, item) +class DeepDictableSetting: + def to_deep_dict(self): + _dict = self.__dict__ + for attr in _dict.keys(): + value = _dict[attr] + if isinstance(value, DeepDictableSetting): + _dict[attr] = value.to_deep_dict() + return _dict + + def create_settings(nested): return type("Settings", (Settings,), {"nested": nested})() @@ -281,25 +291,25 @@ class SecurityModuleSettings(Settings): class SecurityDetectionSettings(Settings): pass -class SecurityPolicySettings(Settings): +class SecurityPolicySettings(Settings, DeepDictableSetting): pass -class SecurityPolicyVulnerabilityScanSettings(Settings): +class SecurityPolicyVulnerabilityScanSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyIASTSettings(Settings): +class SecurityPolicyIASTSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyIASTProbingSettings(Settings): +class SecurityPolicyIASTProbingSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyprotectionModeSettings(Settings): +class SecurityPolicyprotectionModeSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyIPBlockingSettings(Settings): +class SecurityPolicyIPBlockingSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyAPIBlockingSettings(Settings): +class SecurityPolicyAPIBlockingSettings(Settings, DeepDictableSetting): pass @@ -851,11 +861,11 @@ def default_host(license_key): _settings.security.detection = SecurityDetectionSettings() _settings.security.policy = SecurityPolicySettings() _settings.security.policy.vulnerabilityScan = SecurityPolicyVulnerabilityScanSettings() -_settings.security.policy.vulnerabilityScan = SecurityPolicyIASTSettings() -_settings.security.policy.vulnerabilityScan = SecurityPolicyIASTProbingSettings() +_settings.security.policy.vulnerabilityScan.iastScan = SecurityPolicyIASTSettings() +_settings.security.policy.vulnerabilityScan.iastScan.probing = SecurityPolicyIASTProbingSettings() _settings.security.policy.protectionMode = SecurityPolicyprotectionModeSettings() _settings.security.policy.protectionMode.ipBlocking = SecurityPolicyIPBlockingSettings() -_settings.security.policy.protectionMode.ipBlocking = SecurityPolicyAPIBlockingSettings() +_settings.security.policy.protectionMode.apiBlocking = SecurityPolicyAPIBlockingSettings() _settings.security.force_complete_disable = _environ_as_bool( "NEW_RELIC_SECURITY_FORCE_COMPLETE_DISABLE", From a609df87cc118ff3b83b9fc9c14f4b3815bc2ec5 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 6 Oct 2022 13:11:11 +0530 Subject: [PATCH 16/79] This contains changes for policy propagation to k2 --- newrelic/config.py | 2 +- newrelic/core/config.py | 50 +++++++++++++++++------------------------ 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index d598774b63..1a14a63e79 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3081,7 +3081,7 @@ def _setup_agent_console(): def _generate_security_policy(): - return _settings.security.policy.to_deep_dict() + return dict(_settings.security.policy) def _generate_security_module_config(): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 81a9024d29..dc3e41a50d 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -94,16 +94,6 @@ def __contains__(self, item): return hasattr(self, item) -class DeepDictableSetting: - def to_deep_dict(self): - _dict = self.__dict__ - for attr in _dict.keys(): - value = _dict[attr] - if isinstance(value, DeepDictableSetting): - _dict[attr] = value.to_deep_dict() - return _dict - - def create_settings(nested): return type("Settings", (Settings,), {"nested": nested})() @@ -291,25 +281,25 @@ class SecurityModuleSettings(Settings): class SecurityDetectionSettings(Settings): pass -class SecurityPolicySettings(Settings, DeepDictableSetting): +class SecurityPolicySettings(Settings): pass -class SecurityPolicyVulnerabilityScanSettings(Settings, DeepDictableSetting): +class SecurityPolicyVulnerabilityScanSettings(Settings): pass -class SecurityPolicyIASTSettings(Settings, DeepDictableSetting): +class SecurityPolicyIASTSettings(Settings): pass -class SecurityPolicyIASTProbingSettings(Settings, DeepDictableSetting): +class SecurityPolicyIASTProbingSettings(Settings): pass -class SecurityPolicyprotectionModeSettings(Settings, DeepDictableSetting): +class SecurityPolicyprotectionModeSettings(Settings): pass -class SecurityPolicyIPBlockingSettings(Settings, DeepDictableSetting): +class SecurityPolicyIPBlockingSettings(Settings): pass -class SecurityPolicyAPIBlockingSettings(Settings, DeepDictableSetting): +class SecurityPolicyAPIBlockingSettings(Settings): pass @@ -922,55 +912,55 @@ def default_host(license_key): _settings.security.policy.vulnerabilityScan.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_ENABLED", - default=False + default=None ) _settings.security.policy.vulnerabilityScan.iastScan.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_ENABLED", - default=False + default=None ) _settings.security.policy.vulnerabilityScan.iastScan.probing.interval = _environ_as_int( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_INTERVAL", - default=1 + default=-1 ) _settings.security.policy.vulnerabilityScan.iastScan.probing.batchSize = _environ_as_int( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_BATCHSIZE", - default=5 + default=-1 ) _settings.security.policy.protectionMode.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_ENABLED", - default=False + default=None ) _settings.security.policy.protectionMode.ipBlocking.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ENABLED", - default=False + default=None ) _settings.security.policy.protectionMode.ipBlocking.attackerIpBlocking = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ATTACKERIPBLOCKING", - default=False + default=None ) _settings.security.policy.protectionMode.ipBlocking.ipDetectViaXFF = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_IPDETECTVIAXFF", - default=False + default=None ) _settings.security.policy.protectionMode.ipBlocking.timeout = _environ_as_int( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_TIMEOUT", - default=120 + default=-1 ) _settings.security.policy.protectionMode.apiBlocking.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_ENABLED", - default=False + default=None ) _settings.security.policy.protectionMode.apiBlocking.protectAllApis = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTALLAPIS", - default=False + default=None ) _settings.security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTKNOWNVULNERABLEAPIS", - default=False + default=None ) _settings.security.policy.protectionMode.apiBlocking.protectAttackedApis = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTATTACKEDAPIS", - default=False + default=None ) From 30a280add772c6b6d46647c72b7bc2b589c885e8 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 6 Oct 2022 13:11:49 +0530 Subject: [PATCH 17/79] Merge branch 'feature/k2i/policy_propagation' into k2_integration From a5dd4184c43beb22ad9ea2870a86ffce05264554 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 7 Oct 2022 17:52:38 +0530 Subject: [PATCH 18/79] updated config for logs upload and exception handling in security module setup --- newrelic/config.py | 40 +++++++++++++++++++++------------------- newrelic/core/config.py | 4 ---- newrelic/newrelic.ini | 3 +-- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 1a14a63e79..3aa551fe0c 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -337,7 +337,6 @@ def _process_configuration(section): _process_setting(section, "security.customer_id", "getint", None) _process_setting(section, "security.log_level", "get", None) _process_setting(section, "security.sec_home_path", "get", None) - _process_setting(section, "security.sec_log_file_name", "get", None) _process_setting(section, "security.detection.disable_rci", "getboolean", None) _process_setting(section, "security.detection.disable_rxss", "getboolean", None) _process_setting(section, "security.detection.disable_deserialization", "getboolean", None) @@ -3110,24 +3109,27 @@ def _setup_security_module(): """Initiates k2 security module and adds a callback to agent startup to propagate NR config """ - if _settings.security.force_complete_disable: - return - # run security module - from k2_python_agent import AgentConfig, ModuleLoadAgent - from functools import partial as Partial - - config =_generate_security_module_config() - policy = _generate_security_policy() - - security_module_agent = ModuleLoadAgent(config) - security_module_agent.initialise() - - security_module_agent.set_policy(policy) - if not _settings.security.enable: - security_module_agent.disable() - # create a callback to reinitialise the security module - callback = Partial(_update_security_module, security_module_agent) - newrelic.core.agent.Agent.run_on_startup(callback) + try: + if _settings.security.force_complete_disable: + return + # run security module + from k2_python_agent import AgentConfig, ModuleLoadAgent + from functools import partial as Partial + + config =_generate_security_module_config() + policy = _generate_security_policy() + + security_module_agent = ModuleLoadAgent(config) + security_module_agent.initialise() + + security_module_agent.set_policy(policy) + if not _settings.security.enable: + security_module_agent.disable() + # create a callback to reinitialise the security module + callback = Partial(_update_security_module, security_module_agent) + newrelic.core.agent.Agent.run_on_startup(callback) + except Exception as k2error: + _logger.error("K2 Startup failed with error %s", k2error) def initialize( diff --git a/newrelic/core/config.py b/newrelic/core/config.py index dc3e41a50d..1c7b6458d8 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -893,10 +893,6 @@ def default_host(license_key): "NEW_RELIC_SECURITY_SEC_HOME_PATH", default="/opt" ) -_settings.security.sec_log_file_name = os.environ.get( - "NEW_RELIC_SECURITY_SEC_LOG_FILE_NAME", - default="/temp/security_log" -) _settings.security.detection.disable_rci = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_DISABLE_RCI", default=False diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index d985f87a02..02b0de3aa0 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -73,8 +73,7 @@ security.customer_id = security.log_level = INFO # security file settings -# security.sec_home_path = /opt -# security.sec_log_file_name = /tmp/security.log +security.sec_home_path = /tmp # vulnerabilty detection disable # security.detection.disable_rci = false From 5f226e8e1b52d5b1e72471661a14a28c246810c8 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 9 Oct 2022 19:00:35 +0530 Subject: [PATCH 19/79] Merge branch 'feature/k2i/logs_config_propagation' into k2_integration From 6d515e845520abf95431f2419faaafdea4d3f18b Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 9 Oct 2022 19:03:03 +0530 Subject: [PATCH 20/79] Changes for setting the trasnsaction id catcher by handing over a lambda which would fetch transaction_id on run --- newrelic/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index 3aa551fe0c..cbac91c845 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3128,6 +3128,11 @@ def _setup_security_module(): # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) newrelic.core.agent.Agent.run_on_startup(callback) + + # set transaction id catcher + security_module_agent.set_transaction_id_catcher( + lambda *args: trace_cache.trace_cache().current_transaction()._transaction_id + ) except Exception as k2error: _logger.error("K2 Startup failed with error %s", k2error) From 34d610bff21a8faacc4a707af5e1917055162109 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 10 Oct 2022 14:49:24 +0530 Subject: [PATCH 21/79] changes to send trace metadata long with transaction id from metadata catcher --- newrelic/config.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index cbac91c845..0274197aba 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3079,6 +3079,15 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) +def _get_transaction_metadata_for_security_module(): + trace = trace_cache.trace_cache().current_trace() + transaction = trace and trace.transaction and trace.transaction + metadata = trace and trace._get_trace_linking_metadata() + transaction_id = transaction._transaction_id + metadata['transaction_id'] = transaction_id + return metadata + + def _generate_security_policy(): return dict(_settings.security.policy) @@ -3130,8 +3139,8 @@ def _setup_security_module(): newrelic.core.agent.Agent.run_on_startup(callback) # set transaction id catcher - security_module_agent.set_transaction_id_catcher( - lambda *args: trace_cache.trace_cache().current_transaction()._transaction_id + security_module_agent.set_metadata_catcher( + _get_transaction_metadata_for_security_module ) except Exception as k2error: _logger.error("K2 Startup failed with error %s", k2error) From e02eacfa685a0244e55381da3d67620959f717e0 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 10 Oct 2022 14:50:35 +0530 Subject: [PATCH 22/79] Merge 'feature/k2i/logs_config_propagation' into k2_integration From 385b96696914a176d0025743470ab2b2e7d2a544 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 12 Oct 2022 15:14:01 +0530 Subject: [PATCH 23/79] This contains multiple changes: 1. Changes to start security module even before the agent is activated, thus, init on active_agent callback have been removed and instead added along with module load phase. 2. Updated agent refresh with policy and linking metadata along with agent config for security module 3. Removed deprecated security module configs constomer_id and accessor_token --- newrelic/config.py | 52 ++++++++++++++++++++++++----------------- newrelic/core/config.py | 8 ------- newrelic/newrelic.ini | 4 ---- 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 0274197aba..c2f204bbe2 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -333,8 +333,6 @@ def _process_configuration(section): _process_setting(section, "security.mode", "get", None) _process_setting(section, "security.validator_service_endpoint_url", "get", None) _process_setting(section, "security.resource_service_endpoint_url", "get", None) - _process_setting(section, "security.accessor_token", "get", None) - _process_setting(section, "security.customer_id", "getint", None) _process_setting(section, "security.log_level", "get", None) _process_setting(section, "security.sec_home_path", "get", None) _process_setting(section, "security.detection.disable_rci", "getboolean", None) @@ -3079,16 +3077,19 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) -def _get_transaction_metadata_for_security_module(): +def _get_linking_metadata_for_security_module(): + import newrelic.agent + context = newrelic.agent.get_linking_metadata() + return context + + +def _get_trace_linking_metadata_for_security_module(): trace = trace_cache.trace_cache().current_trace() - transaction = trace and trace.transaction and trace.transaction - metadata = trace and trace._get_trace_linking_metadata() - transaction_id = transaction._transaction_id - metadata['transaction_id'] = transaction_id + metadata = trace and trace._get_trace_linking_metadata() or {} return metadata -def _generate_security_policy(): +def _generate_security_module_policy(): return dict(_settings.security.policy) @@ -3096,22 +3097,27 @@ def _generate_security_module_config(): from k2_python_agent import AgentConfig config = AgentConfig() config.set_base_config(_settings.security) - # propogate app name and id - agent_instance = newrelic.core.agent.agent_instance() - application = agent_instance.application(_settings.app_name) - if application: - configuration = application.configuration - config.application_id = configuration.entity_guid + config.set_acessor_token(_settings.license_key) config.application_name = _settings.app_name return config def _update_security_module(agent): + """refreshes the security module with latest config and + linking metadata + """ config = _generate_security_module_config() - agent.refresh_agent(config) - if _settings.security.enable: - agent.connect() + policy = _generate_security_module_policy() + metadata = _get_linking_metadata_for_security_module() + # propogate app name and id + agent_instance = newrelic.core.agent.agent_instance() + application = agent_instance.application(_settings.app_name) + if application: + configuration = application.configuration + metadata["agentRunId"] = configuration.agent_run_id + + agent.refresh_agent(config, policy, metadata) def _setup_security_module(): @@ -3126,21 +3132,23 @@ def _setup_security_module(): from functools import partial as Partial config =_generate_security_module_config() - policy = _generate_security_policy() + policy = _generate_security_module_policy() security_module_agent = ModuleLoadAgent(config) security_module_agent.initialise() - security_module_agent.set_policy(policy) if not _settings.security.enable: security_module_agent.disable() + else: + security_module_agent.connect() + # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) newrelic.core.agent.Agent.run_on_startup(callback) - # set transaction id catcher - security_module_agent.set_metadata_catcher( - _get_transaction_metadata_for_security_module + # set trace_linking_metadata catcher + security_module_agent.set_linking_metadata_catcher( + _get_trace_linking_metadata_for_security_module ) except Exception as k2error: _logger.error("K2 Startup failed with error %s", k2error) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 1c7b6458d8..1466b74da9 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -877,14 +877,6 @@ def default_host(license_key): "NEW_RELIC_SECURITY_RESOURCE_SERVICE_ENDPOINT_URL", default=None ) -_settings.security.accessor_token = os.environ.get( - "NEW_RELIC_SECURITY_ACCESSOR_TOKEN", - default=None -) -_settings.security.customer_id = os.environ.get( - "NEW_RELIC_SECURITY_CUSTOMER_ID", - default=None -) _settings.security.log_level = os.environ.get( "NEW_RELIC_SECURITY_LOG_LEVEL", default="INFO" diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 02b0de3aa0..55d5913b0c 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -65,10 +65,6 @@ security.mode = RASP security.validator_service_endpoint_url = security.resource_service_endpoint_url = -# web-protect agent accessor token -security.accessor_token = -security.customer_id = - # log level security.log_level = INFO From ded8f4822b065d3bc751a1613dce67ab69bfac9f Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 12 Oct 2022 16:23:06 +0530 Subject: [PATCH 24/79] minor change to reflect k2 module changes --- newrelic/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newrelic/config.py b/newrelic/config.py index c2f204bbe2..e803fa7a18 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3136,6 +3136,7 @@ def _setup_security_module(): security_module_agent = ModuleLoadAgent(config) security_module_agent.initialise() + security_module_agent.set_policy_from_flat_dict(policy) if not _settings.security.enable: security_module_agent.disable() From 52984303cd55f322689c21c7d1ffa443b9cf0780 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 12 Oct 2022 16:24:11 +0530 Subject: [PATCH 25/79] Merge branch 'feature/k2i/add_linking_metadata_deprecate_old_k2_auth_config' into k2_integration From a2f385e869c7896f60e7c520814f20b6a7e947ea Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 26 Oct 2022 14:20:09 +0530 Subject: [PATCH 26/79] Addition of enforce flag --- newrelic/config.py | 1 + newrelic/core/config.py | 4 ++++ newrelic/newrelic.ini | 1 + 3 files changed, 6 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index e803fa7a18..f4653441b3 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -338,6 +338,7 @@ def _process_configuration(section): _process_setting(section, "security.detection.disable_rci", "getboolean", None) _process_setting(section, "security.detection.disable_rxss", "getboolean", None) _process_setting(section, "security.detection.disable_deserialization", "getboolean", None) + _process_setting(section, "security.policy.enforce", "getboolean", None) _process_setting(section, "security.policy.vulnerabilityScan.enabled", "getboolean", None) _process_setting(section, "security.policy.vulnerabilityScan.iastScan.enabled", "getboolean", None) _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.interval", "getint", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 1466b74da9..dbd6856a38 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -898,6 +898,10 @@ def default_host(license_key): default=False ) +_settings.security.policy.enforce = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_ENFORCE", + default=False +) _settings.security.policy.vulnerabilityScan.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_ENABLED", default=None diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 55d5913b0c..937d958b70 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -77,6 +77,7 @@ security.sec_home_path = /tmp # security.detection.disable_deserialization = false # policy settings +# security.policy.enforce = false # security.policy.vulnerabilityScan.enabled = false # security.policy.vulnerabilityScan.iastScan.enabled = false # security.policy.vulnerabilityScan.iastScan.probing.interval = 1 From a8a7a889361b45288456bd95392ce4026e41ca80 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 21 Dec 2022 13:09:50 +0530 Subject: [PATCH 27/79] Addition of account id in linking metadata --- newrelic/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newrelic/config.py b/newrelic/config.py index d12962b900..5e960fd8ff 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3154,6 +3154,7 @@ def _update_security_module(agent): if application: configuration = application.configuration metadata["agentRunId"] = configuration.agent_run_id + metadata["accountId"] = configuration.account_id agent.refresh_agent(config, policy, metadata) From d925658c362dcda585817a1a440a6e78730ba7a7 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 23 Dec 2022 13:58:47 +0530 Subject: [PATCH 28/79] Temp changes --- newrelic/config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 5e960fd8ff..5e0ba502bf 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3132,8 +3132,8 @@ def _generate_security_module_policy(): def _generate_security_module_config(): - from k2_python_agent import AgentConfig - config = AgentConfig() + from newrelic_security.agent.models.agent_config import NRSecurityAgentConfig + config = NRSecurityAgentConfig() config.set_base_config(_settings.security) config.set_acessor_token(_settings.license_key) config.application_name = _settings.app_name @@ -3167,13 +3167,13 @@ def _setup_security_module(): if _settings.security.force_complete_disable: return # run security module - from k2_python_agent import AgentConfig, ModuleLoadAgent + from newrelic_security.agent.nr_security_agent import NRSecurityAgent from functools import partial as Partial config =_generate_security_module_config() policy = _generate_security_module_policy() - security_module_agent = ModuleLoadAgent(config) + security_module_agent = NRSecurityAgent(config) security_module_agent.initialise() security_module_agent.set_policy_from_flat_dict(policy) @@ -3191,6 +3191,8 @@ def _setup_security_module(): _get_trace_linking_metadata_for_security_module ) except Exception as k2error: + import traceback + traceback.print_tb(k2error.__traceback__) _logger.error("K2 Startup failed with error %s", k2error) From a741c2d8d7ea9e776ccc08151e9f68edd2bba5a4 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 23 Dec 2022 14:33:41 +0530 Subject: [PATCH 29/79] Let the connect be called with refresh (When linking metadata is available) --- newrelic/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 5e960fd8ff..310174aef2 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3179,8 +3179,6 @@ def _setup_security_module(): if not _settings.security.enable: security_module_agent.disable() - else: - security_module_agent.connect() # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) From 0ec4fe4907b6c97e4c9624b6296189a9e2f49eff Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 28 Dec 2022 10:40:44 +0530 Subject: [PATCH 30/79] agent would not connect with startup --- newrelic/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 5e0ba502bf..1d06a55048 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3155,7 +3155,7 @@ def _update_security_module(agent): configuration = application.configuration metadata["agentRunId"] = configuration.agent_run_id metadata["accountId"] = configuration.account_id - + print("REFRESH AGENT!!!!") agent.refresh_agent(config, policy, metadata) @@ -3179,8 +3179,6 @@ def _setup_security_module(): if not _settings.security.enable: security_module_agent.disable() - else: - security_module_agent.connect() # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) From f5a98bad58e8c981356e419410b9ccb4ae642613 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 25 Jan 2023 23:06:41 +0530 Subject: [PATCH 31/79] removal of rest server endpoint config, use of SingletonAgentConfig instead of NRSecurityAgentConfig for generating a config. --- newrelic/config.py | 5 ++--- newrelic/core/config.py | 4 ---- newrelic/newrelic.ini | 1 - 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 1d06a55048..36b6efe0d1 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -332,7 +332,6 @@ def _process_configuration(section): _process_setting(section, "security.enable", "getboolean", None) _process_setting(section, "security.mode", "get", None) _process_setting(section, "security.validator_service_endpoint_url", "get", None) - _process_setting(section, "security.resource_service_endpoint_url", "get", None) _process_setting(section, "security.log_level", "get", None) _process_setting(section, "security.sec_home_path", "get", None) _process_setting(section, "security.detection.disable_rci", "getboolean", None) @@ -3132,8 +3131,8 @@ def _generate_security_module_policy(): def _generate_security_module_config(): - from newrelic_security.agent.models.agent_config import NRSecurityAgentConfig - config = NRSecurityAgentConfig() + from newrelic_security.agent.models.agent_config import SingletonAgentConfig + config = SingletonAgentConfig.instance() config.set_base_config(_settings.security) config.set_acessor_token(_settings.license_key) config.application_name = _settings.app_name diff --git a/newrelic/core/config.py b/newrelic/core/config.py index dbd6856a38..d6363915cf 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -873,10 +873,6 @@ def default_host(license_key): "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_ENDPOINT_URL", default=None ) -_settings.security.resource_service_endpoint_url = os.environ.get( - "NEW_RELIC_SECURITY_RESOURCE_SERVICE_ENDPOINT_URL", - default=None -) _settings.security.log_level = os.environ.get( "NEW_RELIC_SECURITY_LOG_LEVEL", default="INFO" diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 937d958b70..f1a589b2e3 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -63,7 +63,6 @@ security.mode = RASP # web-protect agent endpoint connection URLs security.validator_service_endpoint_url = -security.resource_service_endpoint_url = # log level security.log_level = INFO From 524b3154b0a399846cf51102c325091581d7665d Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 26 Jan 2023 00:31:47 +0530 Subject: [PATCH 32/79] removal of all security agent initialization logic --- newrelic/config.py | 73 ++++------------------------------------------ 1 file changed, 5 insertions(+), 68 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 36b6efe0d1..6253fbf417 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3114,50 +3114,6 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) -def _get_linking_metadata_for_security_module(): - import newrelic.agent - context = newrelic.agent.get_linking_metadata() - return context - - -def _get_trace_linking_metadata_for_security_module(): - trace = trace_cache.trace_cache().current_trace() - metadata = trace and trace._get_trace_linking_metadata() or {} - return metadata - - -def _generate_security_module_policy(): - return dict(_settings.security.policy) - - -def _generate_security_module_config(): - from newrelic_security.agent.models.agent_config import SingletonAgentConfig - config = SingletonAgentConfig.instance() - config.set_base_config(_settings.security) - config.set_acessor_token(_settings.license_key) - config.application_name = _settings.app_name - - return config - - -def _update_security_module(agent): - """refreshes the security module with latest config and - linking metadata - """ - config = _generate_security_module_config() - policy = _generate_security_module_policy() - metadata = _get_linking_metadata_for_security_module() - # propogate app name and id - agent_instance = newrelic.core.agent.agent_instance() - application = agent_instance.application(_settings.app_name) - if application: - configuration = application.configuration - metadata["agentRunId"] = configuration.agent_run_id - metadata["accountId"] = configuration.account_id - print("REFRESH AGENT!!!!") - agent.refresh_agent(config, policy, metadata) - - def _setup_security_module(): """Initiates k2 security module and adds a callback to agent startup to propagate NR config @@ -3165,31 +3121,12 @@ def _setup_security_module(): try: if _settings.security.force_complete_disable: return - # run security module - from newrelic_security.agent.nr_security_agent import NRSecurityAgent - from functools import partial as Partial - - config =_generate_security_module_config() - policy = _generate_security_module_policy() - - security_module_agent = NRSecurityAgent(config) - security_module_agent.initialise() - security_module_agent.set_policy_from_flat_dict(policy) - - if not _settings.security.enable: - security_module_agent.disable() - - # create a callback to reinitialise the security module - callback = Partial(_update_security_module, security_module_agent) - newrelic.core.agent.Agent.run_on_startup(callback) - - # set trace_linking_metadata catcher - security_module_agent.set_linking_metadata_catcher( - _get_trace_linking_metadata_for_security_module - ) + from newrelic_security.api.agent import Agent as SecurityAgent + # initialize security agent + security_agent = SecurityAgent() + # create a callback to reinitialise the security module + newrelic.core.agent.Agent.run_on_startup(security_agent.refresh_agent) except Exception as k2error: - import traceback - traceback.print_tb(k2error.__traceback__) _logger.error("K2 Startup failed with error %s", k2error) From 46637fe6f6d864336714108154793bba42a5c2bb Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 14 Mar 2023 22:01:00 +0530 Subject: [PATCH 33/79] updates to newrelic config for security module --- newrelic/config.py | 28 ++------- newrelic/core/config.py | 127 ++++++++-------------------------------- newrelic/newrelic.ini | 56 +++++++----------- 3 files changed, 52 insertions(+), 159 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 6253fbf417..03df8cde5c 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,29 +328,13 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) - _process_setting(section, "security.force_complete_disable", "getboolean", None) - _process_setting(section, "security.enable", "getboolean", None) + _process_setting(section, "security.agent.enabled", "getboolean", None) + _process_setting(section, "security.enabled", "getboolean", None) _process_setting(section, "security.mode", "get", None) - _process_setting(section, "security.validator_service_endpoint_url", "get", None) - _process_setting(section, "security.log_level", "get", None) - _process_setting(section, "security.sec_home_path", "get", None) - _process_setting(section, "security.detection.disable_rci", "getboolean", None) - _process_setting(section, "security.detection.disable_rxss", "getboolean", None) - _process_setting(section, "security.detection.disable_deserialization", "getboolean", None) - _process_setting(section, "security.policy.enforce", "getboolean", None) - _process_setting(section, "security.policy.vulnerabilityScan.enabled", "getboolean", None) - _process_setting(section, "security.policy.vulnerabilityScan.iastScan.enabled", "getboolean", None) - _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.interval", "getint", None) - _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.batchSize", "getint", None) - _process_setting(section, "security.policy.protectionMode.enabled", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.ipBlocking.enabled", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.ipBlocking.attackerIpBlocking", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.ipBlocking.ipDetectViaXFF", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.ipBlocking.timeout", "getint", None) - _process_setting(section, "security.policy.protectionMode.apiBlocking.enabled", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.apiBlocking.protectAllApis", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.apiBlocking.protectAttackedApis", "getboolean", None) + _process_setting(section, "security.validator_service_url", "get", None) + _process_setting(section, "security.detection.rci.enabled", "getboolean", None) + _process_setting(section, "security.detection.rxss.enabled", "getboolean", None) + _process_setting(section, "security.detection.deserialization.enabled", "getboolean", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index d6363915cf..4d7c14d637 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -281,26 +281,18 @@ class SecurityModuleSettings(Settings): class SecurityDetectionSettings(Settings): pass -class SecurityPolicySettings(Settings): +class SecurityAgentSettings(Settings): pass -class SecurityPolicyVulnerabilityScanSettings(Settings): +class SecurityDetectionRCISettings(Settings): pass -class SecurityPolicyIASTSettings(Settings): +class SecurityDetectionRXSSSettings(Settings): pass -class SecurityPolicyIASTProbingSettings(Settings): +class SecurityDetectionDeserializationSettings(Settings): pass -class SecurityPolicyprotectionModeSettings(Settings): - pass - -class SecurityPolicyIPBlockingSettings(Settings): - pass - -class SecurityPolicyAPIBlockingSettings(Settings): - pass class InfiniteTracingSettings(Settings): @@ -848,107 +840,40 @@ def default_host(license_key): ) _settings.security = SecurityModuleSettings() +_settings.security.agent = SecurityAgentSettings() _settings.security.detection = SecurityDetectionSettings() -_settings.security.policy = SecurityPolicySettings() -_settings.security.policy.vulnerabilityScan = SecurityPolicyVulnerabilityScanSettings() -_settings.security.policy.vulnerabilityScan.iastScan = SecurityPolicyIASTSettings() -_settings.security.policy.vulnerabilityScan.iastScan.probing = SecurityPolicyIASTProbingSettings() -_settings.security.policy.protectionMode = SecurityPolicyprotectionModeSettings() -_settings.security.policy.protectionMode.ipBlocking = SecurityPolicyIPBlockingSettings() -_settings.security.policy.protectionMode.apiBlocking = SecurityPolicyAPIBlockingSettings() - -_settings.security.force_complete_disable = _environ_as_bool( - "NEW_RELIC_SECURITY_FORCE_COMPLETE_DISABLE", +_settings.security.detection.rci = SecurityDetectionRCISettings() +_settings.security.detection.rxss = SecurityDetectionRXSSSettings() +_settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() + + +_settings.security.agent.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_AGENT_ENABLED", default=False ) -_settings.security.enable = _environ_as_bool( - "NEW_RELIC_SECURITY_ENABLE", +_settings.security.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_ENABLED", default=False ) _settings.security.mode = os.environ.get( "NEW_RELIC_SECURITY_MODE", - default="RASP" -) -_settings.security.validator_service_endpoint_url = os.environ.get( - "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_ENDPOINT_URL", - default=None -) -_settings.security.log_level = os.environ.get( - "NEW_RELIC_SECURITY_LOG_LEVEL", - default="INFO" -) -_settings.security.sec_home_path = os.environ.get( - "NEW_RELIC_SECURITY_SEC_HOME_PATH", - default="/opt" -) -_settings.security.detection.disable_rci = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_DISABLE_RCI", - default=False -) -_settings.security.detection.disable_rxss = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_DISABLE_RXSS", - default=False -) -_settings.security.detection.disable_deserialization = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_DISABLE_DESERIALIZATION", - default=False -) - -_settings.security.policy.enforce = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_ENFORCE", - default=False -) -_settings.security.policy.vulnerabilityScan.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_ENABLED", - default=None -) -_settings.security.policy.vulnerabilityScan.iastScan.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_ENABLED", - default=None + default="IAST" ) -_settings.security.policy.vulnerabilityScan.iastScan.probing.interval = _environ_as_int( - "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_INTERVAL", - default=-1 -) -_settings.security.policy.vulnerabilityScan.iastScan.probing.batchSize = _environ_as_int( - "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_BATCHSIZE", - default=-1 -) -_settings.security.policy.protectionMode.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_ENABLED", +_settings.security.validator_service_url = os.environ.get( + "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", default=None ) -_settings.security.policy.protectionMode.ipBlocking.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ENABLED", - default=None +_settings.security.detection.rci.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", + default=True ) -_settings.security.policy.protectionMode.ipBlocking.attackerIpBlocking = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ATTACKERIPBLOCKING", - default=None +_settings.security.detection.rxss.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", + default=True ) -_settings.security.policy.protectionMode.ipBlocking.ipDetectViaXFF = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_IPDETECTVIAXFF", - default=None -) -_settings.security.policy.protectionMode.ipBlocking.timeout = _environ_as_int( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_TIMEOUT", - default=-1 -) -_settings.security.policy.protectionMode.apiBlocking.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_ENABLED", - default=None -) -_settings.security.policy.protectionMode.apiBlocking.protectAllApis = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTALLAPIS", - default=None -) -_settings.security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTKNOWNVULNERABLEAPIS", - default=None -) -_settings.security.policy.protectionMode.apiBlocking.protectAttackedApis = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTATTACKEDAPIS", - default=None +_settings.security.detection.deserialization.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", + default=True ) diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index f1a589b2e3..4e43171a54 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -49,47 +49,31 @@ app_name = Python Application # NEW_RELIC_MONITOR_MODE environment variable. monitor_mode = true -# When enabled, the agent's security module is not loaded -# this would mean changes on soft reloads won't work -security.force_complete_disable = false -# When enabled, the agent collects security data about your -# application and reports this data to the New Relic. -security.enable = true +# Indicates if attack detection security module is to be enabled +security.enabled = false + +# To completely disable security set flag to false If the flag is +# set to false, the security module is not loaded. This property +# is read only once at application start. +security.agent.enabled = false + + # security module provides two modes IAST or RASP # RASP stands for Runtime Application Self Protection # while IAST for Interactive Application Security Testing -# Default mode is RASP -security.mode = RASP +# Default mode is IAST +security.mode = IAST + # web-protect agent endpoint connection URLs -security.validator_service_endpoint_url = - -# log level -security.log_level = INFO - -# security file settings -security.sec_home_path = /tmp - -# vulnerabilty detection disable -# security.detection.disable_rci = false -# security.detection.disable_rxss = false -# security.detection.disable_deserialization = false - -# policy settings -# security.policy.enforce = false -# security.policy.vulnerabilityScan.enabled = false -# security.policy.vulnerabilityScan.iastScan.enabled = false -# security.policy.vulnerabilityScan.iastScan.probing.interval = 1 -# security.policy.vulnerabilityScan.iastScan.probing.batchSize = 5 -# security.policy.protectionMode.enabled = false -# security.policy.protectionMode.ipBlocking.enabled = false -# security.policy.protectionMode.ipBlocking.attackerIpBlocking = false -# security.policy.protectionMode.ipBlocking.ipDetectViaXFF = false -# security.policy.protectionMode.ipBlocking.timeout = 120 -# security.policy.protectionMode.apiBlocking.enabled = false -# security.policy.protectionMode.apiBlocking.protectAllApis = false -# security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = false -# security.policy.protectionMode.apiBlocking.protectAttackedApis = false +security.validator_service_url = wss://csec.nr-data.net + + +# vulnerabilty detection flags +security.detection.rci.enabled = true +security.detection.rxss.enabled = true +security.detection.deserialization.enabled = true + # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the From 5be183660254dfe9d094eb6ad6041f042e7da9e4 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 23 Mar 2023 10:44:15 +0530 Subject: [PATCH 34/79] minor fix --- newrelic/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index 03df8cde5c..b2988dc1a2 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3103,7 +3103,7 @@ def _setup_security_module(): callback to agent startup to propagate NR config """ try: - if _settings.security.force_complete_disable: + if not _settings.security.agent.enabled: return from newrelic_security.api.agent import Agent as SecurityAgent # initialize security agent From ac7e712621782ec45f01c327d9b282b2e70bfd7c Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 17 Apr 2023 14:32:34 +0530 Subject: [PATCH 35/79] point to nr_adaptation with updated newrelic_security package --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f6b8fafb23..c6cb15f394 100644 --- a/setup.py +++ b/setup.py @@ -153,7 +153,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["k2_python_agent @ git+https://github.com/k2io/k2-python-agent@newrelic_integration#egg=k2_python_agent"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent@nr_adaptation#egg=newrelic_security"] ) if with_setuptools: From 28d789a7ca2ad0cf4b63e4550712ab08d30e7abb Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 28 Apr 2023 17:30:26 +0530 Subject: [PATCH 36/79] Refactoring and relocation of security settings in core config --- newrelic/core/config.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 23fc44d5f1..3cf9875102 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -278,7 +278,7 @@ class ApplicationLoggingMetricsSettings(Settings): class ApplicationLoggingLocalDecoratingSettings(Settings): pass -class SecurityModuleSettings(Settings): +class SecuritySettings(Settings): pass class SecurityDetectionSettings(Settings): @@ -414,6 +414,12 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.message_tracer = MessageTracerSettings() _settings.process_host = ProcessHostSettings() _settings.rum = RumSettings() +_settings.security = SecuritySettings() +_settings.security.agent = SecurityAgentSettings() +_settings.security.detection = SecurityDetectionSettings() +_settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() +_settings.security.detection.rci = SecurityDetectionRCISettings() +_settings.security.detection.rxss = SecurityDetectionRXSSSettings() _settings.serverless_mode = ServerlessModeSettings() _settings.slow_sql = SlowSqlSettings() _settings.span_events = SpanEventSettings() @@ -858,14 +864,6 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) -_settings.security = SecurityModuleSettings() -_settings.security.agent = SecurityAgentSettings() -_settings.security.detection = SecurityDetectionSettings() -_settings.security.detection.rci = SecurityDetectionRCISettings() -_settings.security.detection.rxss = SecurityDetectionRXSSSettings() -_settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() - - _settings.security.agent.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_AGENT_ENABLED", default=False From 1f780c2094a0f8e19cf1cee3749ca9bfd1a2c806 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 4 May 2023 10:21:35 +0530 Subject: [PATCH 37/79] initialising security agent before configuring nr apm hooks --- newrelic/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index b1ec8f98d8..c78f4e481c 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3223,6 +3223,8 @@ def initialize( _load_configuration(config_file, environment, ignore_errors, log_file, log_level) + _setup_security_module() + if _settings.monitor_mode or _settings.developer_mode: _settings.enabled = True _setup_instrumentation() @@ -3232,8 +3234,6 @@ def initialize( else: _settings.enabled = False - _setup_security_module() - def filter_app_factory(app, global_conf, config_file, environment=None): initialize(config_file, environment) From 5fb7a423d69bf67bb751951760eb7351f8532b94 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 18 May 2023 00:23:25 +0530 Subject: [PATCH 38/79] fixes in config default values for security config for python 2.7 support --- newrelic/core/config.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 3cf9875102..cb8dccb7bd 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -866,31 +866,31 @@ def default_host(license_key): _settings.security.agent.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_AGENT_ENABLED", - default=False + False ) _settings.security.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_ENABLED", - default=False + False ) _settings.security.mode = os.environ.get( "NEW_RELIC_SECURITY_MODE", - default="IAST" + "IAST" ) _settings.security.validator_service_url = os.environ.get( "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", - default=None + None ) _settings.security.detection.rci.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", - default=True + True ) _settings.security.detection.rxss.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", - default=True + True ) _settings.security.detection.deserialization.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", - default=True + True ) From 11c0049413c81ccc53dee285bd03691e2ef92879 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 26 May 2023 10:24:16 +0530 Subject: [PATCH 39/79] Will use dev branch for security agent --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8eafb356ad..ecc43a64a0 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent@nr_adaptation#egg=newrelic_security"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent@dev#egg=newrelic_security"] ) if with_setuptools: From 093fd3a576aa65ee0ce1b27eafb4a858ce2dc3f8 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 1 Jun 2023 11:45:52 +0530 Subject: [PATCH 40/79] Updated remote for pulling newrelic_security module --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ecc43a64a0..c960cd357a 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent@dev#egg=newrelic_security"] + install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"] ) if with_setuptools: From 88cb32e0829744548fc756bb1125cbd55922d3f4 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 11:05:37 -0700 Subject: [PATCH 41/79] Update install requires line. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c960cd357a..44b4957d32 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"] + install_requires=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"] ) if with_setuptools: From 7a43a7fae139795e40cd802f6c319a85383da67b Mon Sep 17 00:00:00 2001 From: umaannamalai Date: Thu, 1 Jun 2023 18:08:32 +0000 Subject: [PATCH 42/79] [Mega-Linter] Apply linters fixes --- newrelic/config.py | 3 ++- newrelic/core/config.py | 40 +++++++++++++--------------------------- setup.py | 2 +- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 8009938c0e..d856118995 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3184,9 +3184,10 @@ def _setup_security_module(): if not _settings.security.agent.enabled: return from newrelic_security.api.agent import Agent as SecurityAgent + # initialize security agent security_agent = SecurityAgent() - # create a callback to reinitialise the security module + # create a callback to reinitialise the security module newrelic.core.agent.Agent.run_on_startup(security_agent.refresh_agent) except Exception as k2error: _logger.error("K2 Startup failed with error %s", k2error) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index cb8dccb7bd..03758fc74a 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -278,26 +278,31 @@ class ApplicationLoggingMetricsSettings(Settings): class ApplicationLoggingLocalDecoratingSettings(Settings): pass + class SecuritySettings(Settings): pass + class SecurityDetectionSettings(Settings): pass + class SecurityAgentSettings(Settings): pass + class SecurityDetectionRCISettings(Settings): pass + class SecurityDetectionRXSSSettings(Settings): pass + class SecurityDetectionDeserializationSettings(Settings): pass - class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -864,33 +869,14 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) -_settings.security.agent.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_AGENT_ENABLED", - False -) -_settings.security.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_ENABLED", - False -) -_settings.security.mode = os.environ.get( - "NEW_RELIC_SECURITY_MODE", - "IAST" -) -_settings.security.validator_service_url = os.environ.get( - "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", - None -) -_settings.security.detection.rci.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", - True -) -_settings.security.detection.rxss.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", - True -) +_settings.security.agent.enabled = _environ_as_bool("NEW_RELIC_SECURITY_AGENT_ENABLED", False) +_settings.security.enabled = _environ_as_bool("NEW_RELIC_SECURITY_ENABLED", False) +_settings.security.mode = os.environ.get("NEW_RELIC_SECURITY_MODE", "IAST") +_settings.security.validator_service_url = os.environ.get("NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", None) +_settings.security.detection.rci.enabled = _environ_as_bool("NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", True) +_settings.security.detection.rxss.enabled = _environ_as_bool("NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", True) _settings.security.detection.deserialization.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", - True + "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", True ) diff --git a/setup.py b/setup.py index 44b4957d32..acb5a4ad9a 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"] + install_requires=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], ) if with_setuptools: From 33b51fcb06be771dab9ab0e242ca9b2fc2b8daa4 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 11:15:52 -0700 Subject: [PATCH 43/79] Testing install requires. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 44b4957d32..587afeaf95 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"] + install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop"] ) if with_setuptools: From 3d44b8c2388d79b15778a3051180664d219eff2f Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 12:11:05 -0700 Subject: [PATCH 44/79] Restore install requires. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index acb5a4ad9a..5811e85933 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], + install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], ) if with_setuptools: From fe17e3632239387f7f53bd6d5873b0728c4a37b8 Mon Sep 17 00:00:00 2001 From: umaannamalai Date: Thu, 1 Jun 2023 19:12:37 +0000 Subject: [PATCH 45/79] [Mega-Linter] Apply linters fixes --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5811e85933..76340f55e9 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,9 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], + install_requires=[ + "newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security" + ], ) if with_setuptools: From b0e0a5053b3aa3e9e507e582dce8055c8d53bd87 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 13:02:05 -0700 Subject: [PATCH 46/79] switch ordering --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 76340f55e9..59d985b549 100644 --- a/setup.py +++ b/setup.py @@ -151,10 +151,10 @@ def build_extension(self, ext): package_data={ "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, - extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, install_requires=[ "newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security" ], + extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, ) if with_setuptools: From dada30bdc7135197223cd60a049ddf86518f0e75 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 13:22:17 -0700 Subject: [PATCH 47/79] Add dependency links. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 59d985b549..49f83401ff 100644 --- a/setup.py +++ b/setup.py @@ -152,8 +152,9 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, install_requires=[ - "newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security" + "newrelic_security" ], + dependency_links=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, ) From 78d90b65da16156f2bf46ae3aa594a28a522ed48 Mon Sep 17 00:00:00 2001 From: umaannamalai Date: Thu, 1 Jun 2023 20:23:41 +0000 Subject: [PATCH 48/79] [Mega-Linter] Apply linters fixes --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 49f83401ff..321ffbeb90 100644 --- a/setup.py +++ b/setup.py @@ -151,9 +151,7 @@ def build_extension(self, ext): package_data={ "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, - install_requires=[ - "newrelic_security" - ], + install_requires=["newrelic_security"], dependency_links=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, ) From 1b0c1b5cfc2bb94f3dbe02680f6f789d4a98065f Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 15:32:25 -0700 Subject: [PATCH 49/79] Add security settings to web framework conftests. --- setup.py | 3 +-- tests/framework_bottle/conftest.py | 4 ++++ tests/framework_django/conftest.py | 4 ++++ tests/framework_flask/conftest.py | 5 ++++- tox.ini | 2 ++ 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 321ffbeb90..c5a564b048 100644 --- a/setup.py +++ b/setup.py @@ -151,8 +151,7 @@ def build_extension(self, ext): package_data={ "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, - install_requires=["newrelic_security"], - dependency_links=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], + #install_requires=["newrelic-security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic-security"], extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, ) diff --git a/tests/framework_bottle/conftest.py b/tests/framework_bottle/conftest.py index 095a3331f3..f03f6461b5 100644 --- a/tests/framework_bottle/conftest.py +++ b/tests/framework_bottle/conftest.py @@ -23,6 +23,10 @@ 'transaction_tracer.stack_trace_threshold': 0.0, 'debug.log_data_collector_payloads': True, 'debug.record_transaction_failure': True, + "security.agent.enabled": True, + "security.enabled": True, + "security.mode": "IAST", + "security.validator_service_url": "wss://csec-staging.nr-data.net" } collector_agent_registration = collector_agent_registration_fixture( diff --git a/tests/framework_django/conftest.py b/tests/framework_django/conftest.py index 8a43ef5c90..6556447321 100644 --- a/tests/framework_django/conftest.py +++ b/tests/framework_django/conftest.py @@ -25,6 +25,10 @@ 'debug.record_transaction_failure': True, 'debug.log_autorum_middleware': True, 'feature_flag': set(['django.instrumentation.inclusion-tags.r1']), + "security.agent.enabled": True, + "security.enabled": True, + "security.mode": "IAST", + "security.validator_service_url": "wss://csec-staging.nr-data.net" } collector_agent_registration = collector_agent_registration_fixture( diff --git a/tests/framework_flask/conftest.py b/tests/framework_flask/conftest.py index f90ed9b512..8029ad2868 100644 --- a/tests/framework_flask/conftest.py +++ b/tests/framework_flask/conftest.py @@ -19,7 +19,6 @@ from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - _default_settings = { 'transaction_tracer.explain_threshold': 0.0, 'transaction_tracer.transaction_threshold': 0.0, @@ -27,6 +26,10 @@ 'debug.log_data_collector_payloads': True, 'debug.record_transaction_failure': True, 'debug.log_autorum_middleware': True, + "security.agent.enabled": True, + "security.enabled": True, + "security.mode": "IAST", + "security.validator_service_url": "wss://csec-staging.nr-data.net" } collector_agent_registration = collector_agent_registration_fixture( diff --git a/tox.ini b/tox.ini index 147851088c..2059abbe4a 100644 --- a/tox.ini +++ b/tox.ini @@ -173,8 +173,10 @@ deps = {py27,pypy}: pytest==4.6.11 iniconfig coverage + git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic-security WebTest==2.0.35 + # Test Suite Dependencies adapter_cheroot: cheroot adapter_daphne-daphnelatest: daphne From 14d70192a274f1094c639a75ef3e9c7b0ffd5870 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 17:17:29 -0700 Subject: [PATCH 50/79] github ssh url. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2059abbe4a..a974c9d627 100644 --- a/tox.ini +++ b/tox.ini @@ -173,7 +173,7 @@ deps = {py27,pypy}: pytest==4.6.11 iniconfig coverage - git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic-security + git+ssh://git@github.com:newrelic/csec-python-agent.git#egg=newrelic-security WebTest==2.0.35 From 874fd035b2556779c37cf44d818201e6a3aad38f Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 17:26:14 -0700 Subject: [PATCH 51/79] remove ssh prefix. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a974c9d627..3d4a9298b6 100644 --- a/tox.ini +++ b/tox.ini @@ -173,7 +173,7 @@ deps = {py27,pypy}: pytest==4.6.11 iniconfig coverage - git+ssh://git@github.com:newrelic/csec-python-agent.git#egg=newrelic-security + git+git@github.com:newrelic/csec-python-agent.git#egg=newrelic-security WebTest==2.0.35 From 7041783cf73993b0dc725b6fc34af5dd7627ce81 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 12 Jun 2023 17:51:25 +0530 Subject: [PATCH 52/79] Fixed k2 reference to Security Agent --- newrelic/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 8009938c0e..4fd43fcb65 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3177,7 +3177,7 @@ def _setup_agent_console(): def _setup_security_module(): - """Initiates k2 security module and adds a + """Initiates security module and adds a callback to agent startup to propagate NR config """ try: @@ -3188,8 +3188,8 @@ def _setup_security_module(): security_agent = SecurityAgent() # create a callback to reinitialise the security module newrelic.core.agent.Agent.run_on_startup(security_agent.refresh_agent) - except Exception as k2error: - _logger.error("K2 Startup failed with error %s", k2error) + except Exception as csec_error: + _logger.error("Security Agent Startup failed with error %s", csec_error) def initialize( From 34fc7f46dc2cdd1ae5666af728f501c21073e746 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 13 Dec 2023 12:29:30 +0530 Subject: [PATCH 53/79] branch update for develop branch to use k2-python-agent's dev branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b351ae06d4..799090f019 100644 --- a/setup.py +++ b/setup.py @@ -155,7 +155,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@dev"] ) if with_setuptools: From 941a38b01bbe7482466e0e54bb17e821ff3ce998 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 13 Dec 2023 19:23:35 +0530 Subject: [PATCH 54/79] Fix for high security flag handling --- newrelic/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index 7779ebc9d1..c4e0d15a77 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -4071,7 +4071,7 @@ def _setup_security_module(): callback to agent startup to propagate NR config """ try: - if not _settings.security.agent.enabled: + if not _settings.security.agent.enabled or _settings.high_security: return from newrelic_security.api.agent import Agent as SecurityAgent From 675e0098bd4aa954ff46573c6ce223269c09dd20 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 2 Jan 2024 15:42:20 +0530 Subject: [PATCH 55/79] added log in case security is disabled due to config --- newrelic/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newrelic/config.py b/newrelic/config.py index c4e0d15a77..8c31a904b7 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -4072,6 +4072,7 @@ def _setup_security_module(): """ try: if not _settings.security.agent.enabled or _settings.high_security: + _logger.warning("New Relic Security is disabled by one of the user provided config `security.agent.enabled` or `high_security`.") return from newrelic_security.api.agent import Agent as SecurityAgent From 3909f526dc3747dcdd429767f195fc76ac6fa492 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 1 Feb 2024 17:56:39 +0530 Subject: [PATCH 56/79] branch update for newrelic_security module --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 07ba81005a..67f34bce69 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@dev"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@develop"] ) if with_setuptools: From b969fc946dd4dc16f4b6e2d89d0de22c61e0b086 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sat, 24 Feb 2024 00:29:10 +0530 Subject: [PATCH 57/79] branch update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67f34bce69..4423c68d36 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@develop"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@task/python_27_support"] ) if with_setuptools: From 6afb0ce8b02747fa44a4d201fc8823d1ce9bc2a1 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 29 Feb 2024 01:49:05 +0530 Subject: [PATCH 58/79] Updates as per new api changes --- newrelic/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 7779ebc9d1..fbcdea02c4 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -4073,10 +4073,10 @@ def _setup_security_module(): try: if not _settings.security.agent.enabled: return - from newrelic_security.api.agent import Agent as SecurityAgent + from newrelic_security.api.agent import get_agent # initialize security agent - security_agent = SecurityAgent() + security_agent = get_agent() # create a callback to reinitialise the security module newrelic.core.agent.Agent.run_on_startup(security_agent.refresh_agent) except Exception as csec_error: From d1ef2f8216c634f62f2417c603cc1e1206e54119 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 29 Feb 2024 10:48:31 +0530 Subject: [PATCH 59/79] Rolledback SA branch to develop in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4423c68d36..67f34bce69 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@task/python_27_support"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@develop"] ) if with_setuptools: From 9dd7cf8ef65c2520504ebea1b50978cbccb10101 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 13 Mar 2024 12:18:26 +0530 Subject: [PATCH 60/79] introduced a new config: security.request.body_limit --- newrelic/config.py | 1 + newrelic/core/config.py | 4 ++++ newrelic/newrelic.ini | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index 8c31a904b7..a7417407fa 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -346,6 +346,7 @@ def _process_configuration(section): _process_setting(section, "security.detection.rci.enabled", "getboolean", None) _process_setting(section, "security.detection.rxss.enabled", "getboolean", None) _process_setting(section, "security.detection.deserialization.enabled", "getboolean", None) + _process_setting(section, "security.request.body_limit", "get", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 5a83061603..3f4e45fb28 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -332,6 +332,8 @@ class SecurityDetectionRXSSSettings(Settings): class SecurityDetectionDeserializationSettings(Settings): pass +class SecurityRequestSettings(Settings): + pass class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -459,6 +461,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() _settings.security.detection.rci = SecurityDetectionRCISettings() _settings.security.detection.rxss = SecurityDetectionRXSSSettings() +_settings.security.request = SecurityRequestSettings() _settings.serverless_mode = ServerlessModeSettings() _settings.slow_sql = SlowSqlSettings() _settings.span_events = SpanEventSettings() @@ -943,6 +946,7 @@ def default_otlp_host(host): _settings.security.detection.deserialization.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", True ) +_settings.security.request.body_limit = os.environ.get("NEW_RELIC_SECURITY_REQUEST_BODY_LIMIT", None) def global_settings(): diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 4e43171a54..1e85932540 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -75,6 +75,10 @@ security.detection.rxss.enabled = true security.detection.deserialization.enabled = true +# security request body read limiting in kb +security.request.body_limit = 300 + + # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the # containing directory and the file itself are correct, and From ec18d6f286a2b29c6e40a934eaf5f1b4380543fc Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 23 Apr 2024 00:05:59 +0530 Subject: [PATCH 61/79] Updated csec agent branch for testing --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67f34bce69..a0d7027eb6 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@develop"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@task/NR-181060/request_body_truncate"] ) if with_setuptools: From d21f3668f66d1840f749cf0fae665f806cb6c113 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 23 Apr 2024 01:35:41 +0530 Subject: [PATCH 62/79] branch change rollback --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a0d7027eb6..67f34bce69 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@task/NR-181060/request_body_truncate"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@develop"] ) if with_setuptools: From f94180052961023adb2de2310c3ecf457ac152cf Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 23 Apr 2024 15:06:20 +0530 Subject: [PATCH 63/79] Temporary changes to update csec branch to task/python_27_support for testing --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67f34bce69..4423c68d36 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@develop"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@task/python_27_support"] ) if with_setuptools: From 7fb52802611a52ef2de3806891d0b82bf63683bb Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 27 Jun 2024 06:33:23 +0530 Subject: [PATCH 64/79] Updated security agent branch after merge --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4423c68d36..67f34bce69 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@task/python_27_support"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@develop"] ) if with_setuptools: From 0757037e658054ad3a4270f06af8b4a25a1f9f79 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 16 Aug 2024 02:03:13 +0530 Subject: [PATCH 65/79] Updates to read security skip_iast_scan configurations --- newrelic/config.py | 14 ++++++++++ newrelic/core/config.py | 57 +++++++++++++++++++++++++++++++++++++++-- newrelic/newrelic.ini | 42 ++++++++++++++++++++++++++++++ setup.py | 2 +- 4 files changed, 112 insertions(+), 3 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 1e32017986..7a6852ec86 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -346,6 +346,20 @@ def _process_configuration(section): _process_setting(section, "security.detection.rxss.enabled", "getboolean", None) _process_setting(section, "security.detection.deserialization.enabled", "getboolean", None) _process_setting(section, "security.request.body_limit", "get", None) + _process_setting(section, "security.skip_iast_scan.api", "get", _map_inc_excl_attributes) + _process_setting(section, "security.skip_iast_scan.parameters.header", "get", _map_inc_excl_attributes) + _process_setting(section, "security.skip_iast_scan.parameters.query", "get", _map_inc_excl_attributes) + _process_setting(section, "security.skip_iast_scan.parameters.body", "get", _map_inc_excl_attributes) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.insecure_settings", "getboolean", None) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.invalid_file_access", "getboolean", None) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.sql_injection", "getboolean", None) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.nosql_injection", "getboolean", None) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.ldap_injection", "getboolean", None) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.javascript_injection", "getboolean", None) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.command_injection", "getboolean", None) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.xpath_injection", "getboolean", None) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.ssrf", "getboolean", None) + _process_setting(section, "security.skip_iast_scan.iast_detection_category.rxss", "getboolean", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 6e9af73125..1b4161c6f5 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -362,6 +362,15 @@ class SecurityDetectionDeserializationSettings(Settings): class SecurityRequestSettings(Settings): pass +class SecuritySkipIASTScanSettings(Settings): + pass + +class SecuritySkipIASTScanParametersSettings(Settings): + pass + +class SecuritySkipIASTScanIASTDetectionCategorySettings(Settings): + pass + class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -496,6 +505,10 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.security.detection.rci = SecurityDetectionRCISettings() _settings.security.detection.rxss = SecurityDetectionRXSSSettings() _settings.security.request = SecurityRequestSettings() +_settings.security.skip_iast_scan = SecuritySkipIASTScanSettings() +_settings.security.skip_iast_scan = SecuritySkipIASTScanAPISettings() +_settings.security.skip_iast_scan.parameters = SecuritySkipIASTScanParametersSettings() +_settings.security.skip_iast_scan.iast_detection_category = SecuritySkipIASTScanIASTDetectionCategorySettings() _settings.serverless_mode = ServerlessModeSettings() _settings.slow_sql = SlowSqlSettings() _settings.span_events = SpanEventSettings() @@ -1006,8 +1019,48 @@ def default_otlp_host(host): "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", True ) _settings.security.request.body_limit = os.environ.get("NEW_RELIC_SECURITY_REQUEST_BODY_LIMIT", None) - - +_settings.security.skip_iast_scan.api = _environ_as_set( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_API", default="" +) +_settings.security.skip_iast_scan.parameters.header = _environ_as_set( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_PARAMETERS_HEADER", default="" +) +_settings.security.skip_iast_scan.parameters.query = _environ_as_set( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_PARAMETERS_QUERY", default="" +) +_settings.security.skip_iast_scan.parameters.body = _environ_as_set( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_PARAMETERS_BODY", default="" +) +_settings.security.skip_iast_scan.iast_detection_category.insecure_settings = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_INSECURE_SETTINGS", False +) +_settings.security.skip_iast_scan.iast_detection_category.invalid_file_access = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_INVALID_FILE_ACCESS", False +) +_settings.security.skip_iast_scan.iast_detection_category.sql_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_SQL_INJECTION", False +) +_settings.security.skip_iast_scan.iast_detection_category.nosql_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_NOSQL_INJECTION", False +) +_settings.security.skip_iast_scan.iast_detection_category.ldap_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_LDAP_INJECTION", False +) +_settings.security.skip_iast_scan.iast_detection_category.javascript_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_JAVASCRIPT_INJECTION", False +) +_settings.security.skip_iast_scan.iast_detection_category.command_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_COMMAND_INJECTION", False +) +_settings.security.skip_iast_scan.iast_detection_category.xpath_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_XPATH_INJECTION", False +) +_settings.security.skip_iast_scan.iast_detection_category.ssrf = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_SSRF", False +) +_settings.security.skip_iast_scan.iast_detection_category.rxss = _environ_as_bool( + "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_RXSS", False +) def global_settings(): """This returns the default global settings. Generally only used directly in test scripts and test harnesses or when applying global diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 1e85932540..05189a3bc0 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -79,6 +79,48 @@ security.detection.deserialization.enabled = true security.request.body_limit = 300 +# The skip_iast_scan configuration allows users to specify APIs, +# parameters, and categories that should not be scanned by Security Agents. + +# APIs can be specified using regular expression (regex) patterns that +# follow the syntax of Perl 5. The regex pattern should provide a complete +# match for the URL without the endpoint. +# security.skip_iast_scan.api = [".*\/api\/v1\/.*?\/login"] + +# The parameters configuration allows users to specify headers, query +# parameters, and body keys that should be excluded from IAST scans. + +# A list of HTTP header keys. If a request includes any headers with +# these keys, the corresponding IAST scan will be skipped. +# security.skip_iast_scan.parameters.header = ["X-Forwaded-For"] + +# A list of query parameter keys. The presence of these parameters in +# the request's query string will lead to skipping the IAST scan. +# security.skip_iast_scan.parameters.query = ["Q1", "Q2"] + +# A list of keys within the request body. If these keys are found in +# the body content, the IAST scan will be omitted. Supported content +# types for the request body include JSON, XML, and Form-URL-Encoded +# data. +# security.skip_iast_scan.parameters.body = ["object.cc_number"] + +# The iast_detection_category configuration allows users to specify +# which categories of vulnerabilities should not be detected by +# Security Agents. If any of these categories are set to false, +# Security Agents will not generate events or flag vulnerabilities +# for that category. +security.skip_iast_scan.iast_detection_category.insecure_settings = false +security.skip_iast_scan.iast_detection_category.invalid_file_access = false +security.skip_iast_scan.iast_detection_category.sql_injection = false +security.skip_iast_scan.iast_detection_category.nosql_injection = false +security.skip_iast_scan.iast_detection_category.ldap_injection = false +security.skip_iast_scan.iast_detection_category.javascript_injection = false +security.skip_iast_scan.iast_detection_category.command_injection = false +security.skip_iast_scan.iast_detection_category.xpath_injection = false +security.skip_iast_scan.iast_detection_category.ssrf = false +security.skip_iast_scan.iast_detection_category.rxss = false + + # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the # containing directory and the file itself are correct, and diff --git a/setup.py b/setup.py index 67f34bce69..74ddde321d 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@develop"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@task/NR-301856/skip_scan"] ) if with_setuptools: From 6905d13a81747a998525fc6b777ffe22942506ac Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 21 Aug 2024 16:14:42 +0530 Subject: [PATCH 66/79] minor fix --- newrelic/core/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 1b4161c6f5..707ff5d027 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -506,7 +506,6 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.security.detection.rxss = SecurityDetectionRXSSSettings() _settings.security.request = SecurityRequestSettings() _settings.security.skip_iast_scan = SecuritySkipIASTScanSettings() -_settings.security.skip_iast_scan = SecuritySkipIASTScanAPISettings() _settings.security.skip_iast_scan.parameters = SecuritySkipIASTScanParametersSettings() _settings.security.skip_iast_scan.iast_detection_category = SecuritySkipIASTScanIASTDetectionCategorySettings() _settings.serverless_mode = ServerlessModeSettings() From 2c757b377ca2f896e5af0a8944cd6de3d1fc6feb Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 22 Aug 2024 01:22:26 +0530 Subject: [PATCH 67/79] Changes for defining and reading security scan schedule config --- newrelic/config.py | 4 ++++ newrelic/core/config.py | 10 ++++++++++ newrelic/newrelic.ini | 40 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 7a6852ec86..485ac50fc5 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -346,6 +346,10 @@ def _process_configuration(section): _process_setting(section, "security.detection.rxss.enabled", "getboolean", None) _process_setting(section, "security.detection.deserialization.enabled", "getboolean", None) _process_setting(section, "security.request.body_limit", "get", None) + _process_setting(section, "security.scan_schedule.schedule", "get", None) + _process_setting(section, "security.scan_schedule.duration", "getint", None) + _process_setting(section, "security.scan_schedule.delay", "getint", None) + _process_setting(section, "security.scan_schedule.always_sample_traces", "getboolean", None) _process_setting(section, "security.skip_iast_scan.api", "get", _map_inc_excl_attributes) _process_setting(section, "security.skip_iast_scan.parameters.header", "get", _map_inc_excl_attributes) _process_setting(section, "security.skip_iast_scan.parameters.query", "get", _map_inc_excl_attributes) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 707ff5d027..1f218b0cb4 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -362,6 +362,9 @@ class SecurityDetectionDeserializationSettings(Settings): class SecurityRequestSettings(Settings): pass +class SecurityScanScheduleSettings(Settings): + pass + class SecuritySkipIASTScanSettings(Settings): pass @@ -505,6 +508,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.security.detection.rci = SecurityDetectionRCISettings() _settings.security.detection.rxss = SecurityDetectionRXSSSettings() _settings.security.request = SecurityRequestSettings() +_settings.security.scan_schedule = SecurityScanScheduleSettings() _settings.security.skip_iast_scan = SecuritySkipIASTScanSettings() _settings.security.skip_iast_scan.parameters = SecuritySkipIASTScanParametersSettings() _settings.security.skip_iast_scan.iast_detection_category = SecuritySkipIASTScanIASTDetectionCategorySettings() @@ -1018,6 +1022,12 @@ def default_otlp_host(host): "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", True ) _settings.security.request.body_limit = os.environ.get("NEW_RELIC_SECURITY_REQUEST_BODY_LIMIT", None) +_settings.security.scan_schedule.schedule = os.environ.get("NEW_RELIC_SECURITY_SCAN_SCHEDULE_SCHEDULE", None) +_settings.security.scan_schedule.duration = _environ_as_int("NEW_RELIC_SECURITY_SCAN_SCHEDULE_DURATION", -1) +_settings.security.scan_schedule.delay = _environ_as_int("NEW_RELIC_SECURITY_SCAN_SCHEDULE_DELAY", 0) +_settings.security.scan_schedule.always_sample_traces = _environ_as_bool( + "NEW_RELIC_SECURITY_SCAN_SCHEDULE_ALWAYS_SAMPLE_TRACES", False + ) _settings.security.skip_iast_scan.api = _environ_as_set( "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_API", default="" ) diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 05189a3bc0..13f206d2df 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -79,6 +79,42 @@ security.detection.deserialization.enabled = true security.request.body_limit = 300 +# Schedule IAST will allow users to schedule +# the startup and scanning of IAST. Users will have +# the flexibility to start and stop IAST at specific +# times or simply add a delay from the application +# start time + +# The schedule field specifies a cron expression that +# defines when the IAST scan should run. The cron +# expression consists of six fields separated by spaces: +# +# second: specifies the second of the hour (0-59) +# minute: specifies the minute of the hour (0-59) +# hour: specifies the hour of the day (0-23) +# day: specifies the day of the month (1-31) +# month: the month of the year (1-12) +# day_of_week: specifies the day of the week (0-6), where 0 = Sunday +# security.scan_schedule.schedule = "0 0 * * * ?" + +# The duration field specifies the duration of +# the IAST scan in minutes. This determines how +# long the scan will run. +# Default value is forever i.e. no limits +# security.scan_schedule.duration = 300 + +# The delay field specifies the delay in +# minutes before the IAST scan starts. +# This allows you to schedule the scan to +# start at a later time. +# Default value: 0 minutes i.e. no delay +# security.scan_schedule.delay = 300 + +# This allows the newrelic security agent to collect +# sample data even when scan is not active. +# Default value is false. +# security.scan_schedule.always_sample_traces = false + # The skip_iast_scan configuration allows users to specify APIs, # parameters, and categories that should not be scanned by Security Agents. @@ -104,8 +140,8 @@ security.request.body_limit = 300 # data. # security.skip_iast_scan.parameters.body = ["object.cc_number"] -# The iast_detection_category configuration allows users to specify -# which categories of vulnerabilities should not be detected by +# The iast_detection_category configuration allows users to specify +# which categories of vulnerabilities should not be detected by # Security Agents. If any of these categories are set to false, # Security Agents will not generate events or flag vulnerabilities # for that category. From 8b52f2548a36bd005dba3aac3edaf83a71c5f83c Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 27 Aug 2024 10:07:42 +0530 Subject: [PATCH 68/79] Refactored config change for iast scan exclude configs --- newrelic/config.py | 28 +++++++-------- newrelic/core/config.py | 76 +++++++++++++++++++++++------------------ newrelic/newrelic.ini | 32 ++++++++--------- 3 files changed, 72 insertions(+), 64 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 485ac50fc5..efb4a83563 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -345,25 +345,25 @@ def _process_configuration(section): _process_setting(section, "security.detection.rci.enabled", "getboolean", None) _process_setting(section, "security.detection.rxss.enabled", "getboolean", None) _process_setting(section, "security.detection.deserialization.enabled", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.api", "get", _map_inc_excl_attributes) + _process_setting(section, "security.exclude_from_iast_scan.http_request_parameters.header", "get", _map_inc_excl_attributes) + _process_setting(section, "security.exclude_from_iast_scan.http_request_parameters.query", "get", _map_inc_excl_attributes) + _process_setting(section, "security.exclude_from_iast_scan.http_request_parameters.body", "get", _map_inc_excl_attributes) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.insecure_settings", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.invalid_file_access", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.sql_injection", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.nosql_injection", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.ldap_injection", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.javascript_injection", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.command_injection", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.xpath_injection", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.ssrf", "getboolean", None) + _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.rxss", "getboolean", None) _process_setting(section, "security.request.body_limit", "get", None) _process_setting(section, "security.scan_schedule.schedule", "get", None) _process_setting(section, "security.scan_schedule.duration", "getint", None) _process_setting(section, "security.scan_schedule.delay", "getint", None) _process_setting(section, "security.scan_schedule.always_sample_traces", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.api", "get", _map_inc_excl_attributes) - _process_setting(section, "security.skip_iast_scan.parameters.header", "get", _map_inc_excl_attributes) - _process_setting(section, "security.skip_iast_scan.parameters.query", "get", _map_inc_excl_attributes) - _process_setting(section, "security.skip_iast_scan.parameters.body", "get", _map_inc_excl_attributes) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.insecure_settings", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.invalid_file_access", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.sql_injection", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.nosql_injection", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.ldap_injection", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.javascript_injection", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.command_injection", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.xpath_injection", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.ssrf", "getboolean", None) - _process_setting(section, "security.skip_iast_scan.iast_detection_category.rxss", "getboolean", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 1f218b0cb4..8021afd1a2 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -365,13 +365,13 @@ class SecurityRequestSettings(Settings): class SecurityScanScheduleSettings(Settings): pass -class SecuritySkipIASTScanSettings(Settings): +class SecurityExcludeFromIASTScanSettings(Settings): pass -class SecuritySkipIASTScanParametersSettings(Settings): +class SecurityExcludeFromIASTScanHTTPRequestParametersSettings(Settings): pass -class SecuritySkipIASTScanIASTDetectionCategorySettings(Settings): +class SecurityExcludeFromIASTScanIASTDetectionCategorySettings(Settings): pass class InfiniteTracingSettings(Settings): @@ -507,11 +507,13 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() _settings.security.detection.rci = SecurityDetectionRCISettings() _settings.security.detection.rxss = SecurityDetectionRXSSSettings() +_settings.security.exclude_from_iast_scan = SecurityExcludeFromIASTScanSettings() +_settings.security.exclude_from_iast_scan.http_request_parameters = \ + SecurityExcludeFromIASTScanHTTPRequestParametersSettings() +_settings.security.exclude_from_iast_scan.iast_detection_category = \ + SecurityExcludeFromIASTScanIASTDetectionCategorySettings() _settings.security.request = SecurityRequestSettings() _settings.security.scan_schedule = SecurityScanScheduleSettings() -_settings.security.skip_iast_scan = SecuritySkipIASTScanSettings() -_settings.security.skip_iast_scan.parameters = SecuritySkipIASTScanParametersSettings() -_settings.security.skip_iast_scan.iast_detection_category = SecuritySkipIASTScanIASTDetectionCategorySettings() _settings.serverless_mode = ServerlessModeSettings() _settings.slow_sql = SlowSqlSettings() _settings.span_events = SpanEventSettings() @@ -1028,48 +1030,54 @@ def default_otlp_host(host): _settings.security.scan_schedule.always_sample_traces = _environ_as_bool( "NEW_RELIC_SECURITY_SCAN_SCHEDULE_ALWAYS_SAMPLE_TRACES", False ) -_settings.security.skip_iast_scan.api = _environ_as_set( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_API", default="" +_settings.security.exclude_from_iast_scan.api = _environ_as_set( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_API", default="" ) -_settings.security.skip_iast_scan.parameters.header = _environ_as_set( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_PARAMETERS_HEADER", default="" +_settings.security.exclude_from_iast_scan.http_request_parameters.header = _environ_as_set( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_HTTP_REQUEST_PARAMETERS_HEADER", default="" ) -_settings.security.skip_iast_scan.parameters.query = _environ_as_set( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_PARAMETERS_QUERY", default="" +_settings.security.exclude_from_iast_scan.http_request_parameters.query = _environ_as_set( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_HTTP_REQUEST_PARAMETERS_QUERY", default="" ) -_settings.security.skip_iast_scan.parameters.body = _environ_as_set( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_PARAMETERS_BODY", default="" +_settings.security.exclude_from_iast_scan.http_request_parameters.body = _environ_as_set( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_HTTP_REQUEST_PARAMETERS_BODY", default="" ) -_settings.security.skip_iast_scan.iast_detection_category.insecure_settings = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_INSECURE_SETTINGS", False +_settings.security.exclude_from_iast_scan.iast_detection_category.insecure_settings = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_INSECURE_SETTINGS", False ) -_settings.security.skip_iast_scan.iast_detection_category.invalid_file_access = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_INVALID_FILE_ACCESS", False +_settings.security.exclude_from_iast_scan.iast_detection_category.invalid_file_access = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_INVALID_FILE_ACCESS", False ) -_settings.security.skip_iast_scan.iast_detection_category.sql_injection = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_SQL_INJECTION", False +_settings.security.exclude_from_iast_scan.iast_detection_category.sql_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_SQL_INJECTION", False ) -_settings.security.skip_iast_scan.iast_detection_category.nosql_injection = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_NOSQL_INJECTION", False +_settings.security.exclude_from_iast_scan.iast_detection_category.nosql_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_NOSQL_INJECTION", False ) -_settings.security.skip_iast_scan.iast_detection_category.ldap_injection = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_LDAP_INJECTION", False +_settings.security.exclude_from_iast_scan.iast_detection_category.ldap_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_LDAP_INJECTION", False ) -_settings.security.skip_iast_scan.iast_detection_category.javascript_injection = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_JAVASCRIPT_INJECTION", False +_settings.security.exclude_from_iast_scan.iast_detection_category.javascript_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_JAVASCRIPT_INJECTION", False ) -_settings.security.skip_iast_scan.iast_detection_category.command_injection = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_COMMAND_INJECTION", False +_settings.security.exclude_from_iast_scan.iast_detection_category.command_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_COMMAND_INJECTION", False ) -_settings.security.skip_iast_scan.iast_detection_category.xpath_injection = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_XPATH_INJECTION", False +_settings.security.exclude_from_iast_scan.iast_detection_category.xpath_injection = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_XPATH_INJECTION", False ) -_settings.security.skip_iast_scan.iast_detection_category.ssrf = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_SSRF", False +_settings.security.exclude_from_iast_scan.iast_detection_category.ssrf = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_SSRF", False ) -_settings.security.skip_iast_scan.iast_detection_category.rxss = _environ_as_bool( - "NEW_RELIC_SECURITY_SKIP_IAST_SCAN_IAST_DETECTION_CATEGORY_RXSS", False +_settings.security.exclude_from_iast_scan.iast_detection_category.rxss = _environ_as_bool( + "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_RXSS", False ) +_settings.security.scan_schedule.schedule = os.environ.get("NEW_RELIC_SECURITY_SCAN_SCHEDULE_SCHEDULE", None) +_settings.security.scan_schedule.duration = _environ_as_int("NEW_RELIC_SECURITY_SCAN_SCHEDULE_DURATION", -1) +_settings.security.scan_schedule.delay = _environ_as_int("NEW_RELIC_SECURITY_SCAN_SCHEDULE_DELAY", 0) +_settings.security.scan_schedule.always_sample_traces = _environ_as_bool("NEW_RELIC_SECURITY_SCAN_SCHEDULE_ALWAYS_SAMPLE_TRACES", False) + + def global_settings(): """This returns the default global settings. Generally only used directly in test scripts and test harnesses or when applying global diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 13f206d2df..d49be77623 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -115,46 +115,46 @@ security.request.body_limit = 300 # Default value is false. # security.scan_schedule.always_sample_traces = false -# The skip_iast_scan configuration allows users to specify APIs, +# The exclude_from_iast_scan configuration allows users to specify APIs, # parameters, and categories that should not be scanned by Security Agents. # APIs can be specified using regular expression (regex) patterns that # follow the syntax of Perl 5. The regex pattern should provide a complete # match for the URL without the endpoint. -# security.skip_iast_scan.api = [".*\/api\/v1\/.*?\/login"] +# security.exclude_from_iast_scan.api = [".*\/api\/v1\/.*?\/login"] -# The parameters configuration allows users to specify headers, query +# The http_request_parameters configuration allows users to specify headers, query # parameters, and body keys that should be excluded from IAST scans. # A list of HTTP header keys. If a request includes any headers with # these keys, the corresponding IAST scan will be skipped. -# security.skip_iast_scan.parameters.header = ["X-Forwaded-For"] +# security.exclude_from_iast_scan.http_request_parameters.header = ["X-Forwaded-For"] # A list of query parameter keys. The presence of these parameters in # the request's query string will lead to skipping the IAST scan. -# security.skip_iast_scan.parameters.query = ["Q1", "Q2"] +# security.exclude_from_iast_scan.http_request_parameters.query = ["Q1", "Q2"] # A list of keys within the request body. If these keys are found in # the body content, the IAST scan will be omitted. Supported content # types for the request body include JSON, XML, and Form-URL-Encoded # data. -# security.skip_iast_scan.parameters.body = ["object.cc_number"] +# security.exclude_from_iast_scan.http_request_parameters.body = ["object.cc_number"] # The iast_detection_category configuration allows users to specify # which categories of vulnerabilities should not be detected by # Security Agents. If any of these categories are set to false, # Security Agents will not generate events or flag vulnerabilities # for that category. -security.skip_iast_scan.iast_detection_category.insecure_settings = false -security.skip_iast_scan.iast_detection_category.invalid_file_access = false -security.skip_iast_scan.iast_detection_category.sql_injection = false -security.skip_iast_scan.iast_detection_category.nosql_injection = false -security.skip_iast_scan.iast_detection_category.ldap_injection = false -security.skip_iast_scan.iast_detection_category.javascript_injection = false -security.skip_iast_scan.iast_detection_category.command_injection = false -security.skip_iast_scan.iast_detection_category.xpath_injection = false -security.skip_iast_scan.iast_detection_category.ssrf = false -security.skip_iast_scan.iast_detection_category.rxss = false +security.exclude_from_iast_scan.iast_detection_category.insecure_settings = false +security.exclude_from_iast_scan.iast_detection_category.invalid_file_access = false +security.exclude_from_iast_scan.iast_detection_category.sql_injection = false +security.exclude_from_iast_scan.iast_detection_category.nosql_injection = false +security.exclude_from_iast_scan.iast_detection_category.ldap_injection = false +security.exclude_from_iast_scan.iast_detection_category.javascript_injection = false +security.exclude_from_iast_scan.iast_detection_category.command_injection = false +security.exclude_from_iast_scan.iast_detection_category.xpath_injection = false +security.exclude_from_iast_scan.iast_detection_category.ssrf = false +security.exclude_from_iast_scan.iast_detection_category.rxss = false # Sets the name of a file to log agent messages to. Whatever you From 16e1c80eb670822750943378f31cd6315c1c2f4c Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 10 Sep 2024 09:56:14 +0530 Subject: [PATCH 69/79] Fix to incorrectly read security config --- newrelic/config.py | 11 +++++++---- newrelic/newrelic.ini | 10 +++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index efb4a83563..fcc8e6c0a9 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -223,6 +223,9 @@ def _map_default_host_value(license_key): return license_key +def _map_comma_separated_values(s): + return list(map(str.strip, s.split(","))) + # Processing of a single setting from configuration file. @@ -345,10 +348,10 @@ def _process_configuration(section): _process_setting(section, "security.detection.rci.enabled", "getboolean", None) _process_setting(section, "security.detection.rxss.enabled", "getboolean", None) _process_setting(section, "security.detection.deserialization.enabled", "getboolean", None) - _process_setting(section, "security.exclude_from_iast_scan.api", "get", _map_inc_excl_attributes) - _process_setting(section, "security.exclude_from_iast_scan.http_request_parameters.header", "get", _map_inc_excl_attributes) - _process_setting(section, "security.exclude_from_iast_scan.http_request_parameters.query", "get", _map_inc_excl_attributes) - _process_setting(section, "security.exclude_from_iast_scan.http_request_parameters.body", "get", _map_inc_excl_attributes) + _process_setting(section, "security.exclude_from_iast_scan.api", "get", _map_comma_separated_values) + _process_setting(section, "security.exclude_from_iast_scan.http_request_parameters.header", "get", _map_comma_separated_values) + _process_setting(section, "security.exclude_from_iast_scan.http_request_parameters.query", "get", _map_comma_separated_values) + _process_setting(section, "security.exclude_from_iast_scan.http_request_parameters.body", "get", _map_comma_separated_values) _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.insecure_settings", "getboolean", None) _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.invalid_file_access", "getboolean", None) _process_setting(section, "security.exclude_from_iast_scan.iast_detection_category.sql_injection", "getboolean", None) diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index d49be77623..45a72befc8 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -95,7 +95,7 @@ security.request.body_limit = 300 # day: specifies the day of the month (1-31) # month: the month of the year (1-12) # day_of_week: specifies the day of the week (0-6), where 0 = Sunday -# security.scan_schedule.schedule = "0 0 * * * ?" +# security.scan_schedule.schedule = 0 0 * * * ? # The duration field specifies the duration of # the IAST scan in minutes. This determines how @@ -121,24 +121,24 @@ security.request.body_limit = 300 # APIs can be specified using regular expression (regex) patterns that # follow the syntax of Perl 5. The regex pattern should provide a complete # match for the URL without the endpoint. -# security.exclude_from_iast_scan.api = [".*\/api\/v1\/.*?\/login"] +# security.exclude_from_iast_scan.api = .*\/api\/v1\/.*?\/login, .*\/api\/v2\/.*?\/login # The http_request_parameters configuration allows users to specify headers, query # parameters, and body keys that should be excluded from IAST scans. # A list of HTTP header keys. If a request includes any headers with # these keys, the corresponding IAST scan will be skipped. -# security.exclude_from_iast_scan.http_request_parameters.header = ["X-Forwaded-For"] +# security.exclude_from_iast_scan.http_request_parameters.header = X-Forwaded-For, Set-Cookie # A list of query parameter keys. The presence of these parameters in # the request's query string will lead to skipping the IAST scan. -# security.exclude_from_iast_scan.http_request_parameters.query = ["Q1", "Q2"] +# security.exclude_from_iast_scan.http_request_parameters.query = Query Parameter 1, Query Parameter 2 # A list of keys within the request body. If these keys are found in # the body content, the IAST scan will be omitted. Supported content # types for the request body include JSON, XML, and Form-URL-Encoded # data. -# security.exclude_from_iast_scan.http_request_parameters.body = ["object.cc_number"] +# security.exclude_from_iast_scan.http_request_parameters.body = object.cc_number # The iast_detection_category configuration allows users to specify # which categories of vulnerabilities should not be detected by From d97a15cd942039a94eab2e7acf00cdba19f3b9ca Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 17 Sep 2024 13:17:15 +0530 Subject: [PATCH 70/79] Updates for csec agent moved to newrelic org --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67f34bce69..8ab4a2d6be 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@develop"] + install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop"] ) if with_setuptools: From bfc5219c8f0e96cc4d7df52ae14d6ef8c5c52087 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 11 Oct 2024 12:15:19 +0530 Subject: [PATCH 71/79] Removal of security.request.body_limit config --- newrelic/config.py | 1 - newrelic/core/config.py | 5 ----- newrelic/newrelic.ini | 4 ---- 3 files changed, 10 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index fff09bf3b2..d5a4635d5b 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -334,7 +334,6 @@ def _process_configuration(section): _process_setting(section, "security.detection.rci.enabled", "getboolean", None) _process_setting(section, "security.detection.rxss.enabled", "getboolean", None) _process_setting(section, "security.detection.deserialization.enabled", "getboolean", None) - _process_setting(section, "security.request.body_limit", "get", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index fe2ba88d44..27a6ea8d17 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -355,9 +355,6 @@ class SecurityDetectionRXSSSettings(Settings): class SecurityDetectionDeserializationSettings(Settings): pass -class SecurityRequestSettings(Settings): - pass - class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -491,7 +488,6 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() _settings.security.detection.rci = SecurityDetectionRCISettings() _settings.security.detection.rxss = SecurityDetectionRXSSSettings() -_settings.security.request = SecurityRequestSettings() _settings.serverless_mode = ServerlessModeSettings() _settings.slow_sql = SlowSqlSettings() _settings.span_events = SpanEventSettings() @@ -1001,7 +997,6 @@ def default_otlp_host(host): _settings.security.detection.deserialization.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", True ) -_settings.security.request.body_limit = os.environ.get("NEW_RELIC_SECURITY_REQUEST_BODY_LIMIT", None) def global_settings(): diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 02c2202194..e8da2a911b 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -75,10 +75,6 @@ security.detection.rxss.enabled = true security.detection.deserialization.enabled = true -# security request body read limiting in kb -security.request.body_limit = 300 - - # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the # containing directory and the file itself are correct, and From 4a5daeb1a4f219a097fd17d3d49228dfb72ec628 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 15 Oct 2024 09:50:17 +0530 Subject: [PATCH 72/79] Updated remote for security agent --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 74ddde321d..a1643b0f7a 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent.git@task/NR-301856/skip_scan"] + install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@task/NR-301856/skip_scan"] ) if with_setuptools: From 9f9a4d4f1ea4e06a6b18caf45873418d2ef41dfd Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 29 Oct 2024 11:24:43 +0530 Subject: [PATCH 73/79] Branch update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a1643b0f7a..ee66b9dad1 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@task/NR-301856/skip_scan"] + install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@task/NR-333351/scheduled_agent"] ) if with_setuptools: From 126cd8c5fd2ade6e2c73fc78bb4ae5cba06c16ee Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 19 Dec 2024 00:45:34 +0530 Subject: [PATCH 74/79] Minor fix --- newrelic/core/config.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 5408a73c4a..286a04545b 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -355,7 +355,6 @@ class SecurityDetectionRXSSSettings(Settings): class SecurityDetectionDeserializationSettings(Settings): pass -<<<<<<< HEAD class SecurityRequestSettings(Settings): pass @@ -371,8 +370,6 @@ class SecurityExcludeFromIASTScanHTTPRequestParametersSettings(Settings): class SecurityExcludeFromIASTScanIASTDetectionCategorySettings(Settings): pass -======= ->>>>>>> develop class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -506,7 +503,6 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() _settings.security.detection.rci = SecurityDetectionRCISettings() _settings.security.detection.rxss = SecurityDetectionRXSSSettings() -<<<<<<< HEAD _settings.security.exclude_from_iast_scan = SecurityExcludeFromIASTScanSettings() _settings.security.exclude_from_iast_scan.http_request_parameters = \ SecurityExcludeFromIASTScanHTTPRequestParametersSettings() @@ -514,8 +510,6 @@ class EventHarvestConfigHarvestLimitSettings(Settings): SecurityExcludeFromIASTScanIASTDetectionCategorySettings() _settings.security.request = SecurityRequestSettings() _settings.security.scan_schedule = SecurityScanScheduleSettings() -======= ->>>>>>> develop _settings.serverless_mode = ServerlessModeSettings() _settings.slow_sql = SlowSqlSettings() _settings.span_events = SpanEventSettings() From 67481fe389dc2bea7b27b7d0559c44e880d9fcc3 Mon Sep 17 00:00:00 2001 From: ajuniwal Date: Thu, 30 Jan 2025 02:14:18 +0530 Subject: [PATCH 75/79] newrelic_security dependency update --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b2721d60ea..988c708ea1 100644 --- a/setup.py +++ b/setup.py @@ -181,9 +181,8 @@ def build_extension(self, ext): package_data={ "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, - #install_requires=["newrelic-security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic-security"], extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop"] + install_requires=["newrelic_security"] ) if with_setuptools: From 7e556610411de34d7f45b0344f8cc81b6ae205dc Mon Sep 17 00:00:00 2001 From: ajuniwal Date: Thu, 30 Jan 2025 19:32:41 +0530 Subject: [PATCH 76/79] Pushed newrelic security as extra --- newrelic/config.py | 2 ++ setup.py | 7 +------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 787129b296..3aff9fbf1c 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -4867,6 +4867,8 @@ def _setup_security_module(): security_agent = get_agent() # create a callback to reinitialise the security module newrelic.core.agent.Agent.run_on_startup(security_agent.refresh_agent) + except ImportError: + _logger.warn("Security Agent isn't available") except Exception as csec_error: _logger.error("Security Agent Startup failed with error %s", csec_error) diff --git a/setup.py b/setup.py index 9a7ec006e5..e609940d93 100644 --- a/setup.py +++ b/setup.py @@ -181,12 +181,7 @@ def build_extension(self, ext): package_data={ "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, - extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, -<<<<<<< HEAD - install_requires=["newrelic_security"] -======= - install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@task/NR-333351/scheduled_agent"] ->>>>>>> develop + extras_require={"infinite-tracing": ["grpcio", "protobuf"], "iast": ["newrelic_security"]}, ) if with_setuptools: From a55766ff69e3f7a5fef84071f8ec92fa3a390e23 Mon Sep 17 00:00:00 2001 From: himank-k2 <92847262+himank-k2@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:48:18 +0530 Subject: [PATCH 77/79] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b10a789f98..b2721d60ea 100644 --- a/setup.py +++ b/setup.py @@ -183,7 +183,7 @@ def build_extension(self, ext): }, #install_requires=["newrelic-security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic-security"], extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@task/NR-333351/scheduled_agent"] + install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop"] ) if with_setuptools: From eeb534bea9f99404fa9aca396c185315cddf8818 Mon Sep 17 00:00:00 2001 From: ajuniwal Date: Fri, 31 Jan 2025 12:51:03 +0530 Subject: [PATCH 78/79] Added endpoint for security engine as default value and removed redundant security config --- newrelic/core/config.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 8eae3c4f9b..efbcbf093d 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -1060,7 +1060,7 @@ def default_otlp_host(host): _settings.security.agent.enabled = _environ_as_bool("NEW_RELIC_SECURITY_AGENT_ENABLED", False) _settings.security.enabled = _environ_as_bool("NEW_RELIC_SECURITY_ENABLED", False) _settings.security.mode = os.environ.get("NEW_RELIC_SECURITY_MODE", "IAST") -_settings.security.validator_service_url = os.environ.get("NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", None) +_settings.security.validator_service_url = os.environ.get("NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", "wss://csec.nr-data.net") _settings.security.detection.rci.enabled = _environ_as_bool("NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", True) _settings.security.detection.rxss.enabled = _environ_as_bool("NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", True) _settings.security.detection.deserialization.enabled = _environ_as_bool( @@ -1115,10 +1115,6 @@ def default_otlp_host(host): _settings.security.exclude_from_iast_scan.iast_detection_category.rxss = _environ_as_bool( "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_RXSS", False ) -_settings.security.scan_schedule.schedule = os.environ.get("NEW_RELIC_SECURITY_SCAN_SCHEDULE_SCHEDULE", None) -_settings.security.scan_schedule.duration = _environ_as_int("NEW_RELIC_SECURITY_SCAN_SCHEDULE_DURATION", -1) -_settings.security.scan_schedule.delay = _environ_as_int("NEW_RELIC_SECURITY_SCAN_SCHEDULE_DELAY", 0) -_settings.security.scan_schedule.always_sample_traces = _environ_as_bool("NEW_RELIC_SECURITY_SCAN_SCHEDULE_ALWAYS_SAMPLE_TRACES", False) def global_settings(): From 4ebd06086f56fd44af3ed5233b95d47f2c0a0db2 Mon Sep 17 00:00:00 2001 From: ajuniwal Date: Fri, 31 Jan 2025 18:23:49 +0530 Subject: [PATCH 79/79] minor update --- newrelic/newrelic.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index f0eed3529d..29b57313c6 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -142,7 +142,7 @@ security.request.body_limit = 300 # The iast_detection_category configuration allows users to specify # which categories of vulnerabilities should not be detected by -# Security Agents. If any of these categories are set to false, +# Security Agents. If any of these categories are set to true, # Security Agents will not generate events or flag vulnerabilities # for that category. security.exclude_from_iast_scan.iast_detection_category.insecure_settings = false