From 75a60b3f7dd5ed21dd893304eb192ae0fe92a8ee Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju Date: Wed, 30 Apr 2025 19:06:02 +0530 Subject: [PATCH 1/7] feat: support new resolution strategy values --- processor/consent.go | 55 +- processor/consent_test.go | 1417 +++++++++++++++++++++++++++++++------ 2 files changed, 1232 insertions(+), 240 deletions(-) diff --git a/processor/consent.go b/processor/consent.go index 48cc1a45b8..8b145340d7 100644 --- a/processor/consent.go +++ b/processor/consent.go @@ -2,6 +2,7 @@ package processor import ( "fmt" + "strings" "github.com/samber/lo" @@ -12,10 +13,10 @@ import ( ) type ConsentManagementInfo struct { - DeniedConsentIDs []string `json:"deniedConsentIds"` - AllowedConsentIDs interface{} `json:"allowedConsentIds"` // Not used currently but added for future use - Provider string `json:"provider"` - ResolutionStrategy string `json:"resolutionStrategy"` + AllowedConsentIDs []string `json:"allowedConsentIds"` + DeniedConsentIDs []string `json:"deniedConsentIds"` + Provider string `json:"provider"` + ResolutionStrategy string `json:"resolutionStrategy"` } type GenericConsentManagementProviderData struct { @@ -47,14 +48,14 @@ func (proc *Handle) getConsentFilteredDestinations(event types.SingularEventT, s proc.logger.Errorw("failed to get consent management info", "error", err.Error()) } - if len(consentManagementInfo.DeniedConsentIDs) == 0 { + // If the event does not have any consents, do not filter any destinations + if len(consentManagementInfo.AllowedConsentIDs) == 0 && len(consentManagementInfo.DeniedConsentIDs) == 0 { return destinations } return lo.Filter(destinations, func(dest backendconfig.DestinationT, _ int) bool { // Generic consent management if cmpData := proc.getGCMData(sourceID, dest.ID, consentManagementInfo.Provider); len(cmpData.Consents) > 0 { - finalResolutionStrategy := consentManagementInfo.ResolutionStrategy // For custom provider, the resolution strategy is to be picked from the destination config @@ -62,13 +63,23 @@ func (proc *Handle) getConsentFilteredDestinations(event types.SingularEventT, s finalResolutionStrategy = cmpData.ResolutionStrategy } + // TODO: Remove "or" and "and" support once all the SDK clients stop sending it. + // Currently, it is added for backward compatibility. switch finalResolutionStrategy { - // The user must consent to at least one of the configured consents in the destination - case "or": + case "any", "or": + if len(consentManagementInfo.AllowedConsentIDs) > 0 { + // The user must consent to at least one of the configured consents in the destination + return lo.Some(consentManagementInfo.AllowedConsentIDs, cmpData.Consents) || len(cmpData.Consents) == 0 + } + // All of the configured consents should not be in denied return !lo.Every(consentManagementInfo.DeniedConsentIDs, cmpData.Consents) - // The user must consent to all of the configured consents in the destination - default: // "and" + default: // "all" / "and" + if len(consentManagementInfo.AllowedConsentIDs) > 0 { + // The user must consent to all of the configured consents in the destination + return lo.Every(consentManagementInfo.AllowedConsentIDs, cmpData.Consents) + } + // None of the configured consents should be in denied return len(lo.Intersect(cmpData.Consents, consentManagementInfo.DeniedConsentIDs)) == 0 } } @@ -185,10 +196,14 @@ func getGenericConsentManagementData(dest *backendconfig.DestinationT) (ConsentP consentsConfig := providerConfig.Consents if len(consentsConfig) > 0 && providerConfig.Provider != "" { - consentIDs := lo.FilterMap( - consentsConfig, - func(consentsObj GenericConsentsConfig, _ int) (string, bool) { - return consentsObj.Consent, consentsObj.Consent != "" + consentIDs := lo.Map(consentsConfig, func(consentsObj GenericConsentsConfig, _ int) string { + return strings.TrimSpace(consentsObj.Consent) + }) + + consentIDs = lo.FilterMap( + consentIDs, + func(consentID string, _ int) (string, bool) { + return consentID, consentID != "" }, ) @@ -217,10 +232,20 @@ func getConsentManagementInfo(event types.SingularEventT) (ConsentManagementInfo return consentManagementInfo, fmt.Errorf("error unmarshalling consentManagementInfo: %v", unmErr) } + // Ideally, the clean up and filter is not needed for standard providers + // but useful for custom providers where users send this data directly + // to the SDKs. + cleanupPredicate := func(consent string, _ int) (string, bool) { + return strings.TrimSpace(consent), consent != "" + } + + consentManagementInfo.AllowedConsentIDs = lo.FilterMap(consentManagementInfo.AllowedConsentIDs, cleanupPredicate) + consentManagementInfo.DeniedConsentIDs = lo.FilterMap(consentManagementInfo.DeniedConsentIDs, cleanupPredicate) + filterPredicate := func(consent string, _ int) (string, bool) { return consent, consent != "" } - + consentManagementInfo.AllowedConsentIDs = lo.FilterMap(consentManagementInfo.AllowedConsentIDs, filterPredicate) consentManagementInfo.DeniedConsentIDs = lo.FilterMap(consentManagementInfo.DeniedConsentIDs, filterPredicate) } diff --git a/processor/consent_test.go b/processor/consent_test.go index 8f4bde56be..97aea71e13 100644 --- a/processor/consent_test.go +++ b/processor/consent_test.go @@ -12,9 +12,8 @@ import ( ) type ConnectionInfo struct { - sourceId string - destinations []backendconfig.DestinationT - expectedDestIDs []string + sourceId string + destinations []backendconfig.DestinationT } func TestGetOneTrustConsentCategories(t *testing.T) { @@ -24,12 +23,12 @@ func TestGetOneTrustConsentCategories(t *testing.T) { expected []string }{ { - description: "no oneTrustCookieCategories", + description: "should return nil when no oneTrustCookieCategories are configured", dest: &backendconfig.DestinationT{}, expected: nil, }, { - description: "empty oneTrustCookieCategories", + description: "should return nil when oneTrustCookieCategories array is empty", dest: &backendconfig.DestinationT{ Config: map[string]interface{}{ "oneTrustCookieCategories": []interface{}{}, @@ -38,7 +37,7 @@ func TestGetOneTrustConsentCategories(t *testing.T) { expected: nil, }, { - description: "some oneTrustCookieCategories", + description: "should return valid categories when oneTrustCookieCategories contains valid and invalid entries", dest: &backendconfig.DestinationT{ Config: map[string]interface{}{ "oneTrustCookieCategories": []interface{}{ @@ -70,12 +69,12 @@ func TestGetKetchConsentCategories(t *testing.T) { expected []string }{ { - description: "no ketchConsentPurposes", + description: "should return nil when no ketchConsentPurposes are configured", dest: &backendconfig.DestinationT{}, expected: nil, }, { - description: "empty ketchConsentPurposes", + description: "should return nil when ketchConsentPurposes array is empty", dest: &backendconfig.DestinationT{ Config: map[string]interface{}{ "ketchConsentPurposes": []interface{}{}, @@ -84,7 +83,7 @@ func TestGetKetchConsentCategories(t *testing.T) { expected: nil, }, { - description: "some ketchConsentPurposes", + description: "should return valid categories when ketchConsentPurposes contains valid and invalid entries", dest: &backendconfig.DestinationT{ Config: map[string]interface{}{ "ketchConsentPurposes": []interface{}{ @@ -110,13 +109,16 @@ func TestGetKetchConsentCategories(t *testing.T) { func TestFilterDestinations(t *testing.T) { testCases := []struct { - description string - event types.SingularEventT - connectionInfo []ConnectionInfo + description string + event types.SingularEventT + sourceId string + connectionInfo []ConnectionInfo + expectedDestIDs []string }{ { - description: "no denied consent categories", + description: "should not filter any destination when consent management info is not present in the event", event: types.SingularEventT{}, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -124,27 +126,134 @@ func TestFilterDestinations(t *testing.T) { { ID: "destID-1", Config: map[string]interface{}{ - "oneTrustCookieCategories": []interface{}{ - map[string]interface{}{ - "oneTrustCookieCategory": "foo", + "consentManagement": map[string]interface{}{ + "provider": "oneTrust", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, }, }, }, }, }, - expectedDestIDs: []string{"destID-1"}, }, }, + expectedDestIDs: []string{"destID-1"}, }, { - description: "filter out destination with oneTrustCookieCategories", + description: "should not filter any destination when consent IDs are not present in the event", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "provider": "oneTrust", + "resolutionStrategy": "all", + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "oneTrust", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, + }, + }, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1"}, + }, + { + description: "should not filter any destination when consent IDs are empty in the event", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "oneTrust", + "resolutionStrategy": "all", + "allowedConsentIds": []string{}, + "deniedConsentIds": []string{}, + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "oneTrust", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, + }, + }, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1"}, + }, + { + description: "should not filter any destination when consent info is malformed in the event", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "oneTrust", + "resolutionStrategy": "all", + "allowedConsentIds": "dummy", + "deniedConsentIds": "dummy", + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "oneTrust", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, + }, + }, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1"}, + }, + { + description: "should use oneTrustCookieCategories when provider is not specified in legacy SDKs", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -244,22 +353,127 @@ func TestFilterDestinations(t *testing.T) { }, }, }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-7", + Config: map[string]interface{}{ + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "", + }, + }, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-7"}, + }, + { + description: "should use oneTrustCookieCategories when GCM config is not available and provider in the event matches", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "oneTrust", + "resolutionStrategy": "or", // this should be ignored + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{}, + }, + }, + }, + { + ID: "destID-2", + Config: map[string]interface{}{ + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "foo-4", + }, + }, + }, + }, + { + ID: "destID-3", + Config: map[string]interface{}{ + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "foo-1", + }, + }, + }, + }, + { + ID: "destID-4", + Config: map[string]interface{}{ + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "foo-1", + }, + map[string]interface{}{ + "oneTrustCookieCategory": "foo-4", + }, + }, + }, + }, + { + ID: "destID-5", + Config: map[string]interface{}{ + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "foo-1", + }, + map[string]interface{}{ + "oneTrustCookieCategory": "foo-2", + }, + map[string]interface{}{ + "oneTrustCookieCategory": "foo-3", + }, + }, + }, + }, + { + ID: "destID-6", + Config: map[string]interface{}{ + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "foo-1", + }, + map[string]interface{}{ + "oneTrustCookieCategory": "foo-1", + }, + map[string]interface{}{ + "oneTrustCookieCategory": "foo-1", + }, + }, + }, + }, }, - expectedDestIDs: []string{"destID-1", "destID-2"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-2"}, }, { - description: "filter out destination with oneTrustCookieCategories with event containing provider details", + description: "should use oneTrustCookieCategories when GCM config is not available and provider in the event does not exist", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ - "provider": "oneTrust", // this should be ignored - "resolutionStrategy": "and", // this should be ignored - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "resolutionStrategy": "or", // this should be ignored + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -338,12 +552,12 @@ func TestFilterDestinations(t *testing.T) { }, }, }, - expectedDestIDs: []string{"destID-1", "destID-2"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-2"}, }, { - description: "filter out destination with ketchConsentPurposes", + description: "should use ketchConsentPurposes when provider is not specified in legacy SDKs", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ @@ -351,6 +565,7 @@ func TestFilterDestinations(t *testing.T) { }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -461,22 +676,34 @@ func TestFilterDestinations(t *testing.T) { }, }, }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-7", + Config: map[string]interface{}{ + "ketchConsentPurposes": []interface{}{ + map[string]interface{}{ + "purpose": "", + }, + }, + }, + }, }, - expectedDestIDs: []string{"destID-1", "destID-2", "destID-4"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-4", "destID-7"}, }, { - description: "filter out destination with ketchConsentPurposes with event containing provider details", + description: "should use ketchConsentPurposes when GCM config is not available and provider in the event matches", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ - "provider": "ketch", // this should be ignored - "resolutionStrategy": "or", // this should be ignored - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "provider": "ketch", + "resolutionStrategy": "or", // this should be ignored + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -555,21 +782,21 @@ func TestFilterDestinations(t *testing.T) { }, }, }, - expectedDestIDs: []string{"destID-1", "destID-2", "destID-4"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-4"}, }, { - description: "filter out destination with generic consent management (Ketch)", + description: "should use ketchConsentPurposes when GCM config is not available and provider in the event does not exist", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ - "provider": "ketch", - "resolutionStrategy": "or", - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "resolutionStrategy": "or", // this should be ignored + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -577,25 +804,17 @@ func TestFilterDestinations(t *testing.T) { { ID: "destID-1", Config: map[string]interface{}{ - "consentManagement": []interface{}{ - map[string]interface{}{ - "provider": "ketch", - "consents": []map[string]interface{}{ - {}, - }, - }, + "ketchConsentPurposes": []interface{}{ + map[string]interface{}{}, }, }, }, { ID: "destID-2", Config: map[string]interface{}{ - "consentManagement": []interface{}{ + "ketchConsentPurposes": []interface{}{ map[string]interface{}{ - "provider": "ketch", - "consents": []map[string]interface{}{ - {"consent": "foo-4"}, - }, + "purpose": "foo-4", }, }, }, @@ -603,12 +822,9 @@ func TestFilterDestinations(t *testing.T) { { ID: "destID-3", Config: map[string]interface{}{ - "consentManagement": []interface{}{ + "ketchConsentPurposes": []interface{}{ map[string]interface{}{ - "provider": "ketch", - "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - }, + "purpose": "foo-1", }, }, }, @@ -616,13 +832,12 @@ func TestFilterDestinations(t *testing.T) { { ID: "destID-4", Config: map[string]interface{}{ - "consentManagement": []interface{}{ + "ketchConsentPurposes": []interface{}{ map[string]interface{}{ - "provider": "ketch", - "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - {"consent": "foo-4"}, - }, + "purpose": "foo-1", + }, + map[string]interface{}{ + "purpose": "foo-4", }, }, }, @@ -630,14 +845,15 @@ func TestFilterDestinations(t *testing.T) { { ID: "destID-5", Config: map[string]interface{}{ - "consentManagement": []interface{}{ + "ketchConsentPurposes": []interface{}{ map[string]interface{}{ - "provider": "ketch", - "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - {"consent": "foo-2"}, - {"consent": "foo-3"}, - }, + "purpose": "foo-1", + }, + map[string]interface{}{ + "purpose": "foo-2", + }, + map[string]interface{}{ + "purpose": "foo-3", }, }, }, @@ -645,20 +861,132 @@ func TestFilterDestinations(t *testing.T) { { ID: "destID-6", Config: map[string]interface{}{ - "consentManagement": []interface{}{ + "ketchConsentPurposes": []interface{}{ map[string]interface{}{ - "provider": "ketch", - "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - {"consent": "foo-1"}, - {"consent": "foo-1"}, + "purpose": "foo-1", + }, + map[string]interface{}{ + "purpose": "foo-1", + }, + map[string]interface{}{ + "purpose": "foo-1", + }, + }, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-4"}, + }, + { + description: "should use generic consent management config for Ketch", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "ketch", + "resolutionStrategy": "or", + "allowedConsentIds": []string{"foo-4"}, + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "ketch", + "consents": []map[string]interface{}{ + {}, + }, + }, + }, + }, + }, + { + ID: "destID-2", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "ketch", + "consents": []map[string]interface{}{ + {"consent": "foo-4"}, + }, + }, + }, + }, + }, + { + ID: "destID-3", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "ketch", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + }, + }, + }, + }, + }, + { + ID: "destID-4", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "ketch", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + {"consent": "foo-4"}, + }, + }, + }, + }, + }, + { + ID: "destID-5", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "ketch", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + {"consent": "foo-2"}, + {"consent": "foo-3"}, + }, + }, + }, + }, + }, + { + ID: "destID-6", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "ketch", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + {"consent": "foo-1"}, + {"consent": "foo-1"}, }, }, }, }, }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-7", + Config: map[string]interface{}{ + "consentManagement": []interface{}{}, + }, + }, }, - expectedDestIDs: []string{"destID-1", "destID-2", "destID-4"}, }, // Some destinations are connected to different source with different consent management info { @@ -719,21 +1047,23 @@ func TestFilterDestinations(t *testing.T) { }, }, }, - expectedDestIDs: []string{"destID-1", "destID-2", "destID-4", "destID-5"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-4", "destID-7"}, }, { - description: "filter out destination when generic consent management is unavailable and falls back to legacy consents (Ketch)", + description: "should filter out destination when generic consent management is unavailable and falls back to legacy consents for Ketch", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ "provider": "ketch", "resolutionStrategy": "or", - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "allowedConsentIds": []string{"foo-4"}, + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -816,22 +1146,35 @@ func TestFilterDestinations(t *testing.T) { }, }, }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-6", + Config: map[string]interface{}{ + "ketchConsentPurposes": []interface{}{ + map[string]interface{}{ + "purpose": "", + }, + }, + }, + }, }, - expectedDestIDs: []string{"destID-2", "destID-3", "destID-5"}, }, }, + expectedDestIDs: []string{"destID-2", "destID-3", "destID-5", "destID-6"}, }, { - description: "filter out destination with generic consent management (OneTrust)", + description: "should filter out destination with generic consent management for OneTrust", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ "provider": "oneTrust", "resolutionStrategy": "and", - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "allowedConsentIds": []string{"foo-4"}, + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -919,8 +1262,14 @@ func TestFilterDestinations(t *testing.T) { }, }, }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-7", + Config: map[string]interface{}{ + "consentManagement": []interface{}{}, + }, + }, }, - expectedDestIDs: []string{"destID-1", "destID-2"}, }, // Some destinations are connected to different source with different consent management info { @@ -981,21 +1330,23 @@ func TestFilterDestinations(t *testing.T) { }, }, }, - expectedDestIDs: []string{"destID-1", "destID-2", "destID-4", "destID-5"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-7"}, }, { - description: "filter out destination when generic consent management is unavailable and falls back to legacy consents (OneTrust)", + description: "should filter out destination when generic consent management is unavailable and falls back to legacy consents for OneTrust", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ "provider": "oneTrust", "resolutionStrategy": "and", - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "allowedConsentIds": []string{"foo-4"}, + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -1057,42 +1408,572 @@ func TestFilterDestinations(t *testing.T) { }, }, }, - // Destination with GCM (not for OneTrust) and oneTrustCookieCategories but oneTrustCookieCategories will be preferred + // Destination with GCM (not for OneTrust) and oneTrustCookieCategories but oneTrustCookieCategories will be preferred + { + ID: "destID-5", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, + }, + }, + }, + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "foo-5", + }, + }, + }, + }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-6", + Config: map[string]interface{}{ + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "", + }, + }, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-2", "destID-3", "destID-5", "destID-6"}, + }, + { + description: "should filter out destination with generic consent management for Custom with AND strategy", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "or", // this will be ignored and the value from the destination config will be used + "allowedConsentIds": []string{"foo-4"}, + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "and", + "consents": []map[string]interface{}{ + {}, + }, + }, + }, + }, + }, + { + ID: "destID-2", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "and", + "consents": []map[string]interface{}{ + {"consent": "foo-4"}, + }, + }, + }, + }, + }, + { + ID: "destID-3", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "and", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + }, + }, + }, + }, + }, + { + ID: "destID-4", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "and", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + {"consent": "foo-4"}, + }, + }, + }, + }, + }, + { + ID: "destID-5", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "and", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + {"consent": "foo-2"}, + {"consent": "foo-3"}, + }, + }, + }, + }, + }, + { + ID: "destID-6", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "and", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + {"consent": "foo-1"}, + {"consent": "foo-1"}, + }, + }, + }, + }, + }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-7", + Config: map[string]interface{}{ + "consentManagement": []interface{}{}, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-7"}, + }, + { + description: "should filter out destination with generic consent management for Custom with OR strategy", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "and", // this will be ignored and the value from the destination config will be used + "allowedConsentIds": []string{"foo-4"}, + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "or", + "consents": []map[string]interface{}{ + {}, + }, + }, + }, + }, + }, + { + ID: "destID-2", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "or", + "consents": []map[string]interface{}{ + {"consent": "foo-4"}, + }, + }, + }, + }, + }, + { + ID: "destID-3", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "or", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + }, + }, + }, + }, + }, + { + ID: "destID-4", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "or", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + {"consent": "foo-4"}, + }, + }, + }, + }, + }, + { + ID: "destID-5", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "or", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + {"consent": "foo-2"}, + {"consent": "foo-3"}, + }, + }, + }, + }, + }, + { + ID: "destID-6", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "or", + "consents": []map[string]interface{}{ + {"consent": "foo-1"}, + {"consent": "foo-1"}, + {"consent": "foo-1"}, + }, + }, + }, + }, + }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-7", + Config: map[string]interface{}{ + "consentManagement": []interface{}{}, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-4", "destID-7"}, + }, + { + description: "should filter out destinations when generic consent management for Custom is unavailable but does not fall back to legacy consents", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "or", + "allowedConsentIds": []string{"foo-4"}, + "deniedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "ketch", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, + }, + }, + }, + }, + }, + // Destination with ketchConsentPurposes but it'll not be used + { + ID: "destID-2", + Config: map[string]interface{}{ + "ketchConsentPurposes": []interface{}{ + map[string]interface{}{ + "purpose": "foo-1", + }, + }, + }, + }, + // Destination with oneTrustCookieCategories but it'll not be used + { + ID: "destID-3", + Config: map[string]interface{}{ + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "foo-1", + }, + }, + }, + }, + // Destination with ketchConsentPurposes and oneTrustCookieCategories but it'll not be used + { + ID: "destID-4", + Config: map[string]interface{}{ + "ketchConsentPurposes": []interface{}{ + map[string]interface{}{ + "purpose": "foo-1", + }, + }, + "oneTrustCookieCategories": []interface{}{ + map[string]interface{}{ + "oneTrustCookieCategory": "foo-1", + }, + }, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-3", "destID-4"}, + }, + { + description: "should treat resolution strategy 'any' as 'or' for custom provider", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "custom", + "allowedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, + "deniedConsentIds": []string{"foo-4"}, + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "any", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, + }, + }, + }, + }, + }, + { + ID: "destID-2", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "any", + "consents": []map[string]interface{}{ + { + "consent": "foo-2", + }, + { + "consent": "foo-3", + }, + }, + }, + }, + }, + }, + { + ID: "destID-3", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "any", + "consents": []map[string]interface{}{ + { + "consent": "foo-3", + }, + { + "consent": "foo-4", + }, + }, + }, + }, + }, + }, + { + ID: "destID-4", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "any", + "consents": []map[string]interface{}{ + { + "consent": "foo-4", + }, + }, + }, + }, + }, + }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-5", + Config: map[string]interface{}{ + "consentManagement": []interface{}{}, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-3", "destID-5"}, + }, + { + description: "should treat resolution strategy 'all' as 'and' for custom provider", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "custom", + "allowedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, + "deniedConsentIds": []string{"foo-4"}, + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "all", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, + { + "consent": "foo-2", + }, + { + "consent": "foo-3", + }, + }, + }, + }, + }, + }, + { + ID: "destID-2", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "all", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, + { + "consent": "foo-2", + }, + { + "consent": "foo-4", + }, + }, + }, + }, + }, + }, + { + ID: "destID-3", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "custom", + "resolutionStrategy": "all", + "consents": []map[string]interface{}{ + { + "consent": "foo-4", + }, + }, + }, + }, + }, + }, { - ID: "destID-5", + ID: "destID-4", Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", + "provider": "custom", + "resolutionStrategy": "all", "consents": []map[string]interface{}{ { "consent": "foo-1", }, + { + "consent": "foo-2", + }, }, }, }, - "oneTrustCookieCategories": []interface{}{ - map[string]interface{}{ - "oneTrustCookieCategory": "foo-5", - }, - }, + }, + }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-5", + Config: map[string]interface{}{ + "consentManagement": []interface{}{}, }, }, }, - expectedDestIDs: []string{"destID-2", "destID-3", "destID-5"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-4", "destID-5"}, }, { - description: "filter out destination with generic consent management (Custom - AND)", + description: "should treat resolution strategy 'any' as 'or' for ketch provider", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ - "provider": "custom", - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "provider": "ketch", + "resolutionStrategy": "any", + "allowedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, + "deniedConsentIds": []string{"foo-4"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -1102,10 +1983,11 @@ func TestFilterDestinations(t *testing.T) { Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "and", + "provider": "ketch", "consents": []map[string]interface{}{ - {}, + { + "consent": "foo-1", + }, }, }, }, @@ -1116,10 +1998,14 @@ func TestFilterDestinations(t *testing.T) { Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "and", + "provider": "ketch", "consents": []map[string]interface{}{ - {"consent": "foo-4"}, + { + "consent": "foo-2", + }, + { + "consent": "foo-3", + }, }, }, }, @@ -1130,10 +2016,14 @@ func TestFilterDestinations(t *testing.T) { Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "and", + "provider": "ketch", "consents": []map[string]interface{}{ - {"consent": "foo-1"}, + { + "consent": "foo-3", + }, + { + "consent": "foo-4", + }, }, }, }, @@ -1144,64 +2034,41 @@ func TestFilterDestinations(t *testing.T) { Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "and", + "provider": "ketch", "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - {"consent": "foo-4"}, + { + "consent": "foo-4", + }, }, }, }, }, }, + // empty consents. Consent management is practically not configured for this destination. { ID: "destID-5", Config: map[string]interface{}{ - "consentManagement": []interface{}{ - map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "and", - "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - {"consent": "foo-2"}, - {"consent": "foo-3"}, - }, - }, - }, - }, - }, - { - ID: "destID-6", - Config: map[string]interface{}{ - "consentManagement": []interface{}{ - map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "and", - "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - {"consent": "foo-1"}, - {"consent": "foo-1"}, - }, - }, - }, + "consentManagement": []interface{}{}, }, }, }, - expectedDestIDs: []string{"destID-1", "destID-2"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-3", "destID-5"}, }, { - description: "filter out destination with generic consent management (Custom - OR)", + description: "should treat resolution strategy 'all' as 'and' for oneTrust provider", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "or", - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "provider": "oneTrust", + "resolutionStrategy": "all", + "allowedConsentIds": []string{"foo-1", "foo-2", "foo-3"}, + "deniedConsentIds": []string{"foo-4"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -1211,10 +2078,17 @@ func TestFilterDestinations(t *testing.T) { Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "or", + "provider": "oneTrust", "consents": []map[string]interface{}{ - {}, + { + "consent": "foo-1", + }, + { + "consent": "foo-2", + }, + { + "consent": "foo-3", + }, }, }, }, @@ -1225,10 +2099,17 @@ func TestFilterDestinations(t *testing.T) { Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "or", + "provider": "oneTrust", "consents": []map[string]interface{}{ - {"consent": "foo-4"}, + { + "consent": "foo-1", + }, + { + "consent": "foo-2", + }, + { + "consent": "foo-4", + }, }, }, }, @@ -1239,10 +2120,11 @@ func TestFilterDestinations(t *testing.T) { Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "or", + "provider": "oneTrust", "consents": []map[string]interface{}{ - {"consent": "foo-1"}, + { + "consent": "foo-4", + }, }, }, }, @@ -1253,64 +2135,131 @@ func TestFilterDestinations(t *testing.T) { Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "or", + "provider": "oneTrust", "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - {"consent": "foo-4"}, + { + "consent": "foo-1", + }, + { + "consent": "foo-2", + }, }, }, }, }, }, + // empty consents. Consent management is practically not configured for this destination. { ID: "destID-5", + Config: map[string]interface{}{ + "consentManagement": []interface{}{}, + }, + }, + }, + }, + }, + expectedDestIDs: []string{"destID-1", "destID-4", "destID-5"}, + }, + { + description: "should use denied consent IDs from event when allowed consent IDs are not present (resolution strategy 'any')", + event: types.SingularEventT{ + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "provider": "ketch", + "resolutionStrategy": "any", + "deniedConsentIds": []string{"foo-4", "foo-5"}, + }, + }, + }, + sourceId: "sourceID-1", + connectionInfo: []ConnectionInfo{ + { + sourceId: "sourceID-1", + destinations: []backendconfig.DestinationT{ + { + ID: "destID-1", Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "or", + "provider": "ketch", "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - {"consent": "foo-2"}, - {"consent": "foo-3"}, + { + "consent": "foo-1", + }, + { + "consent": "foo-2", + }, + { + "consent": "foo-3", + }, }, }, }, }, }, { - ID: "destID-6", + ID: "destID-2", Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "or", + "provider": "ketch", "consents": []map[string]interface{}{ - {"consent": "foo-1"}, - {"consent": "foo-1"}, - {"consent": "foo-1"}, + { + "consent": "foo-1", + }, + { + "consent": "foo-2", + }, + { + "consent": "foo-4", + }, + }, + }, + }, + }, + }, + { + ID: "destID-3", + Config: map[string]interface{}{ + "consentManagement": []interface{}{ + map[string]interface{}{ + "provider": "ketch", + "consents": []map[string]interface{}{ + { + "consent": "foo-4", + }, + { + "consent": "foo-5", + }, }, }, }, }, }, + // empty consents. Consent management is practically not configured for this destination. + { + ID: "destID-4", + Config: map[string]interface{}{ + "consentManagement": []interface{}{}, + }, + }, }, - expectedDestIDs: []string{"destID-1", "destID-2", "destID-4"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-2", "destID-4"}, }, { - description: "filter out destination when generic consent management (Custom) is unavailable but doesn't falls back to legacy consents", + description: "should use denied consent IDs from event when allowed consent IDs are not present (resolution strategy 'all')", event: types.SingularEventT{ "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ - "provider": "custom", - "resolutionStrategy": "or", - "deniedConsentIds": []interface{}{"foo-1", "foo-2", "foo-3"}, + "provider": "oneTrust", + "resolutionStrategy": "all", + "deniedConsentIds": []string{"foo-4", "foo-5"}, }, }, }, + sourceId: "sourceID-1", connectionInfo: []ConnectionInfo{ { sourceId: "sourceID-1", @@ -1320,58 +2269,72 @@ func TestFilterDestinations(t *testing.T) { Config: map[string]interface{}{ "consentManagement": []interface{}{ map[string]interface{}{ - "provider": "ketch", + "provider": "oneTrust", "consents": []map[string]interface{}{ { "consent": "foo-1", }, + { + "consent": "foo-2", + }, + { + "consent": "foo-3", + }, }, }, }, }, }, - // Destination with ketchConsentPurposes but it'll not be used { ID: "destID-2", Config: map[string]interface{}{ - "ketchConsentPurposes": []interface{}{ + "consentManagement": []interface{}{ map[string]interface{}{ - "purpose": "foo-1", + "provider": "oneTrust", + "consents": []map[string]interface{}{ + { + "consent": "foo-1", + }, + { + "consent": "foo-2", + }, + { + "consent": "foo-4", + }, + }, }, }, }, }, - // Destination with oneTrustCookieCategories but it'll not be used { ID: "destID-3", Config: map[string]interface{}{ - "oneTrustCookieCategories": []interface{}{ + "consentManagement": []interface{}{ map[string]interface{}{ - "oneTrustCookieCategory": "foo-1", + "provider": "oneTrust", + "consents": []map[string]interface{}{ + { + "consent": "foo-4", + }, + { + "consent": "foo-5", + }, + }, }, }, }, }, - // Destination with ketchConsentPurposes and oneTrustCookieCategories but it'll not be used + // empty consents. Consent management is practically not configured for this destination. { ID: "destID-4", Config: map[string]interface{}{ - "ketchConsentPurposes": []interface{}{ - map[string]interface{}{ - "purpose": "foo-1", - }, - }, - "oneTrustCookieCategories": []interface{}{ - map[string]interface{}{ - "oneTrustCookieCategory": "foo-1", - }, - }, + "consentManagement": []interface{}{}, }, }, }, - expectedDestIDs: []string{"destID-1", "destID-2", "destID-3", "destID-4"}, }, }, + expectedDestIDs: []string{"destID-1", "destID-4"}, }, } @@ -1383,21 +2346,27 @@ func TestFilterDestinations(t *testing.T) { proc.config.genericConsentManagementMap = make(SourceConsentMap) proc.logger = logger.NewLogger().Child("processor") - for _, connectionInfo := range tc.connectionInfo { - proc.config.genericConsentManagementMap[SourceID(connectionInfo.sourceId)] = make(DestConsentMap) - for _, dest := range connectionInfo.destinations { + for _, connection := range tc.connectionInfo { + proc.config.genericConsentManagementMap[SourceID(connection.sourceId)] = make(DestConsentMap) + for _, dest := range connection.destinations { proc.config.oneTrustConsentCategoriesMap[dest.ID] = getOneTrustConsentCategories(&dest) proc.config.ketchConsentCategoriesMap[dest.ID] = getKetchConsentCategories(&dest) - proc.config.genericConsentManagementMap[SourceID(connectionInfo.sourceId)][DestinationID(dest.ID)], _ = getGenericConsentManagementData(&dest) + proc.config.genericConsentManagementMap[SourceID(connection.sourceId)][DestinationID(dest.ID)], _ = getGenericConsentManagementData(&dest) } } - for _, connectionInfo := range tc.connectionInfo { - filteredDestinations := proc.getConsentFilteredDestinations(tc.event, connectionInfo.sourceId, connectionInfo.destinations) - require.EqualValues(t, connectionInfo.expectedDestIDs, lo.Map(filteredDestinations, func(dest backendconfig.DestinationT, _ int) string { - return dest.ID - })) - } + // Find the connection based on the source ID we're interested in + connection, _ := lo.Find(tc.connectionInfo, func(connection ConnectionInfo) bool { + return connection.sourceId == tc.sourceId + }) + + // Get the filtered destinations based on the event and the connection info + filteredDestinations := proc.getConsentFilteredDestinations(tc.event, tc.sourceId, connection.destinations) + + // Assert that the filtered destinations are as expected + require.EqualValues(t, tc.expectedDestIDs, lo.Map(filteredDestinations, func(dest backendconfig.DestinationT, _ int) string { + return dest.ID + })) }) } } @@ -1437,7 +2406,7 @@ func TestGetConsentManagementInfo(t *testing.T) { expected: defConsentManagementInfo, }, { - description: "should return default values for denied consent IDs when it is missing in the event", + description: "should return default values for allowed and denied consent IDs when they are missing in the event", input: types.SingularEventT{ "anonymousId": "123", "type": "track", @@ -1448,25 +2417,18 @@ func TestGetConsentManagementInfo(t *testing.T) { "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ "provider": "custom", - "allowedConsentIds": []string{ - "consent category 1", - "consent category 2", - }, }, }, }, expected: ConsentManagementInfo{ - Provider: "custom", - AllowedConsentIDs: []interface{}{ - "consent category 1", - "consent category 2", - }, - DeniedConsentIDs: []string{}, + Provider: "custom", ResolutionStrategy: "", + AllowedConsentIDs: []string{}, + DeniedConsentIDs: []string{}, }, }, { - description: "should return default values for denied consent IDs when it is malformed", + description: "should return default values for allowed and denied consent IDs when they are malformed", input: types.SingularEventT{ "anonymousId": "123", "type": "track", @@ -1476,8 +2438,9 @@ func TestGetConsentManagementInfo(t *testing.T) { }, "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ - "provider": "custom", - "deniedConsentIds": "dummy", + "provider": "custom", + "deniedConsentIds": "dummy1", + "allowedConsentIds": "dummy2", }, }, }, @@ -1526,9 +2489,12 @@ func TestGetConsentManagementInfo(t *testing.T) { }, "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ - "deniedConsentIds": map[string]interface{}{ // this should have been an array + "allowedConsentIds": map[string]interface{}{ // this should have been an array "consent": "consent category 1", }, + "deniedConsentIds": map[string]interface{}{ // this should have been an array + "consent": "consent category 2", + }, }, }, }, @@ -1546,9 +2512,9 @@ func TestGetConsentManagementInfo(t *testing.T) { "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ "provider": "custom", - "allowedConsentIds": map[string]interface{}{ - "C0001": "consent category 1", - "C0002": "consent category 2", + "allowedConsentIds": []string{ + "consent category 1", + "consent category 2", }, "deniedConsentIds": []string{ "consent category 3", @@ -1563,9 +2529,9 @@ func TestGetConsentManagementInfo(t *testing.T) { }, expected: ConsentManagementInfo{ Provider: "custom", - AllowedConsentIDs: map[string]interface{}{ - "C0001": "consent category 1", - "C0002": "consent category 2", + AllowedConsentIDs: []string{ + "consent category 1", + "consent category 2", }, DeniedConsentIDs: []string{ "consent category 3", @@ -1595,7 +2561,8 @@ func TestGetConsentManagementInfo(t *testing.T) { }, }, expected: ConsentManagementInfo{ - Provider: "", + Provider: "", + AllowedConsentIDs: []string{}, DeniedConsentIDs: []string{ "consent category 3", "consent category 4", @@ -1681,10 +2648,10 @@ func TestGetGenericConsentManagementData(t *testing.T) { "provider": "ketch", "consents": []map[string]interface{}{ { - "consent": "purpose 1", + "consent": " purpose 1 ", }, { - "consent": "", + "consent": " ", }, { "consent": "purpose 2", @@ -1695,13 +2662,13 @@ func TestGetGenericConsentManagementData(t *testing.T) { "provider": "custom", "consents": []map[string]interface{}{ { - "consent": "custom consent 1", + "consent": "custom consent 1 ", }, { "consent": "", }, { - "consent": "custom consent 2", + "consent": " custom consent 2", }, { "consent": "", @@ -1768,7 +2735,7 @@ func TestGetGenericConsentManagementData(t *testing.T) { "consent": "", }, { - "consent": "", + "consent": " ", }, }, "resolutionStrategy": "or", From a3c9313851079ba71951dd081ea68995513a1b35 Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju Date: Wed, 30 Apr 2025 21:24:49 +0530 Subject: [PATCH 2/7] fix: misc issues and handle backward compatibility --- processor/consent.go | 53 ++++++++++++++++++++++++++++++------- processor/consent_test.go | 45 ++++++++++++++++++++++++++++--- processor/processor_test.go | 2 +- 3 files changed, 86 insertions(+), 14 deletions(-) diff --git a/processor/consent.go b/processor/consent.go index 8b145340d7..8e8d570a61 100644 --- a/processor/consent.go +++ b/processor/consent.go @@ -13,10 +13,17 @@ import ( ) type ConsentManagementInfo struct { - AllowedConsentIDs []string `json:"allowedConsentIds"` - DeniedConsentIDs []string `json:"deniedConsentIds"` - Provider string `json:"provider"` - ResolutionStrategy string `json:"resolutionStrategy"` + AllowedConsentIDs []string + DeniedConsentIDs []string + Provider string + ResolutionStrategy string +} + +type EventConsentManagementInfo struct { + AllowedConsentIDs interface{} `json:"allowedConsentIds"` + DeniedConsentIDs []string `json:"deniedConsentIds"` + Provider string `json:"provider"` + ResolutionStrategy string `json:"resolutionStrategy"` } type GenericConsentManagementProviderData struct { @@ -220,6 +227,7 @@ func getGenericConsentManagementData(dest *backendconfig.DestinationT) (ConsentP } func getConsentManagementInfo(event types.SingularEventT) (ConsentManagementInfo, error) { + consentManagementInfoFromEvent := EventConsentManagementInfo{} consentManagementInfo := ConsentManagementInfo{} if consentManagement, ok := misc.MapLookup(event, "context", "consentManagement").(map[string]interface{}); ok { consentManagementObjBytes, mErr := jsonrs.Marshal(consentManagement) @@ -227,21 +235,48 @@ func getConsentManagementInfo(event types.SingularEventT) (ConsentManagementInfo return consentManagementInfo, fmt.Errorf("error marshalling consentManagement: %v", mErr) } - unmErr := jsonrs.Unmarshal(consentManagementObjBytes, &consentManagementInfo) + unmErr := jsonrs.Unmarshal(consentManagementObjBytes, &consentManagementInfoFromEvent) if unmErr != nil { return consentManagementInfo, fmt.Errorf("error unmarshalling consentManagementInfo: %v", unmErr) } + consentManagementInfo.DeniedConsentIDs = consentManagementInfoFromEvent.DeniedConsentIDs + consentManagementInfo.Provider = consentManagementInfoFromEvent.Provider + consentManagementInfo.ResolutionStrategy = consentManagementInfoFromEvent.ResolutionStrategy + + // This is to support really old version of the JS SDK v3 that sent this data as an object + // for OneTrust provider. + // Handle AllowedConsentIDs based on its type (array or map) + switch val := consentManagementInfoFromEvent.AllowedConsentIDs.(type) { + case []interface{}: + // Convert []interface{} to []string + consentManagementInfo.AllowedConsentIDs = make([]string, 0, len(val)) + for _, v := range val { + if strVal, ok := v.(string); ok { + consentManagementInfo.AllowedConsentIDs = append(consentManagementInfo.AllowedConsentIDs, strVal) + } + } + case []string: + // Already a string array + consentManagementInfo.AllowedConsentIDs = val + case map[string]interface{}: + // Use keys from the map (legacy OneTrust format) + consentManagementInfo.AllowedConsentIDs = lo.Keys(val) + default: + consentManagementInfo.AllowedConsentIDs = []string{} + } + // Ideally, the clean up and filter is not needed for standard providers // but useful for custom providers where users send this data directly // to the SDKs. - cleanupPredicate := func(consent string, _ int) (string, bool) { - return strings.TrimSpace(consent), consent != "" + sanitizePredicate := func(consent string, _ int) string { + return strings.TrimSpace(consent) } - consentManagementInfo.AllowedConsentIDs = lo.FilterMap(consentManagementInfo.AllowedConsentIDs, cleanupPredicate) - consentManagementInfo.DeniedConsentIDs = lo.FilterMap(consentManagementInfo.DeniedConsentIDs, cleanupPredicate) + consentManagementInfo.AllowedConsentIDs = lo.Map(consentManagementInfo.AllowedConsentIDs, sanitizePredicate) + consentManagementInfo.DeniedConsentIDs = lo.Map(consentManagementInfo.DeniedConsentIDs, sanitizePredicate) + // Filter out empty values filterPredicate := func(consent string, _ int) (string, bool) { return consent, consent != "" } diff --git a/processor/consent_test.go b/processor/consent_test.go index 97aea71e13..5f0b1585a8 100644 --- a/processor/consent_test.go +++ b/processor/consent_test.go @@ -2517,7 +2517,7 @@ func TestGetConsentManagementInfo(t *testing.T) { "consent category 2", }, "deniedConsentIds": []string{ - "consent category 3", + " consent category 3 ", "", "consent category 4", "", @@ -2552,9 +2552,9 @@ func TestGetConsentManagementInfo(t *testing.T) { "context": map[string]interface{}{ "consentManagement": map[string]interface{}{ "deniedConsentIds": []string{ - "consent category 3", - "", - "consent category 4", + "consent category 3 ", + " ", + " consent category 4", "", }, }, @@ -2570,6 +2570,43 @@ func TestGetConsentManagementInfo(t *testing.T) { ResolutionStrategy: "", }, }, + { + description: "should return consent management info when consent management data is sent from older SDKs with allowed consent IDs as an object", + input: types.SingularEventT{ + "anonymousId": "123", + "type": "track", + "event": "test", + "properties": map[string]interface{}{ + "category": "test", + }, + "context": map[string]interface{}{ + "consentManagement": map[string]interface{}{ + "allowedConsentIds": map[string]interface{}{ + "C0": "consent category 1", + "C1": "consent category 2", + }, + "deniedConsentIds": []string{ + "consent category 3 ", + " ", + " consent category 4", + "", + }, + }, + }, + }, + expected: ConsentManagementInfo{ + Provider: "", + AllowedConsentIDs: []string{ + "C0", + "C1", + }, + DeniedConsentIDs: []string{ + "consent category 3", + "consent category 4", + }, + ResolutionStrategy: "", + }, + }, } for _, testCase := range testCases { diff --git a/processor/processor_test.go b/processor/processor_test.go index 9ba7026cfa..e1abe5eb73 100644 --- a/processor/processor_test.go +++ b/processor/processor_test.go @@ -3653,7 +3653,7 @@ var _ = Describe("Processor", Ordered, func() { "destination-definition-name-enabled", ), )), - ).To(Equal(8)) // all except D13 + ).To(Equal(6)) // all except D6, D13, and D14 Expect(processor.isDestinationAvailable(eventWithDeniedConsentsGCM, SourceIDGCM, "")).To(BeTrue()) Expect( From 5d8f22dd3d161707d214c4b98841f5246ca3755d Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju Date: Wed, 30 Apr 2025 21:39:15 +0530 Subject: [PATCH 3/7] test: fix test assertion value --- processor/processor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processor/processor_test.go b/processor/processor_test.go index e1abe5eb73..9ccbefe486 100644 --- a/processor/processor_test.go +++ b/processor/processor_test.go @@ -3665,7 +3665,7 @@ var _ = Describe("Processor", Ordered, func() { "destination-definition-name-enabled", ), )), - ).To(Equal(7)) // all except D6 and D7 + ).To(Equal(6)) // all except D6 and D7 Expect(processor.isDestinationAvailable(eventWithDeniedConsentsGCMKetch, SourceIDGCM, "")).To(BeTrue()) Expect( From a91bddb1a30d698deef0ef8834397833d01cf131 Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju Date: Wed, 30 Apr 2025 22:02:56 +0530 Subject: [PATCH 4/7] test: sort the object keys to prevent intermittent failures --- processor/consent.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/processor/consent.go b/processor/consent.go index 8e8d570a61..5e24b25a85 100644 --- a/processor/consent.go +++ b/processor/consent.go @@ -2,6 +2,7 @@ package processor import ( "fmt" + "sort" "strings" "github.com/samber/lo" @@ -261,7 +262,9 @@ func getConsentManagementInfo(event types.SingularEventT) (ConsentManagementInfo consentManagementInfo.AllowedConsentIDs = val case map[string]interface{}: // Use keys from the map (legacy OneTrust format) + // Also, sort the keys consentManagementInfo.AllowedConsentIDs = lo.Keys(val) + sort.Strings(consentManagementInfo.AllowedConsentIDs) default: consentManagementInfo.AllowedConsentIDs = []string{} } From 87d495ff137ef8aa6ccc2d5caefd22a6ca64b9db Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju Date: Thu, 1 May 2025 10:36:08 +0530 Subject: [PATCH 5/7] test: fix test assertion value --- processor/processor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processor/processor_test.go b/processor/processor_test.go index 9ccbefe486..b45c9aa053 100644 --- a/processor/processor_test.go +++ b/processor/processor_test.go @@ -3677,7 +3677,7 @@ var _ = Describe("Processor", Ordered, func() { "destination-definition-name-enabled", ), )), - ).To(Equal(8)) // all except D7 + ).To(Equal(6)) // all except D7 // some unknown destination ID is passed destination will be unavailable Expect(processor.isDestinationAvailable(eventWithDeniedConsentsGCMKetch, SourceIDGCM, "unknown-destination")).To(BeFalse()) From 03a5b94d3c909759f05e3d859e631fa002bc276c Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju Date: Thu, 1 May 2025 16:54:09 +0530 Subject: [PATCH 6/7] chore: clarify using inline documentation --- processor/consent.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processor/consent.go b/processor/consent.go index 5e24b25a85..46f0f55acf 100644 --- a/processor/consent.go +++ b/processor/consent.go @@ -262,7 +262,8 @@ func getConsentManagementInfo(event types.SingularEventT) (ConsentManagementInfo consentManagementInfo.AllowedConsentIDs = val case map[string]interface{}: // Use keys from the map (legacy OneTrust format) - // Also, sort the keys + // Also, sort the keys as the unmarshalling of map is not deterministic of the order of keys + // and it can cause flaky tests. consentManagementInfo.AllowedConsentIDs = lo.Keys(val) sort.Strings(consentManagementInfo.AllowedConsentIDs) default: From 675fb737cf13414d02dd91e0a767995421d9286b Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju Date: Thu, 8 May 2025 14:07:55 +0530 Subject: [PATCH 7/7] feat: emit a stat when source sends resoultion strategy in legacy format --- processor/consent.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/processor/consent.go b/processor/consent.go index 46f0f55acf..f6d9fad974 100644 --- a/processor/consent.go +++ b/processor/consent.go @@ -7,6 +7,7 @@ import ( "github.com/samber/lo" + "github.com/rudderlabs/rudder-go-kit/stats" backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jsonrs" "github.com/rudderlabs/rudder-server/processor/types" @@ -66,6 +67,15 @@ func (proc *Handle) getConsentFilteredDestinations(event types.SingularEventT, s if cmpData := proc.getGCMData(sourceID, dest.ID, consentManagementInfo.Provider); len(cmpData.Consents) > 0 { finalResolutionStrategy := consentManagementInfo.ResolutionStrategy + // Emit a stat when the source sent resolution strategy is "or" or "and" in the event payload + // For custom provider, the resolution strategy is to be picked from the destination config + // Config backend still serves the value in the older format for backward compatibility + // Once we realize no SDKs are sending the data in this format, we have to update the config backend + // and also remove this stat. + if finalResolutionStrategy == "or" || finalResolutionStrategy == "and" { + proc.statsFactory.NewTaggedStat("processor_legacy_consent_resolution_strategy_events_count", stats.CountType, stats.Tags{"sourceId": sourceID}).Count(1) + } + // For custom provider, the resolution strategy is to be picked from the destination config if consentManagementInfo.Provider == "custom" { finalResolutionStrategy = cmpData.ResolutionStrategy