Merge pull request #7 from eko/filter-stats-logs

Added ability to filter audit logs and stats with a regexp

backend/README.md

@@ -29,9 +29,11 @@ Here are the available configuration options available as environment variable:
 | APP_AUDIT_CLEAN_DAYS_TO_KEEP | `7` | Audit logs number of days to keep in database |
 | APP_AUDIT_CLEAN_DELAY | `1h` | Audit logs clean delay |
 | APP_AUDIT_FLUSH_DELAY | `3s` | Delay in which audit logs will be batch into database |
+| APP_AUDIT_RESOURCE_KIND_REGEX | `.*` | Filter which resource kind will be added on audit logs |
 | APP_STATS_CLEAN_DAYS_TO_KEEP | `30` | Statistics number of days to keep in database |
 | APP_STATS_CLEAN_DELAY | `1h` | Statistics clean delay |
 | APP_STATS_FLUSH_DELAY | `3s` | Delay in which statistics will be batch into database |
+| APP_STATS_RESOURCE_KIND_REGEX | `.*` | Filter which resource kind will be added on statistics |
 | AUTH_ACCESS_TOKEN_DURATION | `6h` | Access token duration  |
 | AUTH_DOMAIN | `http://localhost:8080` | OAuth domain to be used  |
 | AUTH_JWT_SIGN_STRING | `4uthz-s3cr3t-valu3-pl3as3-ch4ng3!` | Default HMAC to use for JWT tokens |

backend/configs/app.go

@@ -6,20 +6,24 @@ type App struct {
 	AuditCleanDelay            time.Duration `config:"app_audit_clean_delay"`
 	AuditCleanDaysToKeep       int           `config:"app_audit_clean_days_to_keep"`
 	AuditFlushDelay            time.Duration `config:"app_audit_flush_delay"`
+	AuditResourceKindRegex     string        `config:"app_audit_resource_kind_regex"`
 	DispatcherEventChannelSize int           `config:"dispatcher_event_channel_size"`
 	StatsCleanDelay            time.Duration `config:"app_stats_clean_delay"`
 	StatsCleanDaysToKeep       int           `config:"app_stats_clean_days_to_keep"`
 	StatsFlushDelay            time.Duration `config:"app_stats_flush_delay"`
+	StatsResourceKindRegex     string        `config:"app_stats_resource_kind_regex"`
 }
 
 func newApp() *App {
 	return &App{
 		AuditCleanDelay:            1 * time.Hour,
 		AuditCleanDaysToKeep:       7,
 		AuditFlushDelay:            3 * time.Second,
+		AuditResourceKindRegex:     `.*`,
 		DispatcherEventChannelSize: 10000,
 		StatsCleanDelay:            1 * time.Hour,
 		StatsCleanDaysToKeep:       30,
 		StatsFlushDelay:            3 * time.Second,
+		StatsResourceKindRegex:     `.*`,
 	}
 }

backend/internal/audit/subscriber.go

@@ -2,6 +2,7 @@ package audit
 
 import (
 	"context"
+	"regexp"
 	"time"
 
 	"github.com/eko/authz/backend/configs"
@@ -14,10 +15,11 @@ import (
 )
 
 type subscriber struct {
-	logger          *slog.Logger
-	dispatcher      event.Dispatcher
-	auditManager    manager.Audit
-	auditFlushDelay time.Duration
+	logger            *slog.Logger
+	dispatcher        event.Dispatcher
+	auditManager      manager.Audit
+	flushDelay        time.Duration
+	resourceKindRegex *regexp.Regexp
 }
 
 func NewSubscriber(
@@ -27,10 +29,11 @@ func NewSubscriber(
 	auditManager manager.Audit,
 ) *subscriber {
 	return &subscriber{
-		logger:          logger,
-		dispatcher:      dispatcher,
-		auditManager:    auditManager,
-		auditFlushDelay: cfg.AuditFlushDelay,
+		logger:            logger,
+		dispatcher:        dispatcher,
+		auditManager:      auditManager,
+		flushDelay:        cfg.AuditFlushDelay,
+		resourceKindRegex: regexp.MustCompile(cfg.AuditResourceKindRegex),
 	}
 }
 
@@ -81,8 +84,8 @@ func (s *subscriber) handleCheckEvents(eventChan chan *event.Event) {
 				IsAllowed:     checkEvent.IsAllowed,
 			}
 
-			if checkEvent.CompiledPilicy != nil {
-				audit.PolicyID = checkEvent.CompiledPilicy.PolicyID
+			if checkEvent.CompiledPolicy != nil {
+				audit.PolicyID = checkEvent.CompiledPolicy.PolicyID
 			}
 
 			audits = append(audits, audit)
@@ -91,10 +94,17 @@ func (s *subscriber) handleCheckEvents(eventChan chan *event.Event) {
 		if err := s.auditManager.BatchAdd(audits); err != nil {
 			s.logger.Error("Audit: unable to batch add audit events", err)
 		}
-	}, spooler.WithFlushInterval(s.auditFlushDelay))
+	}, spooler.WithFlushInterval(s.flushDelay))
 
-	for event := range eventChan {
-		spooler.Add(event)
+	for eventItem := range eventChan {
+		checkEvent, ok := eventItem.Data.(*event.CheckEvent)
+		if !ok {
+			continue
+		}
+
+		if s.resourceKindRegex.MatchString(checkEvent.ResourceKind) {
+			spooler.Add(eventItem)
+		}
 	}
 }
 

backend/internal/audit/subscriber_test.go

@@ -1,6 +1,7 @@
 package audit
 
 import (
+	"regexp"
 	"testing"
 	"time"
 
@@ -38,7 +39,8 @@ func TestNewSubscriber(t *testing.T) {
 	assert.Equal(logger, subscriberInstance.logger)
 	assert.Equal(dispatcher, subscriberInstance.dispatcher)
 	assert.Equal(auditManager, subscriberInstance.auditManager)
-	assert.Equal(cfg.AuditFlushDelay, subscriberInstance.auditFlushDelay)
+	assert.Equal(cfg.AuditFlushDelay, subscriberInstance.flushDelay)
+	assert.Equal(regexp.MustCompile(cfg.AuditResourceKindRegex), subscriberInstance.resourceKindRegex)
 }
 
 func TestHandleCheckEvents(t *testing.T) {
@@ -81,3 +83,53 @@ func TestHandleCheckEvents(t *testing.T) {
 	// Wait 20ms to ensure the spool is triggered.
 	<-time.After(20 * time.Millisecond)
 }
+
+func TestHandleCheckEvents_WithResourceKindRegexp(t *testing.T) {
+	// Given
+	ctrl := gomock.NewController(t)
+
+	cfg := &configs.App{
+		AuditFlushDelay:        10 * time.Millisecond,
+		AuditResourceKindRegex: "^post.*",
+	}
+
+	logger := slog.New(log.NewNopHandler())
+
+	dispatcher := event.NewMockDispatcher(ctrl)
+
+	auditManager := manager.NewMockAudit(ctrl)
+	auditManager.EXPECT().BatchAdd(gomock.Len(3)).Times(1)
+
+	subscriber := NewSubscriber(cfg, logger, dispatcher, auditManager)
+
+	eventChan := make(chan *event.Event, 1)
+
+	// When - Then
+	go subscriber.handleCheckEvents(eventChan)
+
+	eventChan <- &event.Event{
+		Timestamp: 123457,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "category", ResourceValue: "1", Action: "delete", IsAllowed: true},
+	}
+	eventChan <- &event.Event{
+		Timestamp: 123456,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "post", ResourceValue: "1", Action: "edit", IsAllowed: true},
+	}
+	eventChan <- &event.Event{
+		Timestamp: 123456,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "post", ResourceValue: "2", Action: "edit", IsAllowed: false},
+	}
+	eventChan <- &event.Event{
+		Timestamp: 123457,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "post", ResourceValue: "3", Action: "delete", IsAllowed: true},
+	}
+	eventChan <- &event.Event{
+		Timestamp: 123457,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "category", ResourceValue: "2", Action: "delete", IsAllowed: false},
+	}
+
+	close(eventChan)
+
+	// Wait 20ms to ensure the spool is triggered.
+	<-time.After(20 * time.Millisecond)
+}

backend/internal/entity/manager/compiled.go

@@ -93,7 +93,7 @@ func (m *compiledPolicyManager) IsAllowed(principalID string, resourceKind strin
 		ResourceValue:  resourceValue,
 		Action:         actionID,
 		IsAllowed:      isAllowed,
-		CompiledPilicy: compiledPolicy,
+		CompiledPolicy: compiledPolicy,
 	}); err != nil {
 		m.logger.Error("unable to dispatch check event", err)
 	}

backend/internal/event/event.go

@@ -22,5 +22,5 @@ type CheckEvent struct {
 	ResourceValue  string
 	Action         string
 	IsAllowed      bool
-	CompiledPilicy *model.CompiledPolicy
+	CompiledPolicy *model.CompiledPolicy
 }

backend/internal/stats/subscriber.go

@@ -2,6 +2,7 @@ package stats
 
 import (
 	"context"
+	"regexp"
 	"time"
 
 	"github.com/eko/authz/backend/configs"
@@ -13,10 +14,11 @@ import (
 )
 
 type subscriber struct {
-	logger          *slog.Logger
-	dispatcher      event.Dispatcher
-	statsManager    manager.Stats
-	statsFlushDelay time.Duration
+	logger            *slog.Logger
+	dispatcher        event.Dispatcher
+	statsManager      manager.Stats
+	statsFlushDelay   time.Duration
+	resourceKindRegex *regexp.Regexp
 }
 
 func NewSubscriber(
@@ -26,10 +28,11 @@ func NewSubscriber(
 	statsManager manager.Stats,
 ) *subscriber {
 	return &subscriber{
-		logger:          logger,
-		dispatcher:      dispatcher,
-		statsManager:    statsManager,
-		statsFlushDelay: cfg.StatsFlushDelay,
+		logger:            logger,
+		dispatcher:        dispatcher,
+		statsManager:      statsManager,
+		statsFlushDelay:   cfg.StatsFlushDelay,
+		resourceKindRegex: regexp.MustCompile(cfg.StatsResourceKindRegex),
 	}
 }
 
@@ -83,8 +86,15 @@ func (s *subscriber) handleCheckEvents(eventChan chan *event.Event) {
 		}
 	}, spooler.WithFlushInterval(s.statsFlushDelay))
 
-	for event := range eventChan {
-		spooler.Add(event)
+	for eventItem := range eventChan {
+		checkEvent, ok := eventItem.Data.(*event.CheckEvent)
+		if !ok {
+			continue
+		}
+
+		if s.resourceKindRegex.MatchString(checkEvent.ResourceKind) {
+			spooler.Add(eventItem)
+		}
 	}
 }
 

backend/internal/stats/subscriber_test.go

@@ -81,3 +81,53 @@ func TestHandleCheckEvents(t *testing.T) {
 	// Wait 20ms to ensure the spool is triggered.
 	<-time.After(20 * time.Millisecond)
 }
+
+func TestHandleCheckEvents_WithResourceKindRegexp(t *testing.T) {
+	// Given
+	ctrl := gomock.NewController(t)
+
+	cfg := &configs.App{
+		StatsFlushDelay:        10 * time.Millisecond,
+		StatsResourceKindRegex: "^post.*",
+	}
+
+	logger := slog.New(log.NewNopHandler())
+
+	dispatcher := event.NewMockDispatcher(ctrl)
+
+	statsManager := manager.NewMockStats(ctrl)
+	statsManager.EXPECT().BatchAddCheck(int64(123457), int64(2), int64(1)).Times(1)
+
+	subscriber := NewSubscriber(cfg, logger, dispatcher, statsManager)
+
+	eventChan := make(chan *event.Event, 1)
+
+	// When - Then
+	go subscriber.handleCheckEvents(eventChan)
+
+	eventChan <- &event.Event{
+		Timestamp: 123457,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "category", ResourceValue: "1", Action: "delete", IsAllowed: true},
+	}
+	eventChan <- &event.Event{
+		Timestamp: 123456,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "post", ResourceValue: "1", Action: "edit", IsAllowed: true},
+	}
+	eventChan <- &event.Event{
+		Timestamp: 123456,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "post", ResourceValue: "2", Action: "edit", IsAllowed: false},
+	}
+	eventChan <- &event.Event{
+		Timestamp: 123457,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "post", ResourceValue: "3", Action: "delete", IsAllowed: true},
+	}
+	eventChan <- &event.Event{
+		Timestamp: 123457,
+		Data:      &event.CheckEvent{Principal: "user1", ResourceKind: "category", ResourceValue: "2", Action: "delete", IsAllowed: false},
+	}
+
+	close(eventChan)
+
+	// Wait 20ms to ensure the spool is triggered.
+	<-time.After(20 * time.Millisecond)
+}