Add adaptertypes package for ontology integration

1password/client.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -15,6 +14,7 @@ import (
 
 	"github.com/refractionPOINT/go-uspclient"
 	"github.com/refractionPOINT/go-uspclient/protocol"
+	"github.com/refractionPOINT/usp-adapters/adaptertypes"
 	"github.com/refractionPOINT/usp-adapters/utils"
 )
 
@@ -31,15 +31,8 @@ type opRequest struct {
 	Cursor    string `json:"cursor,omitempty"`
 }
 
-var URL = map[string]string{
-	"business":   "https://events.1password.com",
-	"enterprise": "https://events.ent.1password.com",
-	"ca":         "https://events.1password.ca",
-	"eu":         "https://events.1password.eu",
-}
-
 type OnePasswordAdapter struct {
-	conf       OnePasswordConfig
+	conf       adaptertypes.OnePasswordConfig
 	uspClient  *uspclient.Client
 	httpClient *http.Client
 
@@ -52,30 +45,7 @@ type OnePasswordAdapter struct {
 	ctx context.Context
 }
 
-type OnePasswordConfig struct {
-	ClientOptions uspclient.ClientOptions `json:"client_options" yaml:"client_options"`
-	Token         string                  `json:"token" yaml:"token"`
-	Endpoint      string                  `json:"endpoint" yaml:"endpoint"`
-}
-
-func (c *OnePasswordConfig) Validate() error {
-	if err := c.ClientOptions.Validate(); err != nil {
-		return fmt.Errorf("client_options: %v", err)
-	}
-	if c.Token == "" {
-		return errors.New("missing token")
-	}
-	if c.Endpoint == "" {
-		return errors.New("missing endpoint")
-	}
-	_, ok := URL[c.Endpoint]
-	if !strings.HasPrefix(c.Endpoint, "https://") && !ok {
-		return fmt.Errorf("invalid endpoint, not https or in %v", URL)
-	}
-	return nil
-}
-
-func NewOnePasswordpAdapter(conf OnePasswordConfig) (*OnePasswordAdapter, chan struct{}, error) {
+func NewOnePasswordpAdapter(conf adaptertypes.OnePasswordConfig) (*OnePasswordAdapter, chan struct{}, error) {
 	var err error
 	a := &OnePasswordAdapter{
 		conf:   conf,
@@ -85,7 +55,7 @@ func NewOnePasswordpAdapter(conf OnePasswordConfig) (*OnePasswordAdapter, chan s
 
 	if strings.HasPrefix(conf.Endpoint, "https://") {
 		a.endpoint = conf.Endpoint
-	} else if v, ok := URL[conf.Endpoint]; ok {
+	} else if v, ok := adaptertypes.OnePasswordURL[conf.Endpoint]; ok {
 		a.endpoint = v
 	} else {
 		return nil, nil, fmt.Errorf("not a valid api endpoint: %s", conf.Endpoint)

adaptertypes/1password.go

@@ -0,0 +1,38 @@
+package adaptertypes
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+var OnePasswordURL = map[string]string{
+	"business":   "https://events.1password.com",
+	"enterprise": "https://events.ent.1password.com",
+	"ca":         "https://events.1password.ca",
+	"eu":         "https://events.1password.eu",
+}
+
+// OnePasswordConfig defines the configuration for the 1Password adapter
+type OnePasswordConfig struct {
+	ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"`
+	Token         string        `json:"token" yaml:"token" description:"1Password Events API bearer token" category:"auth" sensitive:"true" llmguidance:"Generate token in 1Password admin console under Integrations > Events Reporting"`
+	Endpoint      string        `json:"endpoint" yaml:"endpoint" description:"1Password API endpoint" category:"source" example:"business" llmguidance:"Options: 'business', 'enterprise', 'ca' (Canada), 'eu' (Europe). Or provide full HTTPS URL"`
+}
+
+func (c *OnePasswordConfig) Validate() error {
+	if err := c.ClientOptions.Validate(); err != nil {
+		return fmt.Errorf("client_options: %v", err)
+	}
+	if c.Token == "" {
+		return errors.New("missing token")
+	}
+	if c.Endpoint == "" {
+		return errors.New("missing endpoint")
+	}
+	_, ok := OnePasswordURL[c.Endpoint]
+	if !strings.HasPrefix(c.Endpoint, "https://") && !ok {
+		return fmt.Errorf("invalid endpoint, not https or in %v", OnePasswordURL)
+	}
+	return nil
+}

adaptertypes/azure_event_hub.go

@@ -0,0 +1,22 @@
+package adaptertypes
+
+import (
+	"errors"
+	"fmt"
+)
+
+// EventHubConfig defines the configuration for the Azure Event Hub adapter
+type EventHubConfig struct {
+	ClientOptions    ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"`
+	ConnectionString string        `json:"connection_string" yaml:"connection_string" description:"Azure Event Hub connection string" category:"auth" sensitive:"true" llmguidance:"Found in Azure Portal under Event Hubs > Shared access policies. Requires 'Listen' permission"`
+}
+
+func (c *EventHubConfig) Validate() error {
+	if err := c.ClientOptions.Validate(); err != nil {
+		return fmt.Errorf("client_options: %v", err)
+	}
+	if c.ConnectionString == "" {
+		return errors.New("missing connection_string")
+	}
+	return nil
+}

adaptertypes/bigquery.go

@@ -0,0 +1,38 @@
+package adaptertypes
+
+import (
+	"errors"
+)
+
+// BigQueryConfig defines the configuration for the Google BigQuery adapter
+type BigQueryConfig struct {
+	ClientOptions       ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"`
+	ProjectId           string        `json:"project_id" yaml:"project_id" description:"GCP project ID for billing/quota" category:"source" example:"my-project-123"`
+	BigQueryProject     string        `json:"bigquery_project" yaml:"bigquery_project" description:"GCP project containing the BigQuery dataset" category:"source" example:"analytics-project"`
+	DatasetName         string        `json:"dataset_name" yaml:"dataset_name" description:"BigQuery dataset name" category:"source" example:"logs_dataset"`
+	TableName           string        `json:"table_name" yaml:"table_name" description:"BigQuery table name" category:"source" example:"security_events"`
+	ServiceAccountCreds string        `json:"service_account_creds,omitempty" yaml:"service_account_creds,omitempty" description:"GCP service account JSON credentials" category:"auth" sensitive:"true"`
+	SqlQuery            string        `json:"sql_query" yaml:"sql_query" description:"SQL query to execute" category:"source" llmguidance:"Use WHERE clause to filter data. Include timestamp column for incremental queries"`
+	QueryInterval       string        `json:"query_interval" yaml:"query_interval" description:"Interval between query executions" category:"behavior" example:"1h" default:"1h" llmguidance:"Format: duration string like '1h', '30m', '24h'"`
+	IsOneTimeLoad       bool          `json:"is_one_time_load" yaml:"is_one_time_load" description:"If true, run query once and exit. If false, run continuously at intervals" category:"behavior" default:"false"`
+}
+
+func (bq *BigQueryConfig) Validate() error {
+	if bq.ProjectId == "" {
+		return errors.New("missing project_id")
+	}
+	// this will usually be th same as projectID but could be different if using outside project dataset such as a public data set
+	if bq.BigQueryProject == "" {
+		return errors.New("missing bigquery project name")
+	}
+	if bq.DatasetName == "" {
+		return errors.New("missing dataset_name")
+	}
+	if bq.TableName == "" {
+		return errors.New("missing table_name")
+	}
+	if bq.SqlQuery == "" {
+		return errors.New("missing sql query")
+	}
+	return nil
+}

adaptertypes/bitwarden.go

@@ -0,0 +1,55 @@
+package adaptertypes
+
+import (
+	"errors"
+	"fmt"
+)
+
+// BitwardenConfig defines the configuration for the Bitwarden adapter
+type BitwardenConfig struct {
+	ClientOptions    ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"`
+	ClientID         string        `json:"client_id" yaml:"client_id" description:"Bitwarden API client ID" category:"auth"`
+	ClientSecret     string        `json:"client_secret" yaml:"client_secret" description:"Bitwarden API client secret" category:"auth" sensitive:"true"`
+	Region           string        `json:"region" yaml:"region" description:"Bitwarden region" category:"source" example:"US" llmguidance:"Options: 'US' or 'EU'"`
+	TokenEndpointURL string        `json:"token_endpoint_url" yaml:"token_endpoint_url" description:"Custom token endpoint URL" category:"source" llmguidance:"Optional. Override default token endpoint"`
+	EventsBaseURL    string        `json:"events_base_url" yaml:"events_base_url" description:"Custom events API base URL" category:"source" llmguidance:"Optional. Override default events endpoint"`
+}
+
+func (c *BitwardenConfig) Validate() error {
+	if err := c.ClientOptions.Validate(); err != nil {
+		return fmt.Errorf("client_options: %v", err)
+	}
+
+	if c.ClientID == "" {
+		return errors.New("missing client_id")
+	}
+	if c.ClientSecret == "" {
+		return errors.New("missing client_secret")
+	}
+
+	// Check if custom URLs are provided
+	hasCustomURLs := c.TokenEndpointURL != "" || c.EventsBaseURL != ""
+
+	if hasCustomURLs {
+		// If custom URLs are provided, both must be set and region must be empty
+		if c.TokenEndpointURL == "" {
+			return errors.New("token_endpoint_url must be set when events_base_url is provided")
+		}
+		if c.EventsBaseURL == "" {
+			return errors.New("events_base_url must be set when token_endpoint_url is provided")
+		}
+		if c.Region != "" {
+			return errors.New("region cannot be set when using custom URLs (token_endpoint_url and events_base_url)")
+		}
+	} else {
+		// If custom URLs are not provided, use region-based configuration
+		if c.Region == "" {
+			c.Region = "us"
+		}
+		if c.Region != "us" && c.Region != "eu" {
+			return fmt.Errorf("invalid region: %s (must be 'us' or 'eu')", c.Region)
+		}
+	}
+
+	return nil
+}

adaptertypes/box.go

@@ -0,0 +1,25 @@
+package adaptertypes
+
+import (
+	"errors"
+	"fmt"
+)
+
+// BoxConfig defines the configuration for the Box adapter
+type BoxConfig struct {
+	ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"`
+	ClientID      string        `json:"client_id" yaml:"client_id" description:"Box OAuth2 client ID" category:"auth"`
+	ClientSecret  string        `json:"client_secret" yaml:"client_secret" description:"Box OAuth2 client secret" category:"auth" sensitive:"true"`
+	SubjectID     string        `json:"subject_id" yaml:"subject_id" description:"Box enterprise or user ID" category:"auth" llmguidance:"Enterprise ID for enterprise events, or user ID for user events"`
+}
+
+func (c *BoxConfig) Validate() error {
+	if err := c.ClientOptions.Validate(); err != nil {
+		return fmt.Errorf("client_options: %v", err)
+	}
+
+	if c.ClientID == "" || c.ClientSecret == "" || c.SubjectID == "" {
+		return errors.New("missing Box client ID, secret, or subject ID")
+	}
+	return nil
+}

adaptertypes/cato.go

@@ -0,0 +1,27 @@
+package adaptertypes
+
+import (
+	"errors"
+	"fmt"
+)
+
+// CatoConfig defines the configuration for the Cato Networks adapter
+type CatoConfig struct {
+	ClientOptions   ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"`
+	WriteTimeoutSec uint64        `json:"write_timeout_sec,omitempty" yaml:"write_timeout_sec,omitempty" description:"Timeout in seconds for writing data to USP" category:"performance" default:"600"`
+	ApiKey          string        `json:"apikey" yaml:"apikey" description:"Cato Networks API key" category:"auth" sensitive:"true"`
+	AccountId       int           `json:"accountid" yaml:"accountid" description:"Cato Networks account ID" category:"source"`
+}
+
+func (c *CatoConfig) Validate() error {
+	if err := c.ClientOptions.Validate(); err != nil {
+		return fmt.Errorf("client_options: %v", err)
+	}
+	if c.AccountId == 0 {
+		return errors.New("missing account id")
+	}
+	if c.ApiKey == "" {
+		return errors.New("missing api key")
+	}
+	return nil
+}

adaptertypes/cylance.go

@@ -0,0 +1,36 @@
+package adaptertypes
+
+import (
+	"errors"
+	"fmt"
+)
+
+const defaultLoggingBaseURL = "https://protectapi.cylance.com"
+
+// CylanceConfig defines the configuration for the Cylance adapter
+type CylanceConfig struct {
+	ClientOptions  ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"`
+	TenantID       string        `json:"tenant_id" yaml:"tenant_id" description:"Cylance tenant ID" category:"auth"`
+	AppID          string        `json:"app_id" yaml:"app_id" description:"Cylance application ID" category:"auth"`
+	AppSecret      string        `json:"app_secret" yaml:"app_secret" description:"Cylance application secret" category:"auth" sensitive:"true"`
+	LoggingBaseURL string        `json:"logging_base_url" yaml:"logging_base_url" description:"Cylance logging API base URL" category:"source" example:"https://protectapi.cylance.com"`
+}
+
+func (c *CylanceConfig) Validate() error {
+	if err := c.ClientOptions.Validate(); err != nil {
+		return fmt.Errorf("client_options: %v", err)
+	}
+	if c.TenantID == "" {
+		return errors.New("missing tenant id")
+	}
+	if c.AppID == "" {
+		return errors.New("missing app id")
+	}
+	if c.AppSecret == "" {
+		return errors.New("missing app secret")
+	}
+	if c.LoggingBaseURL == "" {
+		c.LoggingBaseURL = defaultLoggingBaseURL
+	}
+	return nil
+}

adaptertypes/defender.go

@@ -0,0 +1,30 @@
+package adaptertypes
+
+import (
+	"errors"
+	"fmt"
+)
+
+// DefenderConfig defines the configuration for the Microsoft Defender adapter
+type DefenderConfig struct {
+	ClientOptions ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"`
+	TenantID      string        `json:"tenant_id" yaml:"tenant_id" description:"Azure AD tenant ID" category:"auth" example:"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"`
+	ClientID      string        `json:"client_id" yaml:"client_id" description:"Azure AD application (client) ID" category:"auth" example:"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"`
+	ClientSecret  string        `json:"client_secret" yaml:"client_secret" description:"Azure AD application client secret" category:"auth" sensitive:"true"`
+}
+
+func (c *DefenderConfig) Validate() error {
+	if err := c.ClientOptions.Validate(); err != nil {
+		return fmt.Errorf("client_options: %v", err)
+	}
+	if c.TenantID == "" {
+		return errors.New("missing tenant_id")
+	}
+	if c.ClientID == "" {
+		return errors.New("missing client_id")
+	}
+	if c.ClientSecret == "" {
+		return errors.New("missing client_secret")
+	}
+	return nil
+}

adaptertypes/duo.go

@@ -0,0 +1,30 @@
+package adaptertypes
+
+import (
+	"errors"
+	"fmt"
+)
+
+// DuoConfig defines the configuration for the Duo Security adapter
+type DuoConfig struct {
+	ClientOptions  ClientOptions `json:"client_options" yaml:"client_options" description:"USP client configuration for data ingestion" category:"client"`
+	IntegrationKey string        `json:"integration_key" yaml:"integration_key" description:"Duo Admin API integration key" category:"auth" sensitive:"true" llmguidance:"Generate in Duo Admin Panel > Applications > Admin API"`
+	SecretKey      string        `json:"secret_key" yaml:"secret_key" description:"Duo Admin API secret key" category:"auth" sensitive:"true"`
+	APIHostname    string        `json:"api_hostname" yaml:"api_hostname" description:"Duo API hostname" category:"source" example:"api-xxxxx.duosecurity.com" llmguidance:"Found in Duo Admin Panel under your Admin API application"`
+}
+
+func (c *DuoConfig) Validate() error {
+	if err := c.ClientOptions.Validate(); err != nil {
+		return fmt.Errorf("client_options: %v", err)
+	}
+	if c.IntegrationKey == "" {
+		return errors.New("missing integration_key")
+	}
+	if c.SecretKey == "" {
+		return errors.New("missing secret_key")
+	}
+	if c.APIHostname == "" {
+		return errors.New("missing api_hostname")
+	}
+	return nil
+}