feat: Add AI activity plugins plus HubSpot and Notion user activity ingestion

.github/workflows/config-ui.yml

@@ -36,3 +36,5 @@ jobs:
       run: cd config-ui; yarn
     - name: Run Lint
       run: cd config-ui; yarn tsc && yarn lint
+    - name: Run Tests
+      run: cd config-ui; yarn test

.gitignore

@@ -143,6 +143,7 @@ lake
 *.so
 
 NOTES.md
+.plans/
 
 main
 vendor
@@ -168,4 +169,4 @@ venv
 /backend/plugins/gitextractor/parser/output*
 /backend/plugins/gitextractor/parser/demo_repo/
 
-__debug_bin*
+__debug_bin*

AGENTS.md

@@ -16,6 +16,14 @@ limitations under the License.
 -->
 # Apache DevLake - AI Coding Agent Instructions
 
+## Response Style
+- Keep plans and responses concise by default
+- Prefer short bullets and short paragraphs over long explanations
+- For simple requests, give the minimum useful answer only
+- If brevity and polished grammar conflict, choose brevity
+- Avoid repeating context, restating the prompt, or adding filler
+- Summaries should be direct and approval-friendly so they are fast to scan
+
 ## Project Overview
 Apache DevLake is a dev data platform that ingests data from DevOps tools (GitHub, GitLab, Jira, Jenkins, etc.), transforms it into standardized domain models, and enables metrics/dashboards via Grafana.
 

backend/core/models/domainlayer/ai/ai_activity.go

@@ -0,0 +1,77 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package ai holds domain-layer models for AI coding assistant activity.
+package ai
+
+import (
+	"time"
+
+	"github.com/apache/incubator-devlake/core/models/domainlayer"
+)
+
+// AiActivity records a single day of AI-assisted coding activity for one developer.
+// It is the normalised domain-layer representation shared by all AI tool plugins
+// (Claude, GitHub Copilot, Cursor, Codex, etc.).
+type AiActivity struct {
+	domainlayer.DomainEntity
+
+	// Provider identifies the AI tool (e.g. "claude", "gh-copilot", "cursor", "codex").
+	Provider string `gorm:"type:varchar(100);index" json:"provider"`
+
+	// AccountId is the global DevLake account resolved from UserEmail.
+	// May be empty if no matching account was found.
+	AccountId string `gorm:"type:varchar(255);index" json:"accountId"`
+
+	// UserEmail is the raw email (or login) as returned by the provider.
+	UserEmail string `gorm:"type:varchar(255)" json:"userEmail"`
+
+	// Date is the calendar day the activity was recorded.
+	Date time.Time `gorm:"type:date;index" json:"date"`
+
+	// Type categorises the activity (e.g. "CODE_EDIT", "CHAT").
+	Type string `gorm:"type:varchar(100)" json:"type"`
+
+	// Model is the AI model variant used (e.g. "claude-sonnet-4-5", "gpt-4o").
+	Model string `gorm:"type:varchar(100)" json:"model"`
+
+	// InterfaceType describes how the developer interacts with the tool.
+	// Known values: "cli", "ide_plugin", "web_ui".
+	InterfaceType string `gorm:"type:varchar(50)" json:"interfaceType"`
+
+	// Volume & autocomplete metrics (Copilot / Cursor style)
+	NumSessions      int `json:"numSessions"`
+	SuggestionsCount int `json:"suggestionsCount"` // suggestions shown to the developer
+	AcceptanceCount  int `json:"acceptanceCount"`  // suggestions accepted by the developer
+
+	// Code change metrics
+	LinesAdded   int `json:"linesAdded"`
+	LinesRemoved int `json:"linesRemoved"`
+
+	// Agentic outcome metrics (Claude / Codex style)
+	CommitsCreated int `json:"commitsCreated"`
+	PrsCreated     int `json:"prsCreated"`
+
+	// Cost & token metrics
+	InputTokens      int64   `json:"inputTokens"`
+	OutputTokens     int64   `json:"outputTokens"`
+	EstimatedCostUsd float64 `json:"estimatedCostUsd"`
+}
+
+func (AiActivity) TableName() string {
+	return "ai_activities"
+}

backend/core/models/domainlayer/crossdomain/user_activity.go

@@ -0,0 +1,56 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crossdomain
+
+import (
+	"time"
+
+	"github.com/apache/incubator-devlake/core/models/domainlayer"
+)
+
+// UserActivity is a generic cross-tool activity table for non-code, non-ticket
+// user actions collected from external collaboration platforms such as HubSpot
+// and Notion. It intentionally mirrors the shared-domain pattern used by
+// ai_activities while preserving activity-specific fields.
+type UserActivity struct {
+	domainlayer.DomainEntity
+
+	ConnectionId uint64 `gorm:"index" json:"connectionId"`
+	ScopeId      string `gorm:"type:varchar(255);index" json:"scopeId"`
+
+	SourceSystem  string `gorm:"type:varchar(100);index" json:"sourceSystem"`
+	SourceEventId string `gorm:"type:varchar(255)" json:"sourceEventId"`
+
+	AccountId    string `gorm:"type:varchar(255);index" json:"accountId"`
+	UserEmail    string `gorm:"type:varchar(255);index" json:"userEmail"`
+	UserDisplay  string `gorm:"type:varchar(255)" json:"userDisplay"`
+	NativeUserId string `gorm:"type:varchar(255)" json:"nativeUserId"`
+
+	ActionType string `gorm:"type:varchar(100);index" json:"actionType"`
+	ObjectType string `gorm:"type:varchar(100)" json:"objectType"`
+	ObjectId   string `gorm:"type:varchar(255)" json:"objectId"`
+	ObjectRef  string `gorm:"type:varchar(512)" json:"objectRef"`
+
+	ActionTime time.Time `gorm:"type:timestamp;index" json:"actionTime"`
+	ActionDay  time.Time `gorm:"type:date;index" json:"actionDay"`
+	Summary    string    `gorm:"type:text" json:"summary"`
+}
+
+func (UserActivity) TableName() string {
+	return "user_activities"
+}

backend/core/models/domainlayer/domaininfo/domaininfo.go

@@ -19,6 +19,7 @@ package domaininfo
 
 import (
 	"github.com/apache/incubator-devlake/core/dal"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/ai"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
@@ -70,6 +71,7 @@ func GetDomainTablesInfo() []dal.Tabler {
 		&crossdomain.TeamUser{},
 		&crossdomain.User{},
 		&crossdomain.UserAccount{},
+		&crossdomain.UserActivity{},
 		// devops
 		&devops.CICDPipeline{},
 		&devops.CICDTask{},
@@ -100,5 +102,7 @@ func GetDomainTablesInfo() []dal.Tabler {
 		&qa.QaApi{},
 		&qa.QaTestCase{},
 		&qa.QaTestCaseExecution{},
+		// ai
+		&ai.AiActivity{},
 	}
 }

backend/core/models/migrationscripts/20260311_add_user_project_mapping.go

@@ -0,0 +1,55 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addUserProjectMapping)(nil)
+
+type addUserProjectMapping struct{}
+
+type userProjectMapping20260311 struct {
+	UserLogin   string `gorm:"primaryKey;type:varchar(255)"`
+	ProjectName string `gorm:"primaryKey;type:varchar(255);index"`
+	archived.NoPKModel
+}
+
+func (userProjectMapping20260311) TableName() string {
+	return "user_project_mapping"
+}
+
+func (script *addUserProjectMapping) Up(basicRes context.BasicRes) errors.Error {
+	return migrationhelper.AutoMigrateTables(
+		basicRes,
+		&userProjectMapping20260311{},
+	)
+}
+
+func (*addUserProjectMapping) Version() uint64 {
+	return 20260311120000
+}
+
+func (*addUserProjectMapping) Name() string {
+	return "add user_project_mapping table"
+}

backend/core/models/migrationscripts/20260331_add_ai_activities_unified_schema.go

@@ -0,0 +1,109 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"time"
+
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addAiActivitiesUnifiedSchema)(nil)
+
+type aiActivity20260331 struct {
+	archived.DomainEntity
+	Provider         string    `gorm:"type:varchar(100);index"`
+	AccountId        string    `gorm:"type:varchar(255);index"`
+	UserEmail        string    `gorm:"type:varchar(255)"`
+	Date             time.Time `gorm:"type:date;index"`
+	Type             string    `gorm:"type:varchar(100)"`
+	Model            string    `gorm:"type:varchar(100)"`
+	InterfaceType    string    `gorm:"type:varchar(50)"`
+	NumSessions      int
+	SuggestionsCount int
+	AcceptanceCount  int
+	LinesAdded       int
+	LinesRemoved     int
+	CommitsCreated   int
+	PrsCreated       int
+	InputTokens      int64
+	OutputTokens     int64
+	EstimatedCostUsd float64
+}
+
+func (aiActivity20260331) TableName() string {
+	return "ai_activities"
+}
+
+type addAiActivitiesUnifiedSchema struct{}
+
+// Up runs the migration.
+//
+// Changes to ai_activities:
+//   - Rename commits_by_claude  → commits_created
+//   - Rename prs_by_claude      → prs_created
+//   - Add interface_type   varchar(50)
+//   - Add suggestions_count int (default 0)
+//   - Add acceptance_count  int (default 0)
+//
+// The RENAME is implemented as ADD+COPY+DROP so it works across MySQL, PostgreSQL,
+// and SQLite without dialect-specific RENAME COLUMN support issues.
+// AutoMigrate (called after all migration scripts) will then reconcile any remaining
+// column differences against the updated Go struct.
+func (script *addAiActivitiesUnifiedSchema) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+
+	// If the table doesn't exist (fresh install or e2e tests), completely create it
+	// using AutoMigrate, bypassing the need to rename missing legacy columns.
+	if !db.HasTable("ai_activities") {
+		return db.AutoMigrate(&aiActivity20260331{})
+	}
+
+	// 1. Rename commits_by_claude → commits_created
+	if err := db.AddColumn("ai_activities", "commits_created", "bigint"); err != nil {
+		// column may already exist if migration was partially applied — ignore duplicate
+		_ = err
+	}
+	_ = db.Exec("UPDATE ai_activities SET commits_created = commits_by_claude WHERE commits_by_claude IS NOT NULL")
+	_ = db.DropColumns("ai_activities", "commits_by_claude")
+
+	// 2. Rename prs_by_claude → prs_created
+	if err := db.AddColumn("ai_activities", "prs_created", "bigint"); err != nil {
+		_ = err
+	}
+	_ = db.Exec("UPDATE ai_activities SET prs_created = prs_by_claude WHERE prs_by_claude IS NOT NULL")
+	_ = db.DropColumns("ai_activities", "prs_by_claude")
+
+	// 3. Add new columns (AddColumn is idempotent-friendly — errors on duplicate are silently ignored)
+	_ = db.AddColumn("ai_activities", "interface_type", "varchar(50)")
+	_ = db.AddColumn("ai_activities", "suggestions_count", "bigint")
+	_ = db.AddColumn("ai_activities", "acceptance_count", "bigint")
+
+	return nil
+}
+
+func (*addAiActivitiesUnifiedSchema) Version() uint64 {
+	return 20260331000001
+}
+
+func (*addAiActivitiesUnifiedSchema) Name() string {
+	return "add unified schema fields to ai_activities (rename commits/prs, add interface_type, suggestions, acceptance)"
+}