Merge #131422 #131709

131422: server/license: Limit enterprise trial license usage to once per cluster r=fqazi,rafiss a=spilchen

Previously, clusters could install a second enterprise trial license. To prevent misuse, we are now restricting the use of this license type to only once per cluster. This update adds the necessary tracking mechanism.

A new key has been introduced in the KV system key range to record the number of enterprise trial licenses used, ensuring the data persists across restarts.

This change will be backported to versions 24.2, 24.1, 23.2, and 23.1.

Epic: CRDB-39988
Closes: CRDB-40416
Release note (general change): Attempting to install a second enterprise trial license on the same cluster will now fail.

131709: license-linter: Fix typo r=celiala a=jlinder

Missed a blank comment line.

Part of RE-658

Release note: none

Co-authored-by: Matt Spilchen <[email protected]>
Co-authored-by: James H. Linder <[email protected]>

Author: 3 people

Diff for: pkg/ccl/utilccl/BUILD.bazel

@@ -48,6 +48,7 @@ go_test(
         "//pkg/sql",
         "//pkg/testutils",
         "//pkg/testutils/serverutils",
+        "//pkg/testutils/sqlutils",
         "//pkg/util/envutil",
         "//pkg/util/leaktest",
         "//pkg/util/randutil",

Diff for: pkg/ccl/utilccl/license_check.go

@@ -28,14 +28,25 @@ import (
 	"github.com/cockroachdb/redact"
 )
 
+// trialLicenseUsageCount keeps track of the number of times a free trial
+// license has already been installed on this cluster.
+var trialLicenseUsageCount atomic.Int64
+
 var enterpriseLicense = settings.RegisterStringSetting(
 	settings.SystemVisible,
 	"enterprise.license",
 	"the encoded cluster license",
 	"",
 	settings.WithValidateString(
 		func(sv *settings.Values, s string) error {
-			_, err := decode(s)
+			l, err := decode(s)
+			if err != nil {
+				return err
+			}
+			if l != nil && l.Type == licenseccl.License_Trial && trialLicenseUsageCount.Load() > 0 {
+				return errors.WithHint(errors.Newf("a trial license has previously been installed on this cluster"),
+					"Please install a non-trial license to continue")
+			}
 			return err
 		},
 	),
@@ -315,7 +326,10 @@ func check(l *licenseccl.License, at time.Time, org, feature string, withDetails
 func RegisterCallbackOnLicenseChange(
 	ctx context.Context, st *cluster.Settings, licenseEnforcer *licenseserver.Enforcer,
 ) {
-	refreshFunc := func(ctx context.Context) {
+	// refreshFunc is the function responsible for refreshing the enforcer's state.
+	// The isChange parameter indicates whether the license is actually being updated,
+	// as opposed to merely refreshing the current license.
+	refreshFunc := func(ctx context.Context, isChange bool) {
 		lic, err := getLicense(st)
 		if err != nil {
 			log.Errorf(ctx, "unable to refresh license enforcer for license change: %v", err)
@@ -339,9 +353,17 @@ func RegisterCallbackOnLicenseChange(
 			}
 		}
 		licenseEnforcer.RefreshForLicenseChange(ctx, licenseType, licenseExpiry)
+
+		cnt, err := licenseEnforcer.CalculateTrialUsageCount(ctx, licenseType, isChange)
+		if err != nil {
+			log.Errorf(ctx, "unable to calculate trial license usage count: %v", err)
+			return
+		}
+		trialLicenseUsageCount.Store(cnt)
 	}
 	// Install the hook so that we refresh license details when the license changes.
-	enterpriseLicense.SetOnChange(&st.SV, refreshFunc)
+	enterpriseLicense.SetOnChange(&st.SV,
+		func(ctx context.Context) { refreshFunc(ctx, true /* isChange */) })
 	// Call the refresh function for the current license.
-	refreshFunc(ctx)
+	refreshFunc(ctx, false /* isChange */)
 }

Diff for: pkg/ccl/utilccl/license_check_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
@@ -273,12 +274,14 @@ func TestRefreshLicenseEnforcerOnLicenseChange(t *testing.T) {
 	ts1 := timeutil.Unix(1724329716, 0)
 
 	ctx := context.Background()
-	srv := serverutils.StartServerOnly(t, base.TestServerArgs{
+	srv, sqlDB, _ := serverutils.StartServer(t, base.TestServerArgs{
 		// We are changing a cluster setting that can only be done at the system tenant.
 		DefaultTestTenant: base.TestIsSpecificToStorageLayerAndNeedsASystemTenant,
 		Knobs: base.TestingKnobs{
 			Server: &server.TestingKnobs{
 				LicenseTestingKnobs: license.TestingKnobs{
+					Enable:            true,
+					SkipDisable:       true,
 					OverrideStartTime: &ts1,
 				},
 			},
@@ -298,36 +301,83 @@ func TestRefreshLicenseEnforcerOnLicenseChange(t *testing.T) {
 	require.Equal(t, false, enforcer.GetHasLicense())
 	gracePeriodTS, hasGracePeriod := enforcer.GetGracePeriodEndTS()
 	require.True(t, hasGracePeriod)
-	require.Equal(t, ts1.Add(7*24*time.Hour), gracePeriodTS)
+	require.Equal(t, ts1.Add(30*24*time.Hour), gracePeriodTS)
 
 	jan1st2000 := timeutil.Unix(946728000, 0)
 
 	for i, tc := range []struct {
-		license                string
+		// licenses is a list of licenses to be installed sequentially.
+		// All licenses except the last one must be installed successfully.
+		// The outcome of the final license is controlled by the errRE field.
+		licenses []string
+		// errRE specifies a regular expression that matches the expected error message
+		// when installing the last license. If no error is expected, this should be
+		// set to an empty string.
+		errRE string
+		// expectedGracePeriodEnd is the timestamp representing when the grace period
+		// should end. This value is verified only if the last license is installed
+		// successfully.
 		expectedGracePeriodEnd time.Time
 	}{
 		// Note: all licenses below expire on Jan 1st 2000
 		//
 		// Free license - 30 days grace period
-		{"crl-0-EMDYt8MDGAMiDkNSREIgVW5pdCBUZXN0", jan1st2000.Add(30 * 24 * time.Hour)},
+		{[]string{"crl-0-EMDYt8MDGAMiDkNSREIgVW5pdCBUZXN0"}, "", jan1st2000.Add(30 * 24 * time.Hour)},
 		// Trial license - 7 days grace period
-		{"crl-0-EMDYt8MDGAQiDkNSREIgVW5pdCBUZXN0", jan1st2000.Add(7 * 24 * time.Hour)},
+		{[]string{"crl-0-EMDYt8MDGAQiDkNSREIgVW5pdCBUZXN0"}, "", jan1st2000.Add(7 * 24 * time.Hour)},
 		// Enterprise - no grace period
-		{"crl-0-EMDYt8MDGAEiDkNSREIgVW5pdCBUZXN0KAM", timeutil.UnixEpoch},
+		{[]string{"crl-0-EMDYt8MDGAEiDkNSREIgVW5pdCBUZXN0KAM"}, "", timeutil.UnixEpoch},
 		// No license - 7 days grace period
-		{"", ts1.Add(7 * 24 * time.Hour)},
+		{[]string{""}, "", ts1.Add(30 * 24 * time.Hour)},
+		// Only 1 trial license allowed
+		{[]string{"crl-0-EMDYt8MDGAQiDkNSREIgVW5pdCBUZXN0", "crl-0-EMDYt8MDGAQiDkNSREIgVW5pdCBUZXN0"},
+			"a trial license has previously been installed on this cluster", timeutil.UnixEpoch},
 	} {
 		t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
-			_, err := srv.SQLConn(t).Exec(
-				fmt.Sprintf("SET CLUSTER SETTING enterprise.license = '%s'", tc.license),
-			)
+			// Reset from prior test unit.
+			cnt, err := enforcer.SetTrialUsageCount(ctx, 0, false /* checkOldCount */)
 			require.NoError(t, err)
+			require.Equal(t, int64(0), cnt)
+
+			tdb := sqlutils.MakeSQLRunner(sqlDB)
+
+			// Loop through all but the last license. They should all succeed.
+			for i := 0; i < len(tc.licenses)-1; i++ {
+				sql := fmt.Sprintf("SET CLUSTER SETTING enterprise.license = '%s'", tc.licenses[i])
+				tdb.Exec(t, sql)
+
+				// If installing a trial license, we need to wait for the callback to
+				// bump the count before continuing. We depend on the count to cause an
+				// error if another trial license is installed.
+				l, err := decode(tc.licenses[i])
+				require.NoError(t, err)
+				if l.Type == licenseccl.License_Trial {
+					var cnt int64
+					require.Eventually(t, func() bool {
+						cnt = trialLicenseUsageCount.Load()
+						return cnt > 0
+					}, 20*time.Second, time.Millisecond,
+						"trialLicenseUsageCount last returned %t", cnt)
+				}
+			}
+
+			// Handle the last license separately
+			lastLicense := tc.licenses[len(tc.licenses)-1]
+			sql := fmt.Sprintf("SET CLUSTER SETTING enterprise.license = '%s'", lastLicense)
+
+			if tc.errRE != "" {
+				tdb.ExpectErr(t, tc.errRE, sql) // The last license may result in an error
+				return
+			}
+
+			tdb.Exec(t, sql)
+
 			// The SQL can return back before the callback has finished. So, we wait a
 			// bit to see if the desired state is reached.
 			var hasLicense bool
 			require.Eventually(t, func() bool {
 				hasLicense = enforcer.GetHasLicense()
-				return (tc.license != "") == hasLicense
+				return (lastLicense != "") == hasLicense
 			}, 20*time.Second, time.Millisecond,
 				"GetHasLicense() last returned %t", hasLicense)
 			var ts time.Time

Diff for: pkg/gen/stringer.bzl

@@ -15,6 +15,7 @@ STRINGER_SRCS = [
     "//pkg/multitenant/tenantcapabilities:id_string.go",
     "//pkg/raft/quorum:voteresult_string.go",
     "//pkg/roachpb:leasetype_string.go",
+    "//pkg/server/license:lictype_string.go",
     "//pkg/sql/catalog/catalogkeys:commenttype_string.go",
     "//pkg/sql/catalog/catpb:privilegedescversion_string.go",
     "//pkg/sql/catalog/descpb:formatversion_string.go",

Diff for: pkg/keys/constants.go

@@ -305,6 +305,9 @@ var (
 	// set during cluster initialization, by which a license must be installed to avoid
 	// throttling. The value is stored as the number of seconds since the Unix epoch.
 	ClusterInitGracePeriodTimestamp = roachpb.Key(makeKey(SystemPrefix, roachpb.RKey("lic-gpi-ts")))
+	// TrialLicenseUsageCount is used to keep track of the number of times a trial
+	// license was installed on the cluster.
+	TrialLicenseUsageCount = roachpb.Key(makeKey(SystemPrefix, roachpb.RKey("lic-tluc")))
 	//
 	// NodeIDGenerator is the global node ID generator sequence.
 	NodeIDGenerator = roachpb.Key(makeKey(SystemPrefix, roachpb.RKey("node-idgen")))

Diff for: pkg/keys/doc.go

@@ -249,6 +249,7 @@ var _ = [...]interface{}{
 	NodeLivenessPrefix,              // "\x00liveness-"
 	BootstrapVersionKey,             // "bootstrap-version"
 	ClusterInitGracePeriodTimestamp, // "lic-gpi-ts"
+	TrialLicenseUsageCount,          // "lic-tluc"
 	NodeIDGenerator,                 // "node-idgen"
 	RangeIDGenerator,                // "range-idgen"
 	StatusPrefix,                    // "status-"

Diff for: pkg/server/license/BUILD.bazel

@@ -1,11 +1,13 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("//build:STRINGER.bzl", "stringer")
 
 go_library(
     name = "license",
     srcs = [
         "cclbridge.go",
         "enforcer.go",
         "opts.go",
+        ":gen-lictype-stringer",  # keep
     ],
     importpath = "github.com/cockroachdb/cockroach/pkg/server/license",
     visibility = ["//visibility:public"],
@@ -50,3 +52,10 @@ go_test(
         "@com_github_stretchr_testify//require",
     ],
 )
+
+stringer(
+    name = "gen-lictype-stringer",
+    src = "cclbridge.go",
+    additional_args = ["-linecomment"],
+    typ = "LicType",
+)

Diff for: pkg/server/license/cclbridge.go

@@ -29,10 +29,11 @@ var RegisterCallbackOnLicenseChange = func(context.Context, *cluster.Settings, *
 // enforcer.
 type LicType int
 
+//go:generate stringer -type=LicType -linecomment
 const (
-	LicTypeNone LicType = iota
-	LicTypeTrial
-	LicTypeFree
-	LicTypeEnterprise
-	LicTypeEvaluation
+	LicTypeNone       LicType = iota // none
+	LicTypeTrial                     // trial
+	LicTypeFree                      // free
+	LicTypeEnterprise                // enterprise
+	LicTypeEvaluation                // evaluation
 )

Diff for: pkg/server/license/enforcer.go

@@ -87,6 +87,14 @@ type Enforcer struct {
 	// When enabled, all checks, including telemetry and expired license validation,
 	// are bypassed. This is typically used to disable enforcement for single-node deployments.
 	isDisabled atomic.Bool
+
+	// trialUsageCount keeps track of the number of times a free trial license has
+	// been used on this cluster.
+	trialUsageCount atomic.Int64
+
+	// db is a pointer to the database for use for KV read/writes. This is only
+	// set for the system tenant.
+	db isql.DB
 }
 
 type TestingKnobs struct {
@@ -181,7 +189,7 @@ func (e *Enforcer) Start(ctx context.Context, st *cluster.Settings, opts ...Opti
 	e.maybeLogActiveOverrides(ctx)
 
 	if !startDisabled {
-		if err := e.maybeWriteClusterInitGracePeriodTS(ctx, options); err != nil {
+		if err := e.readClusterMetadata(ctx, options); err != nil {
 			return err
 		}
 	}
@@ -204,9 +212,9 @@ func (e *Enforcer) Start(ctx context.Context, st *cluster.Settings, opts ...Opti
 	return nil
 }
 
-// maybeWriteClusterInitGracePeriodTS checks if the cluster init grace period
-// timestamp needs to be written to the KV layer and writes it if needed.
-func (e *Enforcer) maybeWriteClusterInitGracePeriodTS(ctx context.Context, options options) error {
+// readClusterMetadata will read, and maybe write, cluster metadata for license
+// enforcement. The metadata is stored in the KV system keyspace.
+func (e *Enforcer) readClusterMetadata(ctx context.Context, options options) error {
 	// Secondary tenants do not have access to the system keyspace where
 	// the cluster init grace period is stored. As a fallback, we apply a 7-day
 	// grace period from the tenant's start time, which is used only when no
@@ -223,7 +231,26 @@ func (e *Enforcer) maybeWriteClusterInitGracePeriodTS(ctx context.Context, optio
 		return nil
 	}
 
+	// Save a pointer to the database for future use in updating the trial license
+	// usage count. This is only set for the system tenant.
+	e.db = options.db
+
 	return options.db.Txn(ctx, func(ctx context.Context, txn isql.Txn) error {
+		// Cache the current trial usage count.
+		trialUsageCount, err := txn.KV().Get(ctx, keys.TrialLicenseUsageCount)
+		if err != nil {
+			return err
+		}
+		if trialUsageCount.Value == nil {
+			e.trialUsageCount.Store(0)
+		} else {
+			e.trialUsageCount.Store(trialUsageCount.ValueInt())
+		}
+		log.Infof(ctx, "trial license usage count initialized to %d", e.trialUsageCount.Load())
+
+		// Cache and maybe set the cluster init grace period timestamp. This is the
+		// grace period we will use if the cluster does not have a license installed.
+		//
 		// We could use a conditional put for this logic. However, we want to read
 		// and cache the value, and the common case is that the value will be read.
 		// Only during the initialization of the first node in the cluster will we
@@ -433,7 +460,7 @@ func (e *Enforcer) RefreshForLicenseChange(
 	}
 
 	var sb strings.Builder
-	sb.WriteString("enforcer license updated: ")
+	sb.WriteString(fmt.Sprintf("enforcer license updated: type is %s, ", licType.String()))
 	gpEnd, _ := e.GetGracePeriodEndTS()
 	if !gpEnd.IsZero() {
 		sb.WriteString(fmt.Sprintf("grace period ends at %q, ", gpEnd))
@@ -446,6 +473,65 @@ func (e *Enforcer) RefreshForLicenseChange(
 	log.Infof(ctx, "%s", sb.String())
 }
 
+// CalculateTrialUsageCount returns the number of times a trial license has
+// been used, including the current trial license if a new one is being applied.
+// This function increments the count if the current license is changing and is a trial.
+func (e *Enforcer) CalculateTrialUsageCount(
+	ctx context.Context, currentLicense LicType, isLicenseChange bool,
+) (int64, error) {
+	// If we aren't setting a new trial license, return the cached copy. The e.db
+	// check is necessary as that's needed to read/write to the KV. This will be
+	// set for the system tenant, which is where the license can ever be set anyway.
+	if currentLicense != LicTypeTrial || !isLicenseChange || e.db == nil {
+		return e.trialUsageCount.Load(), nil
+	}
+
+	return e.SetTrialUsageCount(ctx, e.trialUsageCount.Load()+1, true)
+}
+
+// SetTrialUsageCount is an API to set the trial usage count in the enforcer and
+// in the KV. The value in the enforcer is always updated. If checkOldCount is
+// true, the update to the KV is conditional on the old value matching trialUsageCount.
+func (e *Enforcer) SetTrialUsageCount(
+	ctx context.Context, newCount int64, checkOldCount bool,
+) (cnt int64, err error) {
+	if e.db == nil {
+		return 0, errors.AssertionFailedf("no database set")
+	}
+	err = e.db.Txn(ctx, func(ctx context.Context, txn isql.Txn) error {
+		// If checking the old count, we only do the update in the KV if the
+		// existing value for trialUsageCount matches what's in the KV already
+		// (a missing key is treated as 0). This is necessary to ensure a license
+		// change to use the trial license will only increase the count by 1 when
+		// this function is called for each node.
+		if checkOldCount {
+			oldVal, err := txn.KV().Get(ctx, keys.TrialLicenseUsageCount)
+			if err != nil {
+				return err
+			}
+			if oldVal.Value == nil && e.trialUsageCount.Load() != 0 {
+				e.trialUsageCount.Store(0)
+				return nil
+			} else if oldVal.Value != nil && oldVal.ValueInt() != e.trialUsageCount.Load() {
+				e.trialUsageCount.Store(oldVal.ValueInt())
+				return nil
+			}
+		}
+		err = txn.KV().Put(ctx, keys.TrialLicenseUsageCount, newCount)
+		if err != nil {
+			return err
+		}
+		e.trialUsageCount.Store(newCount)
+		return nil
+	})
+	if err != nil {
+		return
+	}
+	cnt = e.trialUsageCount.Load()
+	log.Infof(ctx, "trial license usage count is %d", cnt)
+	return
+}
+
 // Disable turns off all license enforcement for the lifetime of this object.
 func (e *Enforcer) Disable(ctx context.Context) {
 	// We provide an override so that we can continue to test license enforcement