speed up DB itests (#1022)

Author: LexLuthr

harmony/harmonydb/harmonydb.go

@@ -20,8 +20,12 @@ func init() {
 }
 
 func NewFromConfig(cfg Config) (*DB, error) {
-	cfg.SqlEmbedFS = &upgradeFS
-	cfg.DowngradeEmbedFS = &downgradeFS
+	if cfg.SqlEmbedFS == nil {
+		cfg.SqlEmbedFS = &upgradeFS
+	}
+	if cfg.DowngradeEmbedFS == nil {
+		cfg.DowngradeEmbedFS = &downgradeFS
+	}
 	return harmonyquery.NewFromConfig(cfg)
 }
 
@@ -32,7 +36,12 @@ func envElse(env, els string) string {
 	return els
 }
 
-func NewFromConfigWithITestID(t *testing.T, id harmonyquery.ITestID) (*DB, error) {
+func NewFromConfigWithITestID(t *testing.T, id harmonyquery.ITestID, fullMigrations bool) (*DB, error) {
+	sqlFS := &testUpgradeFS
+	if fullMigrations {
+		sqlFS = &upgradeFS
+	}
+
 	db, err := NewFromConfig(Config{
 		Hosts:            []string{envElse(harmonyquery.DefaultHostEnv, "127.0.0.1")},
 		Database:         "yugabyte",
@@ -41,7 +50,7 @@ func NewFromConfigWithITestID(t *testing.T, id harmonyquery.ITestID) (*DB, error
 		Port:             "5433",
 		LoadBalance:      false,
 		ITestID:          id,
-		SqlEmbedFS:       &upgradeFS,
+		SqlEmbedFS:       sqlFS,
 		DowngradeEmbedFS: &downgradeFS,
 	})
 	if err != nil {
@@ -59,6 +68,9 @@ var upgradeFS embed.FS
 //go:embed downgrade
 var downgradeFS embed.FS
 
+//go:embed sql/20230706-itest_scratch.sql
+var testUpgradeFS embed.FS
+
 // A function for clean, idempotent SQL upgrade testing.
 var ITestUpgradeFunc = harmonyquery.ITestUpgradeFunc
 

itests/alertnow_test.go

@@ -21,7 +21,7 @@ func TestAlertNow(t *testing.T) {
 	}
 	// Create dependencies
 	sharedITestID := harmonydb.ITestNewID()
-	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
+	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID, true)
 	require.NoError(t, err)
 
 	an := alertmanager.NewAlertNow(db, "alertNowMachine")

itests/curio_test.go

@@ -76,7 +76,7 @@ func TestCurioHappyPath(t *testing.T) {
 	sharedITestID := harmonydb.ITestNewID()
 	t.Logf("sharedITestID: %s", sharedITestID)
 
-	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
+	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID, true)
 	require.NoError(t, err)
 
 	defer db.ITestDeleteAll()

itests/dyncfg_test.go

@@ -17,7 +17,7 @@ func TestDynamicConfig(t *testing.T) {
 	defer cancel()
 
 	sharedITestID := harmonydb.ITestNewID()
-	cdb, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
+	cdb, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID, true)
 	require.NoError(t, err)
 
 	databaseContents := &config.CurioConfig{

itests/harmonydb_test.go

@@ -20,7 +20,7 @@ func TestCrud(t *testing.T) {
 	defer cancel()
 
 	sharedITestID := harmonydb.ITestNewID()
-	cdb, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
+	cdb, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID, false)
 	require.NoError(t, err)
 
 	//cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
@@ -52,7 +52,7 @@ func TestTransaction(t *testing.T) {
 	defer cancel()
 
 	testID := harmonydb.ITestNewID()
-	cdb, err := harmonydb.NewFromConfigWithITestID(t, testID)
+	cdb, err := harmonydb.NewFromConfigWithITestID(t, testID, false)
 	require.NoError(t, err)
 	_, err = cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)")
 	require.NoError(t, err)
@@ -116,7 +116,7 @@ func TestPartialWalk(t *testing.T) {
 	defer cancel()
 
 	testID := harmonydb.ITestNewID()
-	cdb, err := harmonydb.NewFromConfigWithITestID(t, testID)
+	cdb, err := harmonydb.NewFromConfigWithITestID(t, testID, false)
 	require.NoError(t, err)
 	_, err = cdb.Exec(ctx, `
 			INSERT INTO 
@@ -161,7 +161,7 @@ func TestDowngradeTo(t *testing.T) {
 	defer cancel()
 
 	testID := harmonydb.ITestNewID()
-	cdb, err := harmonydb.NewFromConfigWithITestID(t, testID)
+	cdb, err := harmonydb.NewFromConfigWithITestID(t, testID, true)
 	require.NoError(t, err)
 
 	// The setup: lets make revert files going forward in time, but ignore the past.

itests/market_deal_dynamic_test.go

@@ -79,7 +79,7 @@ func TestMarketDealDynamicMinerUpdate(t *testing.T) {
 	sharedITestID := harmonydb.ITestNewID()
 	t.Logf("sharedITestID: %s", sharedITestID)
 
-	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
+	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID, true)
 	require.NoError(t, err)
 
 	defer db.ITestDeleteAll()
@@ -443,7 +443,7 @@ func TestMarketDealSystemBasic(t *testing.T) {
 	sharedITestID := harmonydb.ITestNewID()
 	t.Logf("sharedITestID: %s", sharedITestID)
 
-	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
+	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID, true)
 	require.NoError(t, err)
 	defer db.ITestDeleteAll()
 

itests/sql_idempotent_test.go

@@ -21,7 +21,7 @@ func TestSQLIdempotent(t *testing.T) {
 	}
 
 	testID := harmonydb.ITestNewID()
-	cdb, err := harmonydb.NewFromConfigWithITestID(t, testID)
+	cdb, err := harmonydb.NewFromConfigWithITestID(t, testID, true)
 	require.NoError(t, err)
 
 	ctx := context.Background()

lib/paths/local_test.go

@@ -84,7 +84,7 @@ func TestLocalStorage(t *testing.T) {
 
 	sharedITestID := harmonydb.ITestNewID()
 
-	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
+	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID, true)
 	require.NoError(t, err)
 
 	index := NewDBIndex(nil, db)

lib/paths/remote_test.go

@@ -61,7 +61,7 @@ func TestMoveShared(t *testing.T) {
 
 	sharedITestID := harmonydb.ITestNewID()
 
-	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
+	db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID, true)
 	require.NoError(t, err)
 
 	index := paths.NewDBIndex(nil, db)