[CI run] AutoGC and Archive as defaults

go/cmd/dolt/cli/arg_parser_helpers.go

@@ -335,7 +335,7 @@ func CreateGCArgParser() *argparser.ArgParser {
 	ap := argparser.NewArgParserWithMaxArgs("gc", 0)
 	ap.SupportsFlag(ShallowFlag, "s", "perform a fast, but incomplete garbage collection pass")
 	ap.SupportsFlag(FullFlag, "f", "perform a full garbage collection, including the old generation")
-	ap.SupportsInt(ArchiveLevelParam, "", "archive compression level", "Specify the archive compression level garbage collection results. Default is 0. Max is 1")
+	ap.SupportsInt(ArchiveLevelParam, "", "archive compression level", "Specify the archive compression level garbage collection results. Default is 1, Disable with 0")
 	return ap
 }
 

go/cmd/dolt/commands/admin/archive_inspect.go

@@ -112,12 +112,11 @@ func (cmd ArchiveInspectCmd) Exec(ctx context.Context, commandStr string, args [
 	cli.Printf("File size: %d bytes\n", inspector.FileSize())
 	cli.Printf("Format version: %d\n", inspector.FormatVersion())
 	cli.Printf("File signature: %s\n", inspector.FileSignature())
-	cli.Println()
-
 	cli.Printf("Chunk count: %d\n", inspector.ChunkCount())
 	cli.Printf("Byte span count: %d\n", inspector.ByteSpanCount())
 	cli.Printf("Index size: %d bytes\n", inspector.IndexSize())
 	cli.Printf("Metadata size: %d bytes\n", inspector.MetadataSize())
+	cli.Printf("Split offset: %d bytes\n", inspector.SplitOffset())
 
 	// Display metadata if present
 	if inspector.MetadataSize() > 0 {

go/cmd/dolt/commands/engine/sqlengine.go

@@ -226,11 +226,12 @@ func NewSqlEngine(
 		// XXX: We force session aware safepoint controller if auto_gc is on.
 		dprocedures.UseSessionAwareSafepointController = true
 		sql.SystemVariables.AssignValues(map[string]interface{}{
+			// NM4 - not sure about this one.
 			dsess.DoltAutoGCEnabled: int8(1),
 		})
 	} else {
 		sql.SystemVariables.AssignValues(map[string]interface{}{
-			dsess.DoltAutoGCEnabled: int8(0),
+			dsess.DoltAutoGCEnabled: int8(1),
 		})
 	}
 

go/cmd/dolt/commands/gc.go

@@ -127,7 +127,7 @@ func (cmd GarbageCollectionCmd) constructDoltGCQuery(apr *argparser.ArgParseResu
 		extraFlag = "--full"
 	}
 
-	archiveLevel := chunks.NoArchive
+	archiveLevel := chunks.SimpleArchive
 	if apr.Contains(cli.ArchiveLevelParam) {
 		lvl, ok := apr.GetInt(cli.ArchiveLevelParam)
 		if !ok {

go/cmd/dolt/commands/sqlserver/server_test.go

@@ -640,8 +640,8 @@ func TestGenerateYamlConfig(t *testing.T) {
   # dolt_transaction_commit: false
   # event_scheduler: "OFF"
   # auto_gc_behavior:
-    # enable: false
-    # archive_level: 0
+    # enable: true
+    # archive_level: 1
 
 listener:
   # host: localhost

go/cmd/dolt/commands/utils.go

@@ -266,7 +266,7 @@ func newLateBindingEngine(
 			// to include auto-gc log lines.
 			nullLgr := logrus.New()
 			nullLgr.SetOutput(io.Discard)
-			config.AutoGCController = sqle.NewAutoGCController(chunks.NoArchive, nullLgr)
+			config.AutoGCController = sqle.NewAutoGCController(chunks.SimpleArchive, nullLgr)
 		}
 
 		se, err := engine.NewSqlEngine(

go/libraries/doltcore/env/actions/commitwalk/commitwalk_test.go

@@ -64,7 +64,7 @@ func createUninitializedEnv() *env.DoltEnv {
 func TestGetDotDotRevisions(t *testing.T) {
 	ctx := context.Background()
 	dEnv := createUninitializedEnv()
-	err := dEnv.InitRepo(context.Background(), types.Format_Default, "Bill Billerson", "[email protected]", env.DefaultInitBranch)
+	err := dEnv.InitRepo(context.Background(), types.Format_DOLT, "Bill Billerson", "[email protected]", env.DefaultInitBranch)
 	require.NoError(t, err)
 
 	cs, err := doltdb.NewCommitSpec(env.DefaultInitBranch)

go/libraries/doltcore/remotesrv/grpc.go

@@ -425,6 +425,7 @@ func (rs *RemoteChunkStore) getUploadUrl(md metadata.MD, repoPath string, tfd *r
 	fileID := hash.New(tfd.Id).String() + tfd.Suffix
 	params := url.Values{}
 	params.Add("num_chunks", strconv.Itoa(int(tfd.NumChunks)))
+	params.Add("split_offset", strconv.Itoa(int(tfd.SplitOffset)))
 	params.Add("content_length", strconv.Itoa(int(tfd.ContentLength)))
 	params.Add("content_hash", base64.RawURLEncoding.EncodeToString(tfd.ContentHash))
 	return &url.URL{

go/libraries/doltcore/remotesrv/http.go

@@ -140,21 +140,22 @@ func (fh filehandler) ServeHTTP(respWr http.ResponseWriter, req *http.Request) {
 			respWr.WriteHeader(http.StatusBadRequest)
 			return
 		}
-		num_chunks, err := strconv.Atoi(ncs)
+		numChunks, err := strconv.Atoi(ncs)
 		if err != nil {
 			logger = logger.WithField("status", http.StatusBadRequest)
 			logger.WithError(err).Warn("bad request: num_chunks parameter did not parse")
 			respWr.WriteHeader(http.StatusBadRequest)
 			return
 		}
+
 		cls := q.Get("content_length")
 		if cls == "" {
 			logger = logger.WithField("status", http.StatusBadRequest)
 			logger.Warn("bad request: content_length parameter not provided")
 			respWr.WriteHeader(http.StatusBadRequest)
 			return
 		}
-		content_length, err := strconv.Atoi(cls)
+		contentLength, err := strconv.Atoi(cls)
 		if err != nil {
 			logger = logger.WithField("status", http.StatusBadRequest)
 			logger.WithError(err).Warn("bad request: content_length parameter did not parse")
@@ -168,15 +169,28 @@ func (fh filehandler) ServeHTTP(respWr http.ResponseWriter, req *http.Request) {
 			respWr.WriteHeader(http.StatusBadRequest)
 			return
 		}
-		content_hash, err := base64.RawURLEncoding.DecodeString(chs)
+		contentHash, err := base64.RawURLEncoding.DecodeString(chs)
 		if err != nil {
 			logger = logger.WithField("status", http.StatusBadRequest)
 			logger.WithError(err).Warn("bad request: content_hash parameter did not parse")
 			respWr.WriteHeader(http.StatusBadRequest)
 			return
 		}
 
-		logger, statusCode = writeTableFile(req.Context(), logger, fh.dbCache, filepath, file, num_chunks, content_hash, uint64(content_length), req.Body)
+		// splitOffset is not required to allow for backwards compatibility with older clients.
+		splitOffset := uint64(0)
+		splitQstr := q.Get("split_offset")
+		if splitQstr != "" {
+			splitOffset, err = strconv.ParseUint(splitQstr, 10, 64)
+			if err != nil {
+				logger = logger.WithField("status", http.StatusBadRequest)
+				logger.WithError(err).Warn("bad request: split_offset parameter did not parse")
+				respWr.WriteHeader(http.StatusBadRequest)
+				return
+			}
+		}
+
+		logger, statusCode = writeTableFile(req.Context(), logger, fh.dbCache, filepath, file, splitOffset, numChunks, contentHash, uint64(contentLength), req.Body)
 	}
 
 	if statusCode != -1 {
@@ -286,7 +300,7 @@ func (u *uploadreader) Close() error {
 	return nil
 }
 
-func writeTableFile(ctx context.Context, logger *logrus.Entry, dbCache DBCache, path, fileId string, numChunks int, contentHash []byte, contentLength uint64, body io.ReadCloser) (*logrus.Entry, int) {
+func writeTableFile(ctx context.Context, logger *logrus.Entry, dbCache DBCache, path, fileId string, splitOffset uint64, numChunks int, contentHash []byte, contentLength uint64, body io.ReadCloser) (*logrus.Entry, int) {
 	if !validateFileName(fileId) {
 		logger = logger.WithField("status", http.StatusBadRequest)
 		logger.Warnf("%s is not a valid hash", fileId)
@@ -300,7 +314,7 @@ func writeTableFile(ctx context.Context, logger *logrus.Entry, dbCache DBCache,
 		return logger, http.StatusInternalServerError
 	}
 
-	err = cs.WriteTableFile(ctx, fileId, numChunks, contentHash, func() (io.ReadCloser, uint64, error) {
+	err = cs.WriteTableFile(ctx, fileId, splitOffset, numChunks, contentHash, func() (io.ReadCloser, uint64, error) {
 		reader := body
 		size := contentLength
 		return &uploadreader{

go/libraries/doltcore/remotestorage/chunk_store.go

@@ -915,19 +915,21 @@ func (dcs *DoltChunkStore) uploadChunks(ctx context.Context, hashToChunk map[has
 		chnks = append(chnks, ch)
 	}
 
+	hashToSplitOffset := make(map[hash.Hash]uint64)
 	hashToCount := make(map[hash.Hash]int)
 	hashToData := make(map[hash.Hash][]byte)
 	hashToContentHash := make(map[hash.Hash][]byte)
 
 	// structuring so this can be done as multiple files in the future.
 	{
-		name, data, err := nbs.WriteChunks(chnks)
+		name, data, splitOffset, err := nbs.WriteChunks(chnks)
 
 		if err != nil {
 			return map[hash.Hash]int{}, err
 		}
 
 		h := hash.Parse(name)
+		hashToSplitOffset[h] = splitOffset
 		hashToData[h] = data
 		hashToCount[h] = len(chnks)
 
@@ -938,7 +940,7 @@ func (dcs *DoltChunkStore) uploadChunks(ctx context.Context, hashToChunk map[has
 	for h, contentHash := range hashToContentHash {
 		// Tables created on this path are always starting from memory tables and ending up as noms table files.
 		// As a result, the suffix is always empty.
-		err := dcs.uploadTableFileWithRetries(ctx, h, "", uint64(hashToCount[h]), contentHash, func() (io.ReadCloser, uint64, error) {
+		err := dcs.uploadTableFileWithRetries(ctx, h, "", hashToSplitOffset[h], uint64(hashToCount[h]), contentHash, func() (io.ReadCloser, uint64, error) {
 			data := hashToData[h]
 			return io.NopCloser(bytes.NewReader(data)), uint64(len(data)), nil
 		})
@@ -950,7 +952,7 @@ func (dcs *DoltChunkStore) uploadChunks(ctx context.Context, hashToChunk map[has
 	return hashToCount, nil
 }
 
-func (dcs *DoltChunkStore) uploadTableFileWithRetries(ctx context.Context, tableFileId hash.Hash, suffix string, numChunks uint64, tableFileContentHash []byte, getContent func() (io.ReadCloser, uint64, error)) error {
+func (dcs *DoltChunkStore) uploadTableFileWithRetries(ctx context.Context, tableFileId hash.Hash, suffix string, splitOffset uint64, numChunks uint64, tableFileContentHash []byte, getContent func() (io.ReadCloser, uint64, error)) error {
 	op := func() error {
 		body, contentLength, err := getContent()
 		if err != nil {
@@ -963,6 +965,7 @@ func (dcs *DoltChunkStore) uploadTableFileWithRetries(ctx context.Context, table
 			ContentHash:   tableFileContentHash,
 			NumChunks:     numChunks,
 			Suffix:        suffix,
+			SplitOffset:   splitOffset,
 		}
 
 		dcs.logf("getting upload location for file %s", tableFileId.String())
@@ -1066,15 +1069,15 @@ func (dcs *DoltChunkStore) SupportedOperations() chunks.TableFileStoreOps {
 }
 
 // WriteTableFile reads a table file from the provided reader and writes it to the chunk store.
-func (dcs *DoltChunkStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) error {
+func (dcs *DoltChunkStore) WriteTableFile(ctx context.Context, fileId string, splitOffset uint64, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) error {
 	suffix := ""
 	if strings.HasSuffix(fileId, nbs.ArchiveFileSuffix) {
 		suffix = nbs.ArchiveFileSuffix
 		fileId = strings.TrimSuffix(fileId, nbs.ArchiveFileSuffix)
 	}
 
 	fileIdBytes := hash.Parse(fileId)
-	return dcs.uploadTableFileWithRetries(ctx, fileIdBytes, suffix, uint64(numChunks), contentHash, getRd)
+	return dcs.uploadTableFileWithRetries(ctx, fileIdBytes, suffix, splitOffset, uint64(numChunks), contentHash, getRd)
 }
 
 // AddTableFilesToManifest adds table files to the manifest
@@ -1195,6 +1198,10 @@ func (drtf DoltRemoteTableFile) NumChunks() int {
 	return int(drtf.info.NumChunks)
 }
 
+func (drtf DoltRemoteTableFile) SplitOffset() uint64 {
+	return drtf.info.SplitOffset
+}
+
 var ErrRemoteTableFileGet = errors.New("HTTP GET for remote table file failed")
 
 func sanitizeSignedUrl(url string) string {

go/libraries/doltcore/servercfg/serverconfig.go

@@ -59,7 +59,7 @@ const (
 	DefaultLogLevel                  = LogLevel_Info
 	DefaultLogFormat                 = LogFormat_Text
 	DefaultAutoCommit                = true
-	DefaultAutoGCBehaviorEnable      = false
+	DefaultAutoGCBehaviorEnable      = true
 	DefaultDoltTransactionCommit     = false
 	DefaultMaxConnections            = 1000
 	DefaultMaxWaitConnections        = 50
@@ -75,7 +75,7 @@ const (
 	DefaultMySQLUnixSocketFilePath   = "/tmp/mysql.sock"
 	DefaultMaxLoggedQueryLen         = 0
 	DefaultEncodeLoggedQuery         = false
-	DefaultCompressionLevel          = 0
+	DefaultCompressionLevel          = 1
 )
 
 func ptr[T any](t T) *T {

go/libraries/doltcore/servercfg/yaml_config.go

@@ -1042,14 +1042,14 @@ type AutoGCBehaviorYAMLConfig struct {
 
 func (a *AutoGCBehaviorYAMLConfig) Enable() bool {
 	if a.Enable_ == nil {
-		return false
+		return true
 	}
 	return *a.Enable_
 }
 
 func (a *AutoGCBehaviorYAMLConfig) ArchiveLevel() int {
 	if a.ArchiveLevel_ == nil {
-		return 0
+		return 1
 	}
 	return *a.ArchiveLevel_
 }

go/libraries/doltcore/servercfg/yaml_config_test.go

@@ -35,8 +35,8 @@ behavior:
     disable_client_multi_statements: false
     event_scheduler: ON
     auto_gc_behavior:
-        enable: false
-        archive_level: 0
+        enable: true
+        archive_level: 1
 
 listener:
     host: localhost