feat: load packages lazily

gno.land/pkg/integration/testdata/addpkg_outofgas.txtar

@@ -7,19 +7,19 @@ gnoland start
 gnokey maketx addpkg -pkgdir $WORK/foo -pkgpath gno.land/r/foo -gas-fee 1000000ugnot -gas-wanted 220000 -broadcast -chainid=tendermint_test test1
 
 
-# add bar package - out of gas at store.GetPackage() with gas 60000
+# add bar package - out of gas at store.GetPackage() with gas 70000
 
-! gnokey maketx addpkg -pkgdir $WORK/bar -pkgpath gno.land/r/bar -gas-fee 1000000ugnot -gas-wanted 60000 -broadcast -chainid=tendermint_test test1
+! gnokey maketx addpkg -pkgdir $WORK/bar -pkgpath gno.land/r/bar -gas-fee 1000000ugnot -gas-wanted 70000 -broadcast -chainid=tendermint_test test1
 
 stderr '--= Error =--'
 stderr 'Data: out of gas error'
 stderr '--= /Error =--'
 
 
 
-# out of gas at store.store.GetTypeSafe()  with gas 63000
+# out of gas at store.store.GetTypeSafe()  with gas 73000
 
-! gnokey maketx addpkg -pkgdir $WORK/bar -pkgpath gno.land/r/bar -gas-fee 1000000ugnot -gas-wanted 63000 -broadcast -chainid=tendermint_test test1
+! gnokey maketx addpkg -pkgdir $WORK/bar -pkgpath gno.land/r/bar -gas-fee 1000000ugnot -gas-wanted 73000 -broadcast -chainid=tendermint_test test1
 
 stderr '--= Error =--'
 stderr 'Data: out of gas error'

gno.land/pkg/integration/testdata/restart.txtar

@@ -4,12 +4,12 @@
 loadpkg gno.land/r/demo/counter $WORK
 gnoland start
 
-gnokey maketx call -pkgpath gno.land/r/demo/counter -func Incr -gas-fee 1000000ugnot -gas-wanted 200000 -broadcast -chainid tendermint_test test1
+gnokey maketx call -pkgpath gno.land/r/demo/counter -func Incr -gas-fee 2000000ugnot -gas-wanted 350000 -broadcast -chainid tendermint_test test1
 stdout '\(1 int\)'
 
 gnoland restart
 
-gnokey maketx call -pkgpath gno.land/r/demo/counter -func Incr -gas-fee 1000000ugnot -gas-wanted 200000 -broadcast -chainid tendermint_test test1
+gnokey maketx call -pkgpath gno.land/r/demo/counter -func Incr -gas-fee 2000000ugnot -gas-wanted 350000 -broadcast -chainid tendermint_test test1
 stdout '\(2 int\)'
 
 -- counter.gno --

gno.land/pkg/integration/testdata/simulate_gas.txtar

@@ -5,11 +5,11 @@ loadpkg gno.land/r/simulate $WORK/simulate
 gnoland start
 
 # simulate only
-gnokey maketx call -pkgpath gno.land/r/simulate -func Hello -gas-fee 1000000ugnot -gas-wanted 2000000 -broadcast -chainid=tendermint_test -simulate only test1
+gnokey maketx call -pkgpath gno.land/r/simulate -func Hello -gas-fee 2000000ugnot -gas-wanted 2200000 -broadcast -chainid=tendermint_test -simulate only test1
 stdout 'GAS USED:   99371'
 
 # simulate skip
-gnokey maketx call -pkgpath gno.land/r/simulate -func Hello -gas-fee 1000000ugnot -gas-wanted 2000000 -broadcast -chainid=tendermint_test -simulate skip test1
+gnokey maketx call -pkgpath gno.land/r/simulate -func Hello -gas-fee 2000000ugnot -gas-wanted 2200000 -broadcast -chainid=tendermint_test -simulate skip test1
 stdout 'GAS USED:   99371' # same as simulate only
 
 

gnovm/pkg/gnolang/machine.go

@@ -224,35 +224,65 @@
 //----------------------------------------
 // top level Run* methods.
 
-// Upon restart, preprocess all MemPackage and save blocknodes.
-// This is a temporary measure until we optimize/make-lazy.
+// PreprocessAllFilesAndSaveBlockNodes prepares the store for lazy loading
+// by registering packages in the store but not fully preprocessing them.
 //
 // NOTE: package paths not beginning with gno.land will be allowed to override,
 // to support cases of stdlibs processed through [RunMemPackagesWithOverrides].
 func (m *Machine) PreprocessAllFilesAndSaveBlockNodes() {
-	ch := m.Store.IterMemPackage()
-	for memPkg := range ch {
+	// Set up a PackageGetter that will do the actual preprocessing when needed
+	pkgGetter := func(pkgPath string, store Store) (*PackageNode, *PackageValue) {
+		// Get the original MemPackage
+		memPkg := store.GetMemPackage(pkgPath)
+		if memPkg == nil {
+			return nil, nil
+		}
+
+		// Do the actual preprocessing when package is requested
 		fset := ParseMemPackage(memPkg)
 		pn := NewPackageNode(Name(memPkg.Name), memPkg.Path, fset)
-		m.Store.SetBlockNode(pn)
-		PredefineFileSet(m.Store, pn, fset)
+		store.SetBlockNode(pn)
+		PredefineFileSet(store, pn, fset)
+
 		for _, fn := range fset.Files {
-			// Save Types to m.Store (while preprocessing).
-			fn = Preprocess(m.Store, pn, fn).(*FileNode)
-			// Save BlockNodes to m.Store.
-			SaveBlockNodes(m.Store, fn)
+			// Save Types to store (while preprocessing)
+			fn = Preprocess(store, pn, fn).(*FileNode)
+			// Save BlockNodes to store
+			SaveBlockNodes(store, fn)
 		}
+
 		// Normally, the fileset would be added onto the
 		// package node only after runFiles(), but we cannot
 		// run files upon restart (only preprocess them).
 		// So, add them here instead.
 		// TODO: is this right?
+
 		if pn.FileSet == nil {
 			pn.FileSet = fset
 		} else {
 			// This happens for non-realm file tests.
 			// TODO ensure the files are the same.
 		}
+
+		pv := pn.NewPackage()
+		store.SetObject(pv)
+		store.SetCachePackage(pv)
+		store.SetBlockNode(pn)
+
+		return pn, pv
+	}
+
+	// Register the PackageGetter with the store
+	m.Store.SetPackageGetter(pkgGetter)
+
+	// Register unprocessed packages in store
+	ch := m.Store.IterMemPackage()
+	for memPkg := range ch {
+		if memPkg == nil {
+			continue
+		}
+		// Just add the package to the store without preprocessing
+		m.Store.AddMemPackage(memPkg)
 	}
 }