Add disk status checking and NPM cache purging functionality (#1329)

Author: ije

internal/storage/storage_s3.go

@@ -107,7 +107,7 @@ func (s3 *s3Storage) Stat(name string) (stat Stat, err error) {
 	if name == "" {
 		return nil, errors.New("name is required")
 	}
-	if s3.fsCache != nil && !strings.HasSuffix(name, ".mjs.map") {
+	if s3.shouldUseFSCache(name) {
 		stat, err = s3.fsCache.Stat(name)
 		if err == nil {
 			return
@@ -160,7 +160,7 @@ func (s3 *s3Storage) Get(name string) (content io.ReadCloser, stat Stat, err err
 	if name == "" {
 		return nil, nil, errors.New("name is required")
 	}
-	if s3.fsCache != nil && !strings.HasSuffix(name, ".mjs.map") {
+	if s3.shouldUseFSCache(name) {
 		content, stat, err = s3.fsCache.Get(name)
 		if err == nil {
 			return
@@ -206,7 +206,7 @@ func (s3 *s3Storage) Get(name string) (content io.ReadCloser, stat Stat, err err
 		contentLength: size,
 		lastModified:  lastModified,
 	}
-	if s3.fsCache != nil && !strings.HasSuffix(name, ".mjs.map") {
+	if s3.shouldUseFSCache(name) {
 		pr, pw := io.Pipe()
 		go func() {
 			unlock := s3.fsCacheLock.Lock(name)
@@ -267,7 +267,7 @@ func (s3 *s3Storage) Put(name string, content io.Reader) (err error) {
 		err = errors.New("missing content length")
 		return
 	}
-	if s3.fsCache != nil && !strings.HasSuffix(name, ".mjs.map") {
+	if s3.shouldUseFSCache(name) {
 		pr, pw := io.Pipe()
 		go func(content io.Reader) {
 			unlock := s3.fsCacheLock.Lock(name)
@@ -295,7 +295,7 @@ func (s3 *s3Storage) Delete(name string) (err error) {
 	if name == "" {
 		return errors.New("key is required")
 	}
-	if s3.fsCache != nil && !strings.HasSuffix(name, ".mjs.map") {
+	if s3.shouldUseFSCache(name) {
 		go s3.fsCache.Delete(name)
 	}
 	req, _ := http.NewRequest("DELETE", s3.apiEndpoint+"/"+name, nil)
@@ -417,6 +417,16 @@ func (s3 *s3Storage) sign(req *http.Request) {
 	req.Header.Set("Authorization", strings.Join([]string{"AWS4-HMAC-SHA256 Credential=" + s3.accessKeyID + "/" + scope, "SignedHeaders=" + strings.Join(signedHeaders, ";"), "Signature=" + toHex(signature)}, ", "))
 }
 
+func (s3 *s3Storage) shouldUseFSCache(name string) bool {
+	if s3.fsCache == nil {
+		return false
+	}
+	if strings.HasSuffix(name, ".mjs.map") || strings.HasSuffix(name, ".d.ts") || strings.HasSuffix(name, ".d.mts") || strings.HasSuffix(name, ".d.cts") {
+		return false
+	}
+	return true
+}
+
 func parseS3Error(resp *http.Response) error {
 	var s3Error s3Error
 	if xml.NewDecoder(resp.Body).Decode(&s3Error) != nil || s3Error.Code == "" {

server/disk.go

@@ -0,0 +1,67 @@
+package server
+
+import (
+	"os"
+	"sync"
+	"syscall"
+
+	"github.com/ije/gox/log"
+)
+
+type DiskStatus uint8
+
+const (
+	DiskStatusError DiskStatus = iota
+	DiskStatusOk
+	DiskStatusLow
+	DiskStatusFull
+)
+
+// npmStorePurgeLock serializes purge operations on the npm store to avoid
+// concurrent rename/remove races within this process.
+var npmStorePurgeLock sync.Mutex
+
+func purgeNPMCacheWhenDiskIsLowOrFull(npmrc *NpmRC, logger *log.Logger) {
+	if status := checkDiskStatus(); status == DiskStatusOk || status == DiskStatusError {
+		return
+	}
+
+	npmStorePurgeLock.Lock()
+	defer npmStorePurgeLock.Unlock()
+
+	npmDir := npmrc.StoreDir()
+	oldDir := npmDir + "_old"
+
+	// Ensure any previous backup directory is removed so that the rename is idempotent.
+	if err := os.RemoveAll(oldDir); err != nil {
+		logger.Errorf("failed to remove previous npm cache backup directory %s: %v", oldDir, err)
+	}
+
+	if err := os.Rename(npmDir, oldDir); err != nil {
+		// If the directory does not exist, there's nothing to purge.
+		if !os.IsNotExist(err) {
+			logger.Errorf("failed to rename npm cache directory %s to %s: %v", npmDir, oldDir, err)
+		}
+		return
+	}
+
+	if err := os.RemoveAll(oldDir); err != nil {
+		logger.Errorf("failed to remove npm cache directory %s: %v", oldDir, err)
+	}
+}
+
+func checkDiskStatus() DiskStatus {
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(config.WorkDir, &stat)
+	if err == nil {
+		avail := stat.Bavail * uint64(stat.Bsize)
+		if avail < 100*MB {
+			return DiskStatusFull
+		} else if avail < 1024*MB {
+			return DiskStatusLow
+		}
+	} else {
+		return DiskStatusError
+	}
+	return DiskStatusOk
+}

server/router.go

@@ -17,7 +17,6 @@ import (
 	"slices"
 	"sort"
 	"strings"
-	"syscall"
 	"time"
 
 	"github.com/esm-dev/esm.sh/internal/fetch"
@@ -80,6 +79,18 @@ func esmRouter(esmStorage storage.Storage, logger *log.Logger) rex.Handle {
 		}
 	}
 
+	// purge npm cache when disk is low or full
+	go func() {
+		// run an initial check before waiting for the first ticker event
+		purgeNPMCacheWhenDiskIsLowOrFull(npmrc, logger)
+
+		ticker := time.NewTicker(1 * time.Hour)
+		defer ticker.Stop()
+		for range ticker.C {
+			go purgeNPMCacheWhenDiskIsLowOrFull(npmrc, logger)
+		}
+	}()
+
 	return func(ctx *rex.Context) any {
 		pathname := ctx.R.URL.Path
 
@@ -288,16 +299,12 @@ func esmRouter(esmStorage storage.Storage, logger *log.Logger) rex.Handle {
 			}
 
 			diskStatus := "ok"
-			var stat syscall.Statfs_t
-			err := syscall.Statfs(config.WorkDir, &stat)
-			if err == nil {
-				avail := stat.Bavail * uint64(stat.Bsize)
-				if avail < 100*MB {
-					diskStatus = "full"
-				} else if avail < 1024*MB {
-					diskStatus = "low"
-				}
-			} else {
+			switch checkDiskStatus() {
+			case DiskStatusFull:
+				diskStatus = "full"
+			case DiskStatusLow:
+				diskStatus = "low"
+			case DiskStatusError:
 				diskStatus = "error"
 			}