diff --git a/commands/commandeer.go b/commands/commandeer.go index c53235cefe5..ffa27f0bec6 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -365,7 +365,11 @@ func (r *rootCommand) Name() string { } func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { - b := newHugoBuilder(r, nil) + var vendor bool + if vendorCmd, ok := cd.Command.(vendoredCommand); ok { + vendor = vendorCmd.IsVendorCommand() + } + b := newHugoBuilder(r, nil, vendor) if !r.buildWatch { defer b.postBuild("Total", time.Now()) diff --git a/commands/commands.go b/commands/commands.go index 10ab106e277..d5b9896dc69 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -23,7 +23,8 @@ import ( func newExec() (*simplecobra.Exec, error) { rootCmd := &rootCommand{ commands: []simplecobra.Commander{ - newHugoBuildCmd(), + newHugoBuildCmd(false), + newHugoBuildCmd(true), newVersionCmd(), newEnvCommand(), newServerCommand(), @@ -42,13 +43,16 @@ func newExec() (*simplecobra.Exec, error) { return simplecobra.New(rootCmd) } -func newHugoBuildCmd() simplecobra.Commander { - return &hugoBuildCommand{} +func newHugoBuildCmd(vendor bool) simplecobra.Commander { + return &hugoBuildCommand{ + vendor: vendor, + } } // hugoBuildCommand just delegates to the rootCommand. type hugoBuildCommand struct { rootCmd *rootCommand + vendor bool } func (c *hugoBuildCommand) Commands() []simplecobra.Commander { @@ -56,12 +60,23 @@ func (c *hugoBuildCommand) Commands() []simplecobra.Commander { } func (c *hugoBuildCommand) Name() string { + if c.vendor { + return "vendor" + } return "build" } +type vendoredCommand interface { + IsVendorCommand() bool +} + +func (c *hugoBuildCommand) IsVendorCommand() bool { + return c.vendor +} + func (c *hugoBuildCommand) Init(cd *simplecobra.Commandeer) error { c.rootCmd = cd.Root.Command.(*rootCommand) - return c.rootCmd.initRootCommand("build", cd) + return c.rootCmd.initRootCommand(c.Name(), cd) } func (c *hugoBuildCommand) PreRun(cd, runner *simplecobra.Commandeer) error { diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go index 4c2d865c071..6ed37750446 100644 --- a/commands/hugobuilder.go +++ b/commands/hugobuilder.go @@ -73,6 +73,8 @@ type hugoBuilder struct { showErrorInBrowser bool errState hugoBuilderErrState + + vendor bool } var errConfigNotSet = errors.New("config not set") @@ -1046,11 +1048,11 @@ func (c *hugoBuilder) loadConfig(cd *simplecobra.Commandeer, running bool) error } } cfg.Set("environment", c.r.environment) - cfg.Set("internal", maps.Params{ "running": running, "watch": watch, "verbose": c.r.isVerbose(), + "vendor": c.vendor, "fastRenderMode": c.fastRenderMode, }) diff --git a/commands/server.go b/commands/server.go index 08ecd5bac7d..a8d4c9eda40 100644 --- a/commands/server.go +++ b/commands/server.go @@ -84,7 +84,7 @@ const ( configChangeGoWork = "go work file" ) -func newHugoBuilder(r *rootCommand, s *serverCommand, onConfigLoaded ...func(reloaded bool) error) *hugoBuilder { +func newHugoBuilder(r *rootCommand, s *serverCommand, vendor bool, onConfigLoaded ...func(reloaded bool) error) *hugoBuilder { var visitedURLs *types.EvictingQueue[string] if s != nil && !s.disableFastRender { visitedURLs = types.NewEvictingQueue[string](20) @@ -92,6 +92,7 @@ func newHugoBuilder(r *rootCommand, s *serverCommand, onConfigLoaded ...func(rel return &hugoBuilder{ r: r, s: s, + vendor: vendor, visitedURLs: visitedURLs, fullRebuildSem: semaphore.NewWeighted(1), debounce: debounce.New(4 * time.Second), @@ -563,6 +564,7 @@ func (c *serverCommand) PreRun(cd, runner *simplecobra.Commandeer) error { c.hugoBuilder = newHugoBuilder( c.r, c, + false, func(reloaded bool) error { if !reloaded { if err := c.createServerPorts(cd); err != nil { diff --git a/common/hugio/writers.go b/common/hugio/writers.go index 6f439cc8b00..322fd3c681a 100644 --- a/common/hugio/writers.go +++ b/common/hugio/writers.go @@ -50,6 +50,17 @@ func NewMultiWriteCloser(writeClosers ...io.WriteCloser) io.WriteCloser { return multiWriteCloser{Writer: io.MultiWriter(writers...), closers: writeClosers} } +// NewWriteCloser creates a new io.WriteCloser with the given writer and closer. +func NewWriteCloser(w io.Writer, closer io.Closer) io.WriteCloser { + return struct { + io.Writer + io.Closer + }{ + w, + closer, + } +} + // ToWriteCloser creates an io.WriteCloser from the given io.Writer. // If it's not already, one will be created with a Close method that does nothing. func ToWriteCloser(w io.Writer) io.WriteCloser { @@ -57,13 +68,7 @@ func ToWriteCloser(w io.Writer) io.WriteCloser { return rw } - return struct { - io.Writer - io.Closer - }{ - w, - io.NopCloser(nil), - } + return NewWriteCloser(w, io.NopCloser(nil)) } // ToReadCloser creates an io.ReadCloser from the given io.Reader. diff --git a/common/hugo/hugo.go b/common/hugo/hugo.go index eecf4bc2f67..42555fda633 100644 --- a/common/hugo/hugo.go +++ b/common/hugo/hugo.go @@ -109,6 +109,11 @@ func (i HugoInfo) IsExtended() bool { return IsExtended } +// IsVendor returns whether we're running as `hugo vendor`. +func (i HugoInfo) IsVendor() bool { + return i.conf.Vendor() +} + // WorkingDir returns the project working directory. func (i HugoInfo) WorkingDir() string { return i.conf.WorkingDir() @@ -166,6 +171,7 @@ type ConfigProvider interface { WorkingDir() string IsMultihost() bool IsMultilingual() bool + Vendor() bool } // NewInfo creates a new Hugo Info object. diff --git a/common/maps/ordered.go b/common/maps/ordered.go index eaa4d73c611..864ba9b32bb 100644 --- a/common/maps/ordered.go +++ b/common/maps/ordered.go @@ -33,6 +33,15 @@ func NewOrdered[K comparable, T any]() *Ordered[K, T] { return &Ordered[K, T]{values: make(map[K]T)} } +// Contains returns whether the map contains the given key. +func (m *Ordered[K, T]) Contains(key K) bool { + if m == nil { + return false + } + _, found := m.values[key] + return found +} + // Set sets the value for the given key. // Note that insertion order is not affected if a key is re-inserted into the map. func (m *Ordered[K, T]) Set(key K, value T) { diff --git a/config/allconfig/allconfig.go b/config/allconfig/allconfig.go index 3c2eddcaba3..ffdad80d999 100644 --- a/config/allconfig/allconfig.go +++ b/config/allconfig/allconfig.go @@ -69,6 +69,7 @@ type InternalConfig struct { Watch bool FastRenderMode bool LiveReloadPort int + Vendor bool } // All non-params config keys for language. diff --git a/config/allconfig/configlanguage.go b/config/allconfig/configlanguage.go index deec61449bb..23deab22cc7 100644 --- a/config/allconfig/configlanguage.go +++ b/config/allconfig/configlanguage.go @@ -137,6 +137,10 @@ func (c ConfigLanguage) Watching() bool { return c.m.Base.Internal.Watch } +func (c ConfigLanguage) Vendor() bool { + return c.m.Base.Internal.Vendor +} + func (c ConfigLanguage) NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager { if !c.Watching() { return identity.NopManager diff --git a/config/commonConfig.go b/config/commonConfig.go index 9dea4a2fcea..529f2de2b35 100644 --- a/config/commonConfig.go +++ b/config/commonConfig.go @@ -131,6 +131,7 @@ func (b BuildConfig) clone() BuildConfig { return b } +// TODO1 remove, but first add a deprecation warning somewhere. func (b BuildConfig) UseResourceCache(err error) bool { if b.UseResourceCacheWhen == "never" { return false diff --git a/config/configProvider.go b/config/configProvider.go index 5bda2c55a65..bf6ab64c345 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -58,6 +58,7 @@ type AllProvider interface { BuildDrafts() bool Running() bool Watching() bool + Vendor() bool NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager FastRenderMode() bool PrintUnusedTemplates() bool diff --git a/hugofs/dirsmerger.go b/hugofs/dirsmerger.go index 9eedb584438..eafde205f98 100644 --- a/hugofs/dirsmerger.go +++ b/hugofs/dirsmerger.go @@ -63,3 +63,41 @@ var AppendDirsMerger overlayfs.DirsMerger = func(lofi, bofi []fs.DirEntry) []fs. return lofi } + +// DirsMergerPreserveDuplicateFunc returns a DirsMerger that will preserve any duplicate +// as defined by the given func. +func DirsMergerPreserveDuplicateFunc(preserveDuplicate func(fs.DirEntry) bool) overlayfs.DirsMerger { + return func(lofi, bofi []fs.DirEntry) []fs.DirEntry { + for _, fi1 := range bofi { + var found bool + if !preserveDuplicate(fi1) { + for _, fi2 := range lofi { + if fi1.Name() == fi2.Name() { + found = true + break + } + } + } + if !found { + lofi = append(lofi, fi1) + } + } + return lofi + } +} + +var FuncDirsMerger2 overlayfs.DirsMerger = func(lofi, bofi []fs.DirEntry) []fs.DirEntry { + for _, bofi := range bofi { + var found bool + for _, lofi := range lofi { + if bofi.Name() == lofi.Name() { + found = true + break + } + } + if !found { + lofi = append(lofi, bofi) + } + } + return lofi +} diff --git a/hugofs/files/classifier.go b/hugofs/files/classifier.go index 543d741d026..1e317c325f4 100644 --- a/hugofs/files/classifier.go +++ b/hugofs/files/classifier.go @@ -47,7 +47,8 @@ const ( ComponentFolderAssets = "assets" ComponentFolderI18n = "i18n" - FolderResources = "resources" + FolderVendor = "_vendor" + FolderResources = "resources" // TODO1 remove. FolderJSConfig = "_jsconfig" // Mounted below /assets with postcss.config.js etc. NameContentData = "_content" diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 38849317492..44a0c4e4803 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -60,6 +60,10 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { for _, rm := range rms { (&rm).clean() + if rm.From == files.FolderVendor { + continue + } + rm.FromBase = files.ResolveComponentFolder(rm.From) if len(rm.To) < 2 { diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index cb7846cd1b8..986a2698cc0 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -18,6 +18,7 @@ package filesystems import ( "fmt" "io" + "io/fs" "os" "path/filepath" "strings" @@ -241,7 +242,11 @@ type SourceFilesystems struct { // Writable filesystem on top the project's resources directory, // with any sub module's resource fs layered below. - ResourcesCache afero.Fs + ResourcesCache afero.Fs // TODO1 remove this. + + // A writable filesystem on top of the project's vendor directory + // with any sub module's vendor fs layered. + VendorFs afero.Fs // The work folder (may be a composite of project and theme components). Work afero.Fs @@ -569,6 +574,7 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { b.result.Layouts = createView(files.ComponentFolderLayouts, b.theBigFs.overlayMounts) b.result.Assets = createView(files.ComponentFolderAssets, b.theBigFs.overlayMounts) b.result.ResourcesCache = b.theBigFs.overlayResources + b.result.VendorFs = b.theBigFs.overlayVendor b.result.RootFss = b.theBigFs.rootFss // data and i18n needs a different merge strategy. @@ -628,6 +634,9 @@ func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesys overlayMountsStatic: overlayfs.New(overlayfs.Options{DirsMerger: hugofs.LanguageDirsMerger}), overlayFull: overlayfs.New(overlayfs.Options{}), overlayResources: overlayfs.New(overlayfs.Options{FirstWritable: true}), + overlayVendor: overlayfs.New(overlayfs.Options{FirstWritable: true, DirsMerger: hugofs.DirsMergerPreserveDuplicateFunc(func(fi fs.DirEntry) bool { + return fi.Name() == "resources.json" + })}), } mods := p.AllModules() @@ -678,6 +687,7 @@ func (b *sourceFilesystemsBuilder) createOverlayFs( collector.overlayMountsFull = appendNopIfEmpty(collector.overlayMountsFull) collector.overlayFull = appendNopIfEmpty(collector.overlayFull) collector.overlayResources = appendNopIfEmpty(collector.overlayResources) + collector.overlayVendor = appendNopIfEmpty(collector.overlayVendor) return nil } @@ -696,7 +706,16 @@ func (b *sourceFilesystemsBuilder) createOverlayFs( return md.dir, hpaths.AbsPathify(md.dir, path) } + modBase := collector.sourceProject + if !md.isMainProject { + modBase = collector.sourceModules + } + for i, mount := range md.Mounts() { + if mount.Target == files.FolderVendor { + collector.overlayVendor = collector.overlayVendor.Append(hugofs.NewBasePathFs(modBase, mount.Source)) + continue + } // Add more weight to early mounts. // When two mounts contain the same filename, // the first entry wins. @@ -744,11 +763,6 @@ func (b *sourceFilesystemsBuilder) createOverlayFs( } } - modBase := collector.sourceProject - if !md.isMainProject { - modBase = collector.sourceModules - } - sourceStatic := modBase rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...) @@ -831,7 +845,8 @@ type filesystemsCollector struct { overlayMountsStatic *overlayfs.OverlayFs overlayMountsFull *overlayfs.OverlayFs overlayFull *overlayfs.OverlayFs - overlayResources *overlayfs.OverlayFs + overlayResources *overlayfs.OverlayFs // TODO1 remove + overlayVendor *overlayfs.OverlayFs rootFss []*hugofs.RootMappingFs diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 8f3b71bafbb..795eb346c9b 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -187,6 +187,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { if err := h.postProcess(infol); err != nil { h.SendError(fmt.Errorf("postProcess: %w", err)) } + + if err := h.writeVendor(infol); err != nil { + h.SendError(fmt.Errorf("writeVendor: %w", err)) + } } if h.Metrics != nil { @@ -693,6 +697,21 @@ func (h *HugoSites) postProcess(l logg.LevelLogger) error { return g.Wait() } +func (h *HugoSites) writeVendor(l logg.LevelLogger) error { + if !h.Conf.Vendor() { + return nil + } + l = l.WithField("step", "writeVendor") + defer loggers.TimeTrackf(l, time.Now(), nil, "") + + v := h.ResourceSpec.Vendorer + if err := v.Finalize(); err != nil { + return err + } + + return nil +} + func (h *HugoSites) writeBuildStats() error { if h.ResourceSpec == nil { panic("h.ResourceSpec is nil") diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go index ff45ec2757f..36a4f1d253d 100644 --- a/hugolib/integrationtest_builder.go +++ b/hugolib/integrationtest_builder.go @@ -350,14 +350,18 @@ func (s *IntegrationTestBuilder) AssertNoRenderShortcodesArtifacts() { } } +func (s *IntegrationTestBuilder) AssertWorkingDir(root string, matches ...string) { + s.AssertFs(s.fs.WorkingDirReadOnly, root, matches...) +} + func (s *IntegrationTestBuilder) AssertPublishDir(matches ...string) { - s.AssertFs(s.fs.PublishDir, matches...) + s.AssertFs(s.fs.PublishDir, "", matches...) } -func (s *IntegrationTestBuilder) AssertFs(fs afero.Fs, matches ...string) { +func (s *IntegrationTestBuilder) AssertFs(fs afero.Fs, root string, matches ...string) { s.Helper() var buff bytes.Buffer - s.Assert(s.printAndCheckFs(fs, "", &buff), qt.IsNil) + s.Assert(s.printAndCheckFs(fs, root, &buff), qt.IsNil) printFsLines := strings.Split(buff.String(), "\n") sort.Strings(printFsLines) content := strings.TrimSpace((strings.Join(printFsLines, "\n"))) @@ -568,6 +572,11 @@ func (s *IntegrationTestBuilder) AddFiles(filenameContent ...string) *Integratio return s } +func (s *IntegrationTestBuilder) RemovePublic() *IntegrationTestBuilder { + s.Assert(s.fs.WorkingDirWritable.RemoveAll("public"), qt.IsNil) + return s +} + func (s *IntegrationTestBuilder) RemoveFiles(filenames ...string) *IntegrationTestBuilder { for _, filename := range filenames { absFilename := s.absFilename(filename) @@ -665,17 +674,18 @@ func (s *IntegrationTestBuilder) initBuilder() error { flags = config.New() } + internal := make(maps.Params) + if s.Cfg.Running { - flags.Set("internal", maps.Params{ - "running": s.Cfg.Running, - "watch": s.Cfg.Running, - }) + internal["running"] = true + internal["watch"] = true + } else if s.Cfg.Watching { - flags.Set("internal", maps.Params{ - "watch": s.Cfg.Watching, - }) + internal["watch"] = true } + flags.Set("internal", internal) + if s.Cfg.WorkingDir != "" { flags.Set("workingDir", s.Cfg.WorkingDir) } diff --git a/internal/js/esbuild/build.go b/internal/js/esbuild/build.go index 33b91eafc94..32c55386cbf 100644 --- a/internal/js/esbuild/build.go +++ b/internal/js/esbuild/build.go @@ -34,14 +34,14 @@ import ( // NewBuildClient creates a new BuildClient. func NewBuildClient(fs *filesystems.SourceFilesystem, rs *resources.Spec) *BuildClient { return &BuildClient{ - rs: rs, + Rs: rs, sfs: fs, } } // BuildClient is a client for building JavaScript resources using esbuild. type BuildClient struct { - rs *resources.Spec + Rs *resources.Spec sfs *filesystems.SourceFilesystem } @@ -52,11 +52,11 @@ func (c *BuildClient) Build(opts Options) (api.BuildResult, error) { dependencyManager = identity.NopManager } - opts.OutDir = c.rs.AbsPublishDir - opts.ResolveDir = c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved + opts.OutDir = c.Rs.AbsPublishDir + opts.ResolveDir = c.Rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved opts.AbsWorkingDir = opts.ResolveDir - opts.TsConfig = c.rs.ResolveJSConfigFile("tsconfig.json") - assetsResolver := newFSResolver(c.rs.Assets.Fs) + opts.TsConfig = c.Rs.ResolveJSConfigFile("tsconfig.json") + assetsResolver := newFSResolver(c.Rs.Assets.Fs) if err := opts.validate(); err != nil { return api.BuildResult{}, err @@ -67,7 +67,7 @@ func (c *BuildClient) Build(opts Options) (api.BuildResult, error) { } var err error - opts.compiled.Plugins, err = createBuildPlugins(c.rs, assetsResolver, dependencyManager, opts) + opts.compiled.Plugins, err = createBuildPlugins(c.Rs, assetsResolver, dependencyManager, opts) if err != nil { return api.BuildResult{}, err } @@ -175,7 +175,7 @@ func (c *BuildClient) Build(opts Options) (api.BuildResult, error) { // Return 1, log the rest. for i, err := range errors { if i > 0 { - c.rs.Logger.Errorf("js.Build failed: %s", err) + c.Rs.Logger.Errorf("js.Build failed: %s", err) } } diff --git a/modules/client.go b/modules/client.go index d16af2d351b..732e686f301 100644 --- a/modules/client.go +++ b/modules/client.go @@ -270,6 +270,7 @@ func (c *Client) Vendor() error { } // Include the resource cache if present. + // TODO1 remove resourcesDir := filepath.Join(dir, files.FolderResources) _, err := c.fs.Stat(resourcesDir) if err == nil { diff --git a/modules/collect.go b/modules/collect.go index 7034a6b161f..288831e6701 100644 --- a/modules/collect.go +++ b/modules/collect.go @@ -400,6 +400,11 @@ func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error { return err } + mounts, err = c.mountVendorDir(mod, mounts) + if err != nil { + return err + } + mod.mounts = mounts return nil } @@ -597,6 +602,26 @@ func (c *collector) loadModules() error { // Matches postcss.config.js etc. var commonJSConfigs = regexp.MustCompile(`(babel|postcss|tailwind)\.config\.js`) +func (c *collector) mountVendorDir(owner *moduleAdapter, mounts []Mount) ([]Mount, error) { + dir := filepath.Join(owner.Dir(), files.FolderVendor) + + add := owner.projectMod + if !add { + if _, err := c.fs.Stat(files.FolderVendor); err == nil { + add = true + } + } + + if add { + mounts = append(mounts, Mount{ + Source: dir, + Target: files.FolderVendor, + }) + } + + return mounts, nil +} + func (c *collector) mountCommonJSConfig(owner *moduleAdapter, mounts []Mount) ([]Mount, error) { for _, m := range mounts { if strings.HasPrefix(m.Target, files.JsConfigFolderMountPrefix) { diff --git a/resources/internal/vendor/vendor.go b/resources/internal/vendor/vendor.go new file mode 100644 index 00000000000..dc2895f1163 --- /dev/null +++ b/resources/internal/vendor/vendor.go @@ -0,0 +1,329 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vendor + +import ( + "encoding/json" + "fmt" + "io" + "io/fs" + "path" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/cespare/xxhash/v2" + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/common/hashing" + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" + "github.com/mitchellh/mapstructure" + "github.com/spf13/afero" +) + +var _ io.Closer = (*ResourceVendor)(nil) + +type Vendorable interface { + // The fully qualified name, using forward slash as path separator, of the function or method that produces this vendorable resource. + // Lower case, e.g. "css/tailwindcss". + VendorName() string + + // The key that identifies the resource transformation. + // Make this shallow so a transformation may be shared between environments, if needed, + // do not include any filenames/content hashes here. + // Typically you can use the VendorKeyFromOpts function to extract this from the options. + VendorScope() map[string]any +} + +type VendorScope struct { + // An optional key that identifies the resource transformation variant. + // The default vendor path is deliberately shallow, so this allows multiple vendored variants of the same resource transformation + // with different configurations. + Key string `json:"key"` + + // A Glob pattern matching the build environment, e.g. “{production,development}” or “*”. The default is “*”. + Environment string `json:"environment"` +} + +// VendorScopeFromOpts extracts the vendor scoped from the given options map. +func VendorScopeFromOpts(m map[string]any) map[string]any { + const vendorScopeKey = "vendorScope" + + for k, v := range m { + if strings.EqualFold(k, vendorScopeKey) { + return maps.ToStringMap(v) + } + } + return nil +} + +type ResourceVendor struct { + // For the from hash. + Digest *xxhash.Digest + + closeFunc func() error + + VendoredFile hugio.ReadSeekCloser + + // TODO1 names. + FinalFrom io.Reader + FinalTo io.Writer +} + +func (v *ResourceVendor) Close() error { + if v.closeFunc != nil { + return v.closeFunc() + } + return nil +} + +func NewVendorer(vendorFs, sourceFs afero.Fs, environment string) (*ResourceVendorer, error) { + rv := &ResourceVendorer{ + vendorFs: vendorFs, + sourceFs: sourceFs, + environment: environment, + } + + if err := rv.init(); err != nil { + return nil, err + } + + return rv, nil +} + +type ResourceVendorer struct { + vendorFs afero.Fs // Fs relative to the vendor root, top layer writatble. + sourceFs afero.Fs // Usually OS filesystem. + environment string + + mu sync.Mutex + + output outputResources + vendoredResources map[string]*maps.Ordered[string, vendoredResource] +} + +type outputResources struct { + Resources []outputResource `json:"resources"` +} + +func (v *ResourceVendorer) init() error { + dir, err := v.vendorFs.Open(vendorResources) + if err != nil { + if herrors.IsNotExist(err) { + return nil + } + return err + } + defer dir.Close() + fis, err := dir.(fs.ReadDirFile).ReadDir(-1) + if err != nil { + return err + } + + v.mu.Lock() + defer v.mu.Unlock() + v.vendoredResources = make(map[string]*maps.Ordered[string, vendoredResource]) + + for _, de := range fis { + if de.Name() == vendorResourcesJSON { + fim := de.(hugofs.FileMetaInfo) + f, err := fim.Meta().Open() + if err != nil { + return err + } + defer f.Close() + + var resources outputResources + if err := json.NewDecoder(f).Decode(&resources); err != nil { + return err + } + vendorDir := filepath.Dir(fim.Meta().Filename) + for _, r := range resources.Resources { + vr := vendoredResource{ + resource: r, + vendorDir: vendorDir, + } + m := v.vendoredResources[r.BasePath] + if m == nil { + m = maps.NewOrdered[string, vendoredResource]() + v.vendoredResources[r.BasePath] = m + } + if !m.Contains(r.ScopeHash) { + if err := vr.init(); err != nil { + return err + } + m.Set(r.ScopeHash, vr) + } + } + + } + } + return nil +} + +type vendoredResource struct { + resource outputResource + vendorDir string + + matchEnvironment glob.Glob +} + +func (v *vendoredResource) init() error { + v.matchEnvironment = glob.MustCompile(v.resource.Scope.Environment) + return nil +} + +func (v *ResourceVendorer) Finalize() error { + // Sort the vendored resources to make the order deterministic. + sort.Slice(v.output.Resources, func(i, j int) bool { + return v.output.Resources[i].BasePath < v.output.Resources[j].BasePath + }) + + vendorFilename := filepath.Join(vendorResources, vendorResourcesJSON) + + f, err := v.vendorFs.Create(vendorFilename) + if err != nil { + return err + } + defer f.Close() + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + if err := enc.Encode(v.output); err != nil { + return err + } + + return nil +} + +type ResourceVendorOptions struct { + Target Vendorable + Name string + InPath string + From io.Reader + To io.Writer +} + +const ( + vendorRoot = "_vendor" + vendorModules = "modules" + vendorResources = "resources" // TODO1 + modules. + vendorResourcesJSON = "resources.json" +) + +type outputResource struct { + // BasePath to the vendored resource, relative to the vendor root. + // Unix style path. + // The full path starting from the vendor root is BasePath/ScopeHash/Path. + BasePath string `json:"basePath"` + + // Path the last path element of the vendored resource. + Path string `json:"path"` + + Scope VendorScope `json:"scope"` + ScopeHash string `json:"scopeHash"` +} + +func (v *ResourceVendorer) OpenVendoredFileForWriting(opts ResourceVendorOptions) (io.WriteCloser, hugio.OpenReadSeekCloser, error) { + vs := opts.Target.VendorScope() + scopeHash := hashing.HashStringHex(vs) + vendorScope := VendorScope{ + Environment: "*", + } + if err := mapstructure.WeakDecode(vs, &vendorScope); err != nil { + return nil, nil, err + } + + vendorBasePath := v.vendorPath(opts) + vendorDir := filepath.Join(vendorBasePath, scopeHash) + if err := v.vendorFs.MkdirAll(vendorDir, 0o755); err != nil { + return nil, nil, fmt.Errorf("failed to create directory %q: %w", vendorBasePath, err) + } + vendorFilename := filepath.Join(vendorDir, opts.InPath) + f, err := helpers.OpenFileForWriting(v.vendorFs, vendorFilename) + if err != nil { + return nil, nil, err + } + open := func() (hugio.ReadSeekCloser, error) { + return v.vendorFs.Open(vendorFilename) + } + + closer := types.CloserFunc(func() error { + vendoredResource := outputResource{ + BasePath: vendorBasePath, + Path: opts.InPath, + Scope: vendorScope, + ScopeHash: scopeHash, + } + v.mu.Lock() + v.output.Resources = append(v.output.Resources, vendoredResource) + v.mu.Unlock() + + return f.Close() + }) + + w := hugio.NewWriteCloser(f, closer) + + return w, open, nil +} + +// OpenVendoredFile opens a vendored file for reading or nil if not found. +func (v *ResourceVendorer) OpenVendoredFile(opts ResourceVendorOptions) (hugio.ReadSeekCloser, hugio.OpenReadSeekCloser, error) { + open := v.VendoredOpenReadSeekCloser(opts) + f, err := open() + if err != nil { + if herrors.IsNotExist(err) { + return nil, nil, nil + } + return nil, nil, err + } + return f, open, nil +} + +func (v *ResourceVendorer) VendoredOpenReadSeekCloser(opts ResourceVendorOptions) hugio.OpenReadSeekCloser { + vendorFilePath := v.vendorPath(opts) + + r, found := v.vendoredResources[vendorFilePath] + + return func() (hugio.ReadSeekCloser, error) { + if !found { + return nil, afero.ErrFileNotFound + } + + var filename string + r.Range(func(key string, vr vendoredResource) bool { + if vr.matchEnvironment.Match(v.environment) { + r := vr.resource + filename = filepath.Join(vr.vendorDir, vendorFilePath, r.ScopeHash, r.Path) + return false + } + return true + }) + if filename == "" { + return nil, afero.ErrFileNotFound + } + // resources/css/tailwindcss + return v.sourceFs.Open(filename) + } +} + +func (v *ResourceVendorer) vendorPath(opts ResourceVendorOptions) string { + n := filepath.ToSlash(path.Join(vendorResources, opts.Target.VendorName())) + return n +} diff --git a/resources/internal/vendor/vendor_test.go b/resources/internal/vendor/vendor_test.go new file mode 100644 index 00000000000..455e2b1a137 --- /dev/null +++ b/resources/internal/vendor/vendor_test.go @@ -0,0 +1,19 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vendor + +import "testing" + +func TestResourceVendor(t *testing.T) { +} diff --git a/resources/resource.go b/resources/resource.go index 6ef9bdae050..481d9d459f1 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -570,7 +570,9 @@ func (l *genericResource) getResourcePaths() internal.ResourcePaths { } func (r *genericResource) tryTransformedFileCache(key string, u *transformationUpdate) io.ReadCloser { - fi, f, meta, found := r.spec.ResourceCache.getFromFile(key) + panic("not implemented") + // TOODO1 remove this, but make _vendor use the file cache API (maybe) + /*fi, f, meta, found := r.spec.ResourceCache.getFromFile(key) if !found { return nil } @@ -579,7 +581,7 @@ func (r *genericResource) tryTransformedFileCache(key string, u *transformationU u.mediaType = mt u.data = meta.MetaData u.targetPath = meta.Target - return f + return f*/ } func (r *genericResource) mergeData(in map[string]any) { @@ -600,24 +602,11 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour r := rc.clone() if u.content != nil { - r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) { - return hugio.NewReadSeekerNoOpCloserFromString(*u.content), nil - } + r.sd.OpenReadSeekCloser = u.content } r.sd.MediaType = u.mediaType - if u.sourceFilename != nil { - if u.sourceFs == nil { - return nil, errors.New("sourceFs is nil") - } - r.setOpenSource(func() (hugio.ReadSeekCloser, error) { - return u.sourceFs.Open(*u.sourceFilename) - }) - } else if u.sourceFs != nil { - return nil, errors.New("sourceFs is set without sourceFilename") - } - if u.targetPath == "" { return nil, errors.New("missing targetPath") } diff --git a/resources/resource_spec.go b/resources/resource_spec.go index f1c30e0a209..1b2250cc12c 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -22,6 +22,7 @@ import ( "github.com/gohugoio/hugo/config/allconfig" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/resources/internal/vendor" "github.com/gohugoio/hugo/resources/jsconfig" "github.com/gohugoio/hugo/resources/page/pagemeta" @@ -91,6 +92,11 @@ func NewSpec( } } + vend, err := vendor.NewVendorer(s.BaseFs.VendorFs, s.BaseFs.SourceFs, conf.Environment) + if err != nil { + return nil, err + } + rs := &Spec{ PathSpec: s, Logger: logger, @@ -103,6 +109,7 @@ func NewSpec( memCache, s, ), + Vendorer: vend, ExecHelper: execHelper, Permalinks: permalinks, @@ -122,6 +129,7 @@ type Spec struct { ErrorSender herrors.ErrorSender BuildClosers types.CloseAdder Rebuilder identity.SignalRebuilder + Vendorer *vendor.ResourceVendorer TextTemplates tpl.TemplateParseFinder diff --git a/resources/resource_transformers/cssjs/tailwindcss.go b/resources/resource_transformers/cssjs/tailwindcss.go index beda7a646fe..c8cc0d7a495 100644 --- a/resources/resource_transformers/cssjs/tailwindcss.go +++ b/resources/resource_transformers/cssjs/tailwindcss.go @@ -25,6 +25,7 @@ import ( "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/resources/internal/vendor" "github.com/gohugoio/hugo/resources/resource" "github.com/mitchellh/mapstructure" ) @@ -51,11 +52,21 @@ func (c *TailwindCSSClient) Process(res resources.ResourceTransformer, options m return res.Transform(&tailwindcssTransformation{rs: c.rs, optionsm: options}) } +var _ vendor.Vendorable = (*tailwindcssTransformation)(nil) + type tailwindcssTransformation struct { optionsm map[string]any rs *resources.Spec } +func (t *tailwindcssTransformation) VendorName() string { + return "css/tailwindcss" +} + +func (t *tailwindcssTransformation) VendorScope() map[string]any { + return vendor.VendorScopeFromOpts(t.optionsm) +} + func (t *tailwindcssTransformation) Key() internal.ResourceTransformationKey { return internal.NewResourceTransformationKey("tailwindcss", t.optionsm) } @@ -120,8 +131,6 @@ func (t *tailwindcssTransformation) Transform(ctx *resources.ResourceTransformat return err } - src := ctx.From - imp := newImportResolver( ctx.From, ctx.InPath, @@ -129,7 +138,7 @@ func (t *tailwindcssTransformation) Transform(ctx *resources.ResourceTransformat t.rs.Assets.Fs, t.rs.Logger, ctx.DependencyManager, ) - src, err = imp.resolve() + src, err := imp.resolve() if err != nil { return err } diff --git a/resources/resource_transformers/cssjs/tailwindcss_integration_test.go b/resources/resource_transformers/cssjs/tailwindcss_integration_test.go index c0d6d3dff45..9c9617650ce 100644 --- a/resources/resource_transformers/cssjs/tailwindcss_integration_test.go +++ b/resources/resource_transformers/cssjs/tailwindcss_integration_test.go @@ -14,9 +14,12 @@ package cssjs_test import ( + "fmt" + "strings" "testing" "github.com/bep/logg" + qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/hugolib" ) @@ -56,7 +59,7 @@ func TestTailwindV4Basic(t *testing.T) { } -- layouts/index.html -- {{ $css := resources.Get "css/styles.css" | css.TailwindCSS }} -CSS: {{ $css.Content | safeCSS }}| +CSS: {{ $css.RelPermalink }}|{{ $css.Content | safeCSS }}| ` b := hugolib.NewIntegrationTestBuilder( @@ -70,3 +73,73 @@ CSS: {{ $css.Content | safeCSS }}| b.AssertFileContent("public/index.html", "/*! tailwindcss v4.0.0") } + +func TestTailwindV4BasicVendorRoundTrip(t *testing.T) { + if !htesting.IsCI() { + t.Skip("Skip long running test when running locally") + } + + filesTemplate := ` +-- hugo.toml -- +baseURL = "https://example.org/" +[internal] +vendor = VENDOR +-- package.json -- +{ + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/bep/hugo-starter-tailwind-basic.git" + }, + "devDependencies": { + "@tailwindcss/cli": "^4.0.0-alpha.26", + "tailwindcss": "^4.0.0-alpha.26" + }, + "name": "hugo-starter-tailwind-basic", + "version": "0.1.0" +} +-- assets/css/styles.css -- +@import "tailwindcss"; + +@theme { + --font-family-display: "Satoshi", "sans-serif"; + + --breakpoint-3xl: 1920px; + + --color-neon-pink: oklch(71.7% 0.25 360); + --color-neon-lime: oklch(91.5% 0.258 129); + --color-neon-cyan: oklch(91.3% 0.139 195.8); +} +-- layouts/index.html -- +{{ $css := resources.Get "css/styles.css" | css.TailwindCSS (dict "vendorScope" (dict "key" "foobar" ) ) }} +CSS: {{ $css.RelPermalink }}|{{ $css.Content | safeCSS }}| +` + + workinDir := t.TempDir() + + for _, vendor := range []bool{true, false} { + files := strings.Replace(filesTemplate, "VENDOR", fmt.Sprint(vendor), 1) + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + TxtarString: files, + NeedsOsFS: true, + NeedsNpmInstall: true, + WorkingDir: workinDir, + LogLevel: logg.LevelError, + }).Build() + + b.Assert(b.H.Conf.Vendor(), qt.Equals, vendor) + + b.AssertWorkingDir("_vendor", "_vendor/resources/css/tailwindcss/22ae68a32f191359/css/styles.css") + + // b.AssertWorkingDir("_vendor", "_vendor/resources/_all/css/tailwindcss/css/styles.css") + + b.AssertFileContent("public/index.html", "/*! tailwindcss v4.0.0") + b.AssertFileContent("_vendor/resources/resources.json", "22ae68a32f191359") + + b.RemovePublic() + + } +} diff --git a/resources/resource_transformers/js/build.go b/resources/resource_transformers/js/build.go index bd943461f0d..129e01329e0 100644 --- a/resources/resource_transformers/js/build.go +++ b/resources/resource_transformers/js/build.go @@ -64,6 +64,7 @@ func (c *Client) transform(opts esbuild.Options, transformCtx *resources.Resourc content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") } + // TODO1 vendor. if err = transformCtx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { return result, err } diff --git a/resources/resource_transformers/js/transform.go b/resources/resource_transformers/js/transform.go index 13909e54cc6..44a04ebbd7a 100644 --- a/resources/resource_transformers/js/transform.go +++ b/resources/resource_transformers/js/transform.go @@ -22,6 +22,7 @@ import ( "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/resources/internal/vendor" ) type buildTransformation struct { @@ -29,6 +30,16 @@ type buildTransformation struct { c *Client } +var _ vendor.Vendorable = (*buildTransformation)(nil) + +func (t *buildTransformation) VendorName() string { + return "js/build" +} + +func (t *buildTransformation) VendorScope() map[string]any { + return vendor.VendorScopeFromOpts(t.optsm) +} + func (t *buildTransformation) Key() internal.ResourceTransformationKey { return internal.NewResourceTransformationKey("jsbuild", t.optsm) } diff --git a/resources/transform.go b/resources/transform.go index 73f3b85d25d..f321f532990 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -30,7 +30,7 @@ import ( "github.com/gohugoio/hugo/resources/images" "github.com/gohugoio/hugo/resources/images/exif" - "github.com/spf13/afero" + "github.com/gohugoio/hugo/resources/internal/vendor" bp "github.com/gohugoio/hugo/bufferpool" @@ -431,8 +431,6 @@ func (r *resourceAdapter) getOrTransform(publish, setContent bool) error { } func (r *resourceAdapter) transform(key string, publish, setContent bool) (*resourceAdapterInner, error) { - cache := r.spec.ResourceCache - b1 := bp.GetBuffer() b2 := bp.GetBuffer() defer bp.PutBuffer(b1) @@ -467,20 +465,14 @@ func (r *resourceAdapter) transform(key string, publish, setContent bool) (*reso tctx.SourcePath = strings.TrimPrefix(tctx.InPath, "/") counter := 0 - writeToFileCache := false - - var transformedContentr io.Reader + isVendorMode := r.spec.Cfg.Vendor() + vendorer := r.spec.Vendorer for i, tr := range r.transformations { if i != 0 { tctx.InMediaType = tctx.OutMediaType } - mayBeCachedOnDisk := transformationsToCacheOnDisk[tr.Key().Name] - if !writeToFileCache { - writeToFileCache = mayBeCachedOnDisk - } - if i > 0 { hasWrites := tctx.To.(*bytes.Buffer).Len() > 0 if hasWrites { @@ -525,38 +517,67 @@ func (r *resourceAdapter) transform(key string, publish, setContent bool) (*reso return fmt.Errorf(msg+": %w", err) } - bcfg := r.spec.BuildConfig() - var tryFileCache bool - if mayBeCachedOnDisk && bcfg.UseResourceCache(nil) { - tryFileCache = true - } else { - err = tr.Transform(tctx) - if err != nil && err != herrors.ErrFeatureNotAvailable { - return nil, newErr(err) - } + if isVendorMode { + err = func() error { + to := tctx.To + var buff bytes.Buffer + if vendorable, ok := tr.(vendor.Vendorable); ok { + vendorOpts := vendor.ResourceVendorOptions{ + Target: vendorable, + InPath: tctx.InPath, // TODO1 + } + f, open, err := vendorer.OpenVendoredFileForWriting(vendorOpts) + if err != nil { + return err + } + updates.content = open + defer f.Close() + + tctx.To = io.MultiWriter(to, f, &buff) + } - if mayBeCachedOnDisk { - tryFileCache = bcfg.UseResourceCache(err) - } - if err != nil && !tryFileCache { + defer func() { + tctx.To = to + }() + + if err := tr.Transform(tctx); err != nil { + return err + } + + return nil + }() + if err != nil { return nil, newErr(err) } - } - - if tryFileCache { - f := r.target.tryTransformedFileCache(key, updates) - if f == nil { + } else { + var handled bool + if vendorable, ok := tr.(vendor.Vendorable); ok { + // TODO1 metadata + folder. + vendorOpts := vendor.ResourceVendorOptions{ + Target: vendorable, + Name: "TODO1 remove me.", // TODO1 + InPath: tctx.InPath, // TODO1 + } + f, open, err := vendorer.OpenVendoredFile(vendorOpts) + if err != nil { + return nil, newErr(err) + } + updates.content = open + if f != nil { + handled = true + _, err = io.Copy(tctx.To, f) + f.Close() + if err != nil { + return nil, newErr(err) + } + } + } + if !handled { + err = tr.Transform(tctx) if err != nil { return nil, newErr(err) } - return nil, newErr(fmt.Errorf("resource %q not found in file cache", key)) } - transformedContentr = f - updates.sourceFs = cache.fileCache.Fs - defer f.Close() - - // The reader above is all we need. - break } if tctx.OutPath != "" { @@ -565,13 +586,15 @@ func (r *resourceAdapter) transform(key string, publish, setContent bool) (*reso } } - if transformedContentr == nil { - updates.updateFromCtx(tctx) - } + // TODO1 + updates.updateFromCtx(tctx) var publishwriters []io.WriteCloser if publish { + if updates.targetPath == "" { + panic("no target path set") + } publicw, err := r.target.openPublishFileForWriting(updates.targetPath) if err != nil { return nil, err @@ -579,33 +602,21 @@ func (r *resourceAdapter) transform(key string, publish, setContent bool) (*reso publishwriters = append(publishwriters, publicw) } - if transformedContentr == nil { - if writeToFileCache { - // Also write it to the cache - fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata()) - if err != nil { - return nil, err - } - updates.sourceFilename = &fi.Name - updates.sourceFs = cache.fileCache.Fs - publishwriters = append(publishwriters, metaw) - } - - // Any transformations reading from From must also write to To. - // This means that if the target buffer is empty, we can just reuse - // the original reader. - if b, ok := tctx.To.(*bytes.Buffer); ok && b.Len() > 0 { - transformedContentr = tctx.To.(*bytes.Buffer) - } else { - transformedContentr = contentrc - } + var transformedContentr io.Reader + // Any transformations reading from From must also write to To. + // This means that if the target buffer is empty, we can just reuse + // the original reader. + if b, ok := tctx.To.(*bytes.Buffer); ok && b.Len() > 0 { + transformedContentr = tctx.To.(*bytes.Buffer) + } else { + transformedContentr = contentrc } + setContent = setContent || updates.content == nil + // Also write it to memory var contentmemw *bytes.Buffer - setContent = setContent || !writeToFileCache - if setContent { contentmemw = bp.GetBuffer() defer bp.PutBuffer(contentmemw) @@ -621,7 +632,7 @@ func (r *resourceAdapter) transform(key string, publish, setContent bool) (*reso if setContent { s := contentmemw.String() - updates.content = &s + updates.content = hugio.NewOpenReadSeekCloser(hugio.NewReadSeekerNoOpCloserFromString(s)) } newTarget, err := r.target.cloneWithUpdates(updates) @@ -716,18 +727,16 @@ type transformableResource interface { } type transformationUpdate struct { - content *string - sourceFilename *string - sourceFs afero.Fs - targetPath string - mediaType media.Type - data map[string]any + content hugio.OpenReadSeekCloser + targetPath string + mediaType media.Type + data map[string]any startCtx ResourceTransformationCtx } func (u *transformationUpdate) isContentChanged() bool { - return u.content != nil || u.sourceFilename != nil + return u.content != nil } func (u *transformationUpdate) toTransformedResourceMetadata() transformedResourceMetadata {