Create pages from _content.gotmpl

Closes #12427
Closes #12485
Closes #6310
Closes #5074

Author: bep

commands/hugobuilder.go

@@ -854,7 +854,7 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
 			h.BaseFs.SourceFilesystems,
 			dynamicEvents)
 
-		onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents)
+		onePageName := pickOneWriteOrCreatePath(h.Conf.ContentTypes(), partitionedEvents.ContentEvents)
 
 		c.printChangeDetected("")
 		c.changeDetector.PrepareNew()

commands/server.go

@@ -46,12 +46,12 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/hugo"
+
 	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/common/urls"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
-	"github.com/gohugoio/hugo/hugofs/files"
 	"github.com/gohugoio/hugo/hugolib"
 	"github.com/gohugoio/hugo/hugolib/filesystems"
 	"github.com/gohugoio/hugo/livereload"
@@ -1188,16 +1188,16 @@ func partitionDynamicEvents(sourceFs *filesystems.SourceFilesystems, events []fs
 	return
 }
 
-func pickOneWriteOrCreatePath(events []fsnotify.Event) string {
+func pickOneWriteOrCreatePath(contentTypes config.ContentTypesProvider, events []fsnotify.Event) string {
 	name := ""
 
 	for _, ev := range events {
 		if ev.Op&fsnotify.Write == fsnotify.Write || ev.Op&fsnotify.Create == fsnotify.Create {
-			if files.IsIndexContentFile(ev.Name) {
+			if contentTypes.IsIndexContentFile(ev.Name) {
 				return ev.Name
 			}
 
-			if files.IsContentFile(ev.Name) {
+			if contentTypes.IsContentFile(ev.Name) {
 				name = ev.Name
 			}
 

common/maps/cache.go

@@ -27,7 +27,12 @@ func NewCache[K comparable, T any]() *Cache[K, T] {
 }
 
 // Delete deletes the given key from the cache.
+// If c is nil, this method is a no-op.
 func (c *Cache[K, T]) Get(key K) (T, bool) {
+	if c == nil {
+		var zero T
+		return zero, false
+	}
 	c.RLock()
 	v, found := c.m[key]
 	c.RUnlock()
@@ -60,6 +65,15 @@ func (c *Cache[K, T]) Set(key K, value T) {
 	c.Unlock()
 }
 
+// ForEeach calls the given function for each key/value pair in the cache.
+func (c *Cache[K, T]) ForEeach(f func(K, T)) {
+	c.RLock()
+	defer c.RUnlock()
+	for k, v := range c.m {
+		f(k, v)
+	}
+}
+
 // SliceCache is a simple thread safe cache backed by a map.
 type SliceCache[T any] struct {
 	m map[string][]T

common/paths/pathparser.go

@@ -25,20 +25,16 @@ import (
 	"github.com/gohugoio/hugo/identity"
 )
 
-var defaultPathParser PathParser
-
 // PathParser parses a path into a Path.
 type PathParser struct {
 	// Maps the language code to its index in the languages/sites slice.
 	LanguageIndex map[string]int
 
 	// Reports whether the given language is disabled.
 	IsLangDisabled func(string) bool
-}
 
-// Parse parses component c with path s into Path using the default path parser.
-func Parse(c, s string) *Path {
-	return defaultPathParser.Parse(c, s)
+	// Reports whether the given ext is a content file.
+	IsContentExt func(string) bool
 }
 
 // NormalizePathString returns a normalized path string using the very basic Hugo rules.
@@ -108,7 +104,6 @@ func (pp *PathParser) parse(component, s string) (*Path, error) {
 		var err error
 		// Preserve the original case for titles etc.
 		p.unnormalized, err = pp.doParse(component, s, pp.newPath(component))
-
 		if err != nil {
 			return nil, err
 		}
@@ -195,23 +190,26 @@ func (pp *PathParser) doParse(component, s string, p *Path) (*Path, error) {
 		}
 	}
 
-	isContentComponent := p.component == files.ComponentFolderContent || p.component == files.ComponentFolderArchetypes
-	isContent := isContentComponent && files.IsContentExt(p.Ext())
-
-	if isContent {
+	if len(p.identifiers) > 0 {
+		isContentComponent := p.component == files.ComponentFolderContent || p.component == files.ComponentFolderArchetypes
+		isContent := isContentComponent && pp.IsContentExt(p.Ext())
 		id := p.identifiers[len(p.identifiers)-1]
 		b := p.s[p.posContainerHigh : id.Low-1]
-		switch b {
-		case "index":
-			p.bundleType = PathTypeLeaf
-		case "_index":
-			p.bundleType = PathTypeBranch
-		default:
-			p.bundleType = PathTypeContentSingle
-		}
+		if isContent {
+			switch b {
+			case "index":
+				p.bundleType = PathTypeLeaf
+			case "_index":
+				p.bundleType = PathTypeBranch
+			default:
+				p.bundleType = PathTypeContentSingle
+			}
 
-		if slashCount == 2 && p.IsLeafBundle() {
-			p.posSectionHigh = 0
+			if slashCount == 2 && p.IsLeafBundle() {
+				p.posSectionHigh = 0
+			}
+		} else if b == files.NameContentData && files.IsContentDataExt(p.Ext()) {
+			p.bundleType = PathTypeContentData
 		}
 	}
 
@@ -246,6 +244,9 @@ const (
 
 	// Branch bundles, e.g. /blog/_index.md
 	PathTypeBranch
+
+	// Content data file, _content.gotmpl.
+	PathTypeContentData
 )
 
 type Path struct {
@@ -521,10 +522,6 @@ func (p *Path) Identifiers() []string {
 	return ids
 }
 
-func (p *Path) IsHTML() bool {
-	return files.IsHTML(p.Ext())
-}
-
 func (p *Path) BundleType() PathType {
 	return p.bundleType
 }
@@ -541,6 +538,10 @@ func (p *Path) IsLeafBundle() bool {
 	return p.bundleType == PathTypeLeaf
 }
 
+func (p *Path) IsContentData() bool {
+	return p.bundleType == PathTypeContentData
+}
+
 func (p Path) ForBundleType(t PathType) *Path {
 	p.bundleType = t
 	return &p

common/paths/pathparser_test.go

@@ -27,6 +27,9 @@ var testParser = &PathParser{
 		"no": 0,
 		"en": 1,
 	},
+	IsContentExt: func(ext string) bool {
+		return ext == "md"
+	},
 }
 
 func TestParse(t *testing.T) {
@@ -333,6 +336,22 @@ func TestParse(t *testing.T) {
 				c.Assert(p.Path(), qt.Equals, "/a/b/c.txt")
 			},
 		},
+		{
+			"Content data file gotmpl",
+			"/a/b/_content.gotmpl",
+			func(c *qt.C, p *Path) {
+				c.Assert(p.Path(), qt.Equals, "/a/b/_content.gotmpl")
+				c.Assert(p.Ext(), qt.Equals, "gotmpl")
+				c.Assert(p.IsContentData(), qt.IsTrue)
+			},
+		},
+		{
+			"Content data file yaml",
+			"/a/b/_content.yaml",
+			func(c *qt.C, p *Path) {
+				c.Assert(p.IsContentData(), qt.IsFalse)
+			},
+		},
 	}
 	for _, test := range tests {
 		c.Run(test.name, func(c *qt.C) {

config/allconfig/allconfig.go

@@ -367,6 +367,7 @@ func (c *Config) CompileConfig(logger loggers.Logger) error {
 		DisabledLanguages: disabledLangs,
 		IgnoredLogs:       ignoredLogIDs,
 		KindOutputFormats: kindOutputFormats,
+		ContentTypes:      media.DefaultContentTypes.FromTypes(c.MediaTypes.Config),
 		CreateTitle:       helpers.GetTitleFunc(c.TitleCaseStyle),
 		IsUglyURLSection:  isUglyURL,
 		IgnoreFile:        ignoreFile,
@@ -402,6 +403,7 @@ type ConfigCompiled struct {
 	BaseURLLiveReload urls.BaseURL
 	ServerInterface   string
 	KindOutputFormats map[string]output.Formats
+	ContentTypes      media.ContentTypes
 	DisabledKinds     map[string]bool
 	DisabledLanguages map[string]bool
 	IgnoredLogs       map[string]bool
@@ -759,7 +761,7 @@ func (c *Configs) Init() error {
 	c.Languages = languages
 	c.LanguagesDefaultFirst = languagesDefaultFirst
 
-	c.ContentPathParser = &paths.PathParser{LanguageIndex: languagesDefaultFirst.AsIndexSet(), IsLangDisabled: c.Base.IsLangDisabled}
+	c.ContentPathParser = &paths.PathParser{LanguageIndex: languagesDefaultFirst.AsIndexSet(), IsLangDisabled: c.Base.IsLangDisabled, IsContentExt: c.Base.C.ContentTypes.IsContentSuffix}
 
 	c.configLangs = make([]config.AllProvider, len(c.Languages))
 	for i, l := range c.LanguagesDefaultFirst {

config/allconfig/allconfig_integration_test.go

@@ -84,3 +84,21 @@ logPathWarnings = true
 	b.Assert(conf.PrintI18nWarnings, qt.Equals, true)
 	b.Assert(conf.PrintPathWarnings, qt.Equals, true)
 }
+
+func TestRedefineContentTypes(t *testing.T) {
+	files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+[mediaTypes]
+[mediaTypes."text/html"]
+suffixes = ["html", "xhtml"]
+`
+
+	b := hugolib.Test(t, files)
+
+	conf := b.H.Configs.Base
+	contentTypes := conf.C.ContentTypes
+
+	b.Assert(contentTypes.HTML.Suffixes(), qt.DeepEquals, []string{"html", "xhtml"})
+	b.Assert(contentTypes.Markdown.Suffixes(), qt.DeepEquals, []string{"md", "mdown", "markdown"})
+}

config/allconfig/configlanguage.go

@@ -144,6 +144,10 @@ func (c ConfigLanguage) NewIdentityManager(name string) identity.Manager {
 	return identity.NewManager(name)
 }
 
+func (c ConfigLanguage) ContentTypes() config.ContentTypesProvider {
+	return c.config.C.ContentTypes
+}
+
 // GetConfigSection is mostly used in tests. The switch statement isn't complete, but what's in use.
 func (c ConfigLanguage) GetConfigSection(s string) any {
 	switch s {

config/configProvider.go

@@ -41,6 +41,7 @@ type AllProvider interface {
 	Dirs() CommonDirs
 	Quiet() bool
 	DirsBase() CommonDirs
+	ContentTypes() ContentTypesProvider
 	GetConfigSection(string) any
 	GetConfig() any
 	CanonifyURLs() bool
@@ -75,6 +76,15 @@ type AllProvider interface {
 	EnableEmoji() bool
 }
 
+// We cannot import the media package as that would create a circular dependency.
+// This interface defineds a sub set of what media.ContentTypes provides.
+type ContentTypesProvider interface {
+	IsContentSuffix(suffix string) bool
+	IsContentFile(filename string) bool
+	IsIndexContentFile(filename string) bool
+	IsHTMLSuffix(suffix string) bool
+}
+
 // Provider provides the configuration settings for Hugo.
 type Provider interface {
 	GetString(key string) string

create/content.go

@@ -29,8 +29,6 @@ import (
 	"github.com/gohugoio/hugo/common/hstrings"
 	"github.com/gohugoio/hugo/common/paths"
 
-	"github.com/gohugoio/hugo/hugofs/files"
-
 	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/gohugoio/hugo/helpers"
@@ -98,7 +96,7 @@ func NewContent(h *hugolib.HugoSites, kind, targetPath string, force bool) error
 			return "", fmt.Errorf("failed to resolve %q to an archetype template", targetPath)
 		}
 
-		if !files.IsContentFile(b.targetPath) {
+		if !h.Conf.ContentTypes().IsContentFile(b.targetPath) {
 			return "", fmt.Errorf("target path %q is not a known content format", b.targetPath)
 		}
 

helpers/content.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/gohugoio/hugo/common/hexec"
 	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/media"
 
 	"github.com/spf13/afero"
 
@@ -135,20 +136,16 @@ func (c *ContentSpec) SanitizeAnchorName(s string) string {
 }
 
 func (c *ContentSpec) ResolveMarkup(in string) string {
-	if c == nil {
-		panic("nil ContentSpec")
-	}
 	in = strings.ToLower(in)
-	switch in {
-	case "md", "markdown", "mdown":
-		return "markdown"
-	case "html", "htm":
-		return "html"
-	default:
-		if conv := c.Converters.Get(in); conv != nil {
-			return conv.Name()
-		}
+
+	if mediaType, found := c.Cfg.ContentTypes().(media.ContentTypes).Types().GetBestMatch(markup.ResolveMarkup(in)); found {
+		return mediaType.SubType
 	}
+
+	if conv := c.Converters.Get(in); conv != nil {
+		return markup.ResolveMarkup(conv.Name())
+	}
+
 	return ""
 }
 
@@ -244,7 +241,7 @@ func (c *ContentSpec) TrimShortHTML(input []byte, markup string) []byte {
 	openingTag := []byte("<p>")
 	closingTag := []byte("</p>")
 
-	if markup == "asciidocext" {
+	if markup == media.DefaultContentTypes.AsciiDoc.SubType {
 		openingTag = []byte("<div class=\"paragraph\">\n<p>")
 		closingTag = []byte("</p>\n</div>")
 	}

helpers/content_test.go

@@ -41,7 +41,7 @@ func TestTrimShortHTML(t *testing.T) {
 		{"markdown", []byte("<h2 id=`a`>b</h2>\n\n<p>c</p>"), []byte("<h2 id=`a`>b</h2>\n\n<p>c</p>")},
 		// Issue 12369
 		{"markdown", []byte("<div class=\"paragraph\">\n<p>foo</p>\n</div>"), []byte("<div class=\"paragraph\">\n<p>foo</p>\n</div>")},
-		{"asciidocext", []byte("<div class=\"paragraph\">\n<p>foo</p>\n</div>"), []byte("foo")},
+		{"asciidoc", []byte("<div class=\"paragraph\">\n<p>foo</p>\n</div>"), []byte("foo")},
 	}
 
 	c := newTestContentSpec(nil)

helpers/general_test.go

@@ -35,9 +35,9 @@ func TestResolveMarkup(t *testing.T) {
 		{"md", "markdown"},
 		{"markdown", "markdown"},
 		{"mdown", "markdown"},
-		{"asciidocext", "asciidocext"},
-		{"adoc", "asciidocext"},
-		{"ad", "asciidocext"},
+		{"asciidocext", "asciidoc"},
+		{"adoc", "asciidoc"},
+		{"ad", "asciidoc"},
 		{"rst", "rst"},
 		{"pandoc", "pandoc"},
 		{"pdc", "pandoc"},