Fix slight bug in katex (#21171)

There is a small bug in #20571 whereby `$a a$b b$` will not be correctly
detected as a math inline block of `a a$b b`. This PR fixes this.

Also reenable test cases as per #21340 

Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/markup/html_test.go

@@ -7,6 +7,7 @@ package markup_test
 import (
 	"context"
 	"io"
+	"os"
 	"strings"
 	"testing"
 
@@ -32,6 +33,7 @@ func TestMain(m *testing.M) {
 	if err := git.InitSimple(context.Background()); err != nil {
 		log.Fatal("git init failed, err: %v", err)
 	}
+	os.Exit(m.Run())
 }
 
 func TestRender_Commits(t *testing.T) {
@@ -336,7 +338,7 @@ func TestRender_emoji(t *testing.T) {
 		`<p>Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span><span class="emoji" aria-label="grinning face with smiling eyes">😄</span> 2 emoji next to each other</p>`)
 	test(
 		"😎🤪🔐🤑❓",
-		`<p><span class="emoji" aria-label="smiling face with sunglasses">😎</span><span class="emoji" aria-label="zany face">🤪</span><span class="emoji" aria-label="locked with key">🔐</span><span class="emoji" aria-label="money-mouth face">🤑</span><span class="emoji" aria-label="question mark">❓</span></p>`)
+		`<p><span class="emoji" aria-label="smiling face with sunglasses">😎</span><span class="emoji" aria-label="zany face">🤪</span><span class="emoji" aria-label="locked with key">🔐</span><span class="emoji" aria-label="money-mouth face">🤑</span><span class="emoji" aria-label="red question mark">❓</span></p>`)
 
 	// should match nothing
 	test(

modules/markup/markdown/markdown_test.go

@@ -6,6 +6,7 @@ package markdown_test
 
 import (
 	"context"
+	"os"
 	"strings"
 	"testing"
 
@@ -37,6 +38,7 @@ func TestMain(m *testing.M) {
 	if err := git.InitSimple(context.Background()); err != nil {
 		log.Fatal("git init failed, err: %v", err)
 	}
+	os.Exit(m.Run())
 }
 
 func TestRender_StandardLinks(t *testing.T) {
@@ -426,3 +428,51 @@ func TestRenderEmojiInLinks_Issue12331(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, expected, res)
 }
+
+func TestMathBlock(t *testing.T) {
+	const nl = "\n"
+	testcases := []struct {
+		testcase string
+		expected string
+	}{
+		{
+			"$a$",
+			`<p><code class="language-math is-loading">a</code></p>` + nl,
+		},
+		{
+			"$ a $",
+			`<p><code class="language-math is-loading">a</code></p>` + nl,
+		},
+		{
+			"$a$ $b$",
+			`<p><code class="language-math is-loading">a</code> <code class="language-math is-loading">b</code></p>` + nl,
+		},
+		{
+			`\(a\) \(b\)`,
+			`<p><code class="language-math is-loading">a</code> <code class="language-math is-loading">b</code></p>` + nl,
+		},
+		{
+			`$a a$b b$`,
+			`<p><code class="language-math is-loading">a a$b b</code></p>` + nl,
+		},
+		{
+			`a a$b b`,
+			`<p>a a$b b</p>` + nl,
+		},
+		{
+			`a$b $a a$b b$`,
+			`<p>a$b <code class="language-math is-loading">a a$b b</code></p>` + nl,
+		},
+		{
+			"$$a$$",
+			`<pre class="code-block is-loading"><code class="chroma language-math display">a</code></pre>` + nl,
+		},
+	}
+
+	for _, test := range testcases {
+		res, err := RenderString(&markup.RenderContext{}, test.testcase)
+		assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase)
+		assert.Equal(t, test.expected, res, "Unexpected result in testcase %q", test.testcase)
+
+	}
+}

modules/markup/markdown/math/inline_parser.go

@@ -37,7 +37,7 @@ func NewInlineBracketParser() parser.InlineParser {
 	return defaultInlineBracketParser
 }
 
-// Trigger triggers this parser on $
+// Trigger triggers this parser on $ or \
 func (parser *inlineParser) Trigger() []byte {
 	return parser.start[0:1]
 }
@@ -50,29 +50,50 @@ func isAlphanumeric(b byte) bool {
 // Parse parses the current line and returns a result of parsing.
 func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node {
 	line, _ := block.PeekLine()
-	opener := bytes.Index(line, parser.start)
-	if opener < 0 {
+
+	if !bytes.HasPrefix(line, parser.start) {
+		// We'll catch this one on the next time round
 		return nil
 	}
-	if opener != 0 && isAlphanumeric(line[opener-1]) {
+
+	precedingCharacter := block.PrecendingCharacter()
+	if precedingCharacter < 256 && isAlphanumeric(byte(precedingCharacter)) {
+		// need to exclude things like `a$` from being considered a start
 		return nil
 	}
 
-	opener += len(parser.start)
-	ender := bytes.Index(line[opener:], parser.end)
-	if ender < 0 {
+	// move the opener marker point at the start of the text
+	opener := len(parser.start)
+
+	// Now look for an ending line
+	ender := opener
+	for {
+		pos := bytes.Index(line[ender:], parser.end)
+		if pos < 0 {
 			return nil
 		}
-	if len(line) > opener+ender+len(parser.end) && isAlphanumeric(line[opener+ender+len(parser.end)]) {
-		return nil
+
+		ender += pos
+
+		// Now we want to check the character at the end of our parser section
+		// that is ender + len(parser.end)
+		pos = ender + len(parser.end)
+		if len(line) <= pos {
+			break
+		}
+		if !isAlphanumeric(line[pos]) {
+			break
+		}
+		// move the pointer onwards
+		ender += len(parser.end)
 	}
 
 	block.Advance(opener)
 	_, pos := block.Position()
 	node := NewInline()
-	segment := pos.WithStop(pos.Start + ender)
+	segment := pos.WithStop(pos.Start + ender - opener)
 	node.AppendChild(node, ast.NewRawTextSegment(segment))
-	block.Advance(ender + len(parser.end))
+	block.Advance(ender - opener + len(parser.end))
 
 	trimBlock(node, block)
 	return node

modules/markup/markdown/meta.go

@@ -88,7 +88,9 @@ func ExtractMetadataBytes(contents []byte, out interface{}) ([]byte, error) {
 		line := contents[start:end]
 		if isYAMLSeparator(line) {
 			front = contents[frontMatterStart:start]
+			if end+1 < len(contents) {
 				body = contents[end+1:]
+			}
 			break
 		}
 	}

modules/markup/markdown/meta_test.go

@@ -61,7 +61,7 @@ func TestExtractMetadataBytes(t *testing.T) {
 		var meta structs.IssueTemplate
 		body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta)
 		assert.NoError(t, err)
-		assert.Equal(t, bodyTest, body)
+		assert.Equal(t, bodyTest, string(body))
 		assert.Equal(t, metaTest, meta)
 		assert.True(t, validateMetadata(meta))
 	})
@@ -82,7 +82,7 @@ func TestExtractMetadataBytes(t *testing.T) {
 		var meta structs.IssueTemplate
 		body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta)
 		assert.NoError(t, err)
-		assert.Equal(t, "", body)
+		assert.Equal(t, "", string(body))
 		assert.Equal(t, metaTest, meta)
 		assert.True(t, validateMetadata(meta))
 	})

modules/markup/markdown/renderconfig.go

@@ -5,10 +5,9 @@
 package markdown
 
 import (
+	"fmt"
 	"strings"
 
-	"code.gitea.io/gitea/modules/log"
-
 	"github.com/yuin/goldmark/ast"
 	"gopkg.in/yaml.v3"
 )
@@ -33,17 +32,13 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error {
 	}
 	rc.yamlNode = value
 
-	type basicRenderConfig struct {
-		Gitea *yaml.Node `yaml:"gitea"`
+	type commonRenderConfig struct {
 		TOC  bool   `yaml:"include_toc"`
 		Lang string `yaml:"lang"`
 	}
-
-	var basic basicRenderConfig
-
-	err := value.Decode(&basic)
-	if err != nil {
-		return err
+	var basic commonRenderConfig
+	if err := value.Decode(&basic); err != nil {
+		return fmt.Errorf("unable to decode into commonRenderConfig %w", err)
 	}
 
 	if basic.Lang != "" {
@@ -51,14 +46,16 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error {
 	}
 
 	rc.TOC = basic.TOC
-	if basic.Gitea == nil {
-		return nil
+
+	type controlStringRenderConfig struct {
+		Gitea string `yaml:"gitea"`
 	}
 
-	var control *string
-	if err := basic.Gitea.Decode(&control); err == nil && control != nil {
-		log.Info("control %v", control)
-		switch strings.TrimSpace(strings.ToLower(*control)) {
+	var stringBasic controlStringRenderConfig
+
+	if err := value.Decode(&stringBasic); err == nil {
+		if stringBasic.Gitea != "" {
+			switch strings.TrimSpace(strings.ToLower(stringBasic.Gitea)) {
 			case "none":
 				rc.Meta = "none"
 			case "table":
@@ -66,22 +63,31 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error {
 			default: // "details"
 				rc.Meta = "details"
 			}
+		}
 		return nil
 	}
 
 	type giteaControl struct {
-		Meta string     `yaml:"meta"`
-		Icon string     `yaml:"details_icon"`
-		TOC  *yaml.Node `yaml:"include_toc"`
-		Lang string     `yaml:"lang"`
+		Meta *string `yaml:"meta"`
+		Icon *string `yaml:"details_icon"`
+		TOC  *bool   `yaml:"include_toc"`
+		Lang *string `yaml:"lang"`
 	}
 
-	var controlStruct *giteaControl
-	if err := basic.Gitea.Decode(controlStruct); err != nil || controlStruct == nil {
-		return err
+	type complexGiteaConfig struct {
+		Gitea *giteaControl `yaml:"gitea"`
+	}
+	var complex complexGiteaConfig
+	if err := value.Decode(&complex); err != nil {
+		return fmt.Errorf("unable to decode into complexRenderConfig %w", err)
 	}
 
-	switch strings.TrimSpace(strings.ToLower(controlStruct.Meta)) {
+	if complex.Gitea == nil {
+		return nil
+	}
+
+	if complex.Gitea.Meta != nil {
+		switch strings.TrimSpace(strings.ToLower(*complex.Gitea.Meta)) {
 		case "none":
 			rc.Meta = "none"
 		case "table":
@@ -89,16 +95,18 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error {
 		default: // "details"
 			rc.Meta = "details"
 		}
+	}
 
-	rc.Icon = strings.TrimSpace(strings.ToLower(controlStruct.Icon))
+	if complex.Gitea.Icon != nil {
+		rc.Icon = strings.TrimSpace(strings.ToLower(*complex.Gitea.Icon))
+	}
 
-	if controlStruct.Lang != "" {
-		rc.Lang = controlStruct.Lang
+	if complex.Gitea.Lang != nil && *complex.Gitea.Lang != "" {
+		rc.Lang = *complex.Gitea.Lang
 	}
 
-	var toc bool
-	if err := controlStruct.TOC.Decode(&toc); err == nil {
-		rc.TOC = toc
+	if complex.Gitea.TOC != nil {
+		rc.TOC = *complex.Gitea.TOC
 	}
 
 	return nil

modules/markup/markdown/renderconfig_test.go

@@ -5,6 +5,7 @@
 package markdown
 
 import (
+	"strings"
 	"testing"
 
 	"gopkg.in/yaml.v3"
@@ -83,7 +84,7 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) {
 			}, `
 				include_toc: true
 				lang: testlang
-`,
+				`,
 		},
 		{
 			"complexlang", &RenderConfig{
@@ -93,7 +94,7 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) {
 			}, `
 				gitea:
 					lang: testlang
-`,
+				`,
 		},
 		{
 			"complexlang2", &RenderConfig{
@@ -140,8 +141,8 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) {
 				Icon: "table",
 				Lang: "",
 			}
-			if err := yaml.Unmarshal([]byte(tt.args), got); err != nil {
-				t.Errorf("RenderConfig.UnmarshalYAML() error = %v", err)
+			if err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", "    ")), got); err != nil {
+				t.Errorf("RenderConfig.UnmarshalYAML() error = %v\n%q", err, tt.args)
 				return
 			}