Fix copy/paste of empty lines (#19798)

* Fix copy/paste of empty newlines again

Fixes: https://github.com/go-gitea/gitea/issues/19331
Regressed by: https://github.com/go-gitea/gitea/pull/18270

Needed to do another newline addition to the Chroma output HTML to get
copy/paste work again. The previous replacement conditions are probably
obsolete, but as I'm not 100% sure, I opted to keep them.

Specifically, the Chroma HTML change mentioned in
https://github.com/go-gitea/gitea/pull/18270#issuecomment-1013350246
broke our previous newline replacement for such empty lines.

Also included are a few changes to make the test more pleasant to work
with.

* run go mod tidy

* add util.Dedent

* copy in the code

Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: Lauris BH <lauris@nix.lv>

modules/highlight/highlight.go

@@ -203,6 +203,8 @@ func File(numLines int, fileName, language string, code []byte) []string {
 			content = "\n"
 		} else if content == `</span><span class="w">` {
 			content += "\n</span>"
+		} else if content == `</span></span><span class="line"><span class="cl">` {
+			content += "\n"
 		}
 		content = strings.TrimSuffix(content, `<span class="w">`)
 		content = strings.TrimPrefix(content, `</span>`)

modules/highlight/highlight_test.go

@@ -5,11 +5,13 @@
 package highlight
 
 import (
-	"reflect"
+	"strings"
 	"testing"
 
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
 
+	"github.com/stretchr/testify/assert"
 	"gopkg.in/ini.v1"
 )
 
@@ -20,13 +22,14 @@ func TestFile(t *testing.T) {
 		numLines int
 		fileName string
 		code     string
-		want     []string
+		want     string
 	}{
 		{
 			name:     ".drone.yml",
 			numLines: 12,
 			fileName: ".drone.yml",
-			code: `kind: pipeline
+			code: util.Dedent(`
+				kind: pipeline
 				name: default
 
 				steps:
@@ -38,31 +41,29 @@ steps:
 					- go get -u
 					- go build -v
 					- go test -v -race -coverprofile=coverage.txt -covermode=atomic
-`,
+			`),
-			want: []string{
+			want: util.Dedent(`
-				`<span class="line"><span class="cl"><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l">pipeline</span>`,
+				<span class="line"><span class="cl"><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l">pipeline</span>
-				`</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">default</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">default</span>
-				`</span></span><span class="line"><span class="cl">`,
+				</span></span><span class="line"><span class="cl">
-				`</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">steps</span><span class="p">:</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">steps</span><span class="p">:</span>
-				`</span></span><span class="line"><span class="cl"><span class="w"></span>- <span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">test</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span>- <span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">test</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l">golang:1.13</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l">golang:1.13</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">environment</span><span class="p">:</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">environment</span><span class="p">:</span>
-				`</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w">		</span><span class="nt">GOPROXY</span><span class="p">:</span><span class="w"> </span><span class="l">https://goproxy.cn</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w">		</span><span class="nt">GOPROXY</span><span class="p">:</span><span class="w"> </span><span class="l">https://goproxy.cn</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">commands</span><span class="p">:</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">commands</span><span class="p">:</span>
-				`</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w">	</span>- <span class="l">go get -u</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w">	</span>- <span class="l">go get -u</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span>- <span class="l">go build -v</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w">	</span>- <span class="l">go build -v</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span>- <span class="l">go test -v -race -coverprofile=coverage.txt -covermode=atomic</span><span class="w">
+				</span></span><span class="line"><span class="cl"><span class="w">	</span>- <span class="l">go test -v -race -coverprofile=coverage.txt -covermode=atomic</span></span></span>
-</span></span></span>`,
+			`),
-				`<span class="w">
-</span>`,
-			},
 		},
 		{
 			name:     ".drone.yml - trailing space",
 			numLines: 13,
 			fileName: ".drone.yml",
-			code: `kind: pipeline
+			code: strings.Replace(util.Dedent(`
-name: default  ` + `
+				kind: pipeline
+				name: default
 
 				steps:
 				- name: test
@@ -73,30 +74,31 @@ steps:
 					- go get -u
 					- go build -v
 					- go test -v -race -coverprofile=coverage.txt -covermode=atomic
-	`,
+			`)+"\n", "name: default", "name: default  ", 1),
-			want: []string{
+			want: util.Dedent(`
-				`<span class="line"><span class="cl"><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l">pipeline</span>`,
+				<span class="line"><span class="cl"><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l">pipeline</span>
-				`</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">default  </span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">default  </span>
-				`</span></span><span class="line"><span class="cl">`,
+				</span></span><span class="line"><span class="cl">
-				`</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">steps</span><span class="p">:</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">steps</span><span class="p">:</span>
-				`</span></span><span class="line"><span class="cl"><span class="w"></span>- <span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">test</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span>- <span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">test</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l">golang:1.13</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l">golang:1.13</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">environment</span><span class="p">:</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">environment</span><span class="p">:</span>
-				`</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w">		</span><span class="nt">GOPROXY</span><span class="p">:</span><span class="w"> </span><span class="l">https://goproxy.cn</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w">		</span><span class="nt">GOPROXY</span><span class="p">:</span><span class="w"> </span><span class="l">https://goproxy.cn</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">commands</span><span class="p">:</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w">	</span><span class="nt">commands</span><span class="p">:</span>
-				`</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w">	</span>- <span class="l">go get -u</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w">	</span>- <span class="l">go get -u</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span>- <span class="l">go build -v</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w">	</span>- <span class="l">go build -v</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span>- <span class="l">go test -v -race -coverprofile=coverage.txt -covermode=atomic</span>`,
+				</span></span><span class="line"><span class="cl"><span class="w">	</span>- <span class="l">go test -v -race -coverprofile=coverage.txt -covermode=atomic</span>
-				`</span></span><span class="line"><span class="cl"><span class="w">	</span></span></span>`,
+				</span></span>
-			},
+				<span class="w">
+				</span>
+			`),
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := File(tt.numLines, tt.fileName, "", []byte(tt.code)); !reflect.DeepEqual(got, tt.want) {
+			got := strings.Join(File(tt.numLines, tt.fileName, "", []byte(tt.code)), "\n")
-				t.Errorf("File() = %v, want %v", got, tt.want)
+			assert.Equal(t, tt.want, got)
-			}
 		})
 	}
 }

modules/util/util.go

@@ -9,6 +9,7 @@ import (
 	"crypto/rand"
 	"errors"
 	"math/big"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -191,3 +192,35 @@ var titleCaser = cases.Title(language.English)
 func ToTitleCase(s string) string {
 	return titleCaser.String(s)
 }
+
+var (
+	whitespaceOnly    = regexp.MustCompile("(?m)^[ \t]+$")
+	leadingWhitespace = regexp.MustCompile("(?m)(^[ \t]*)(?:[^ \t\n])")
+)
+
+// Dedent removes common indentation of a multi-line string along with whitespace around it
+// Based on https://github.com/lithammer/dedent
+func Dedent(s string) string {
+	var margin string
+
+	s = whitespaceOnly.ReplaceAllString(s, "")
+	indents := leadingWhitespace.FindAllStringSubmatch(s, -1)
+
+	for i, indent := range indents {
+		if i == 0 {
+			margin = indent[1]
+		} else if strings.HasPrefix(indent[1], margin) {
+			continue
+		} else if strings.HasPrefix(margin, indent[1]) {
+			margin = indent[1]
+		} else {
+			margin = ""
+			break
+		}
+	}
+
+	if margin != "" {
+		s = regexp.MustCompile("(?m)^"+margin).ReplaceAllString(s, "")
+	}
+	return strings.TrimSpace(s)
+}

modules/util/util_test.go

@@ -225,3 +225,10 @@ func TestToTitleCase(t *testing.T) {
 	assert.Equal(t, ToTitleCase(`foo bar baz`), `Foo Bar Baz`)
 	assert.Equal(t, ToTitleCase(`FOO BAR BAZ`), `Foo Bar Baz`)
 }
+
+func TestDedent(t *testing.T) {
+	assert.Equal(t, Dedent(`
+		foo
+			bar
+	`), "foo\n\tbar")
+}